mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@gmail.com)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 // Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20
  21 namespace Mono.CSharp
  22 {
  23         /// <summary>
  24         ///    Tokenizer for C# source code.
  25         /// </summary>
  26
  27         public class Tokenizer : yyParser.yyInput
  28         {
  29                 class KeywordEntry<T>
  30                 {
  31                         public readonly T Token;
  32                         public KeywordEntry<T> Next;
  33                         public readonly char[] Value;
  34
  35                         public KeywordEntry (string value, T token)
  36                         {
  37                                 this.Value = value.ToCharArray ();
  38                                 this.Token = token;
  39                         }
  40                 }
  41
  42                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  43                 {
  44                         readonly int length;
  45
  46                         public IdentifiersComparer (int length)
  47                         {
  48                                 this.length = length;
  49                         }
  50
  51                         public bool Equals (char[] x, char[] y)
  52                         {
  53                                 for (int i = 0; i < length; ++i)
  54                                         if (x [i] != y [i])
  55                                                 return false;
  56
  57                                 return true;
  58                         }
  59
  60                         public int GetHashCode (char[] obj)
  61                         {
  62                                 int h = 0;
  63                                 for (int i = 0; i < length; ++i)
  64                                         h = (h << 5) - h + obj [i];
  65
  66                                 return h;
  67                         }
  68                 }
  69
  70                 //
  71                 // This class has to be used in the parser only, it reuses token
  72                 // details after each parse
  73                 //
  74                 public class LocatedToken
  75                 {
  76                         int row, column;
  77                         string value;
  78
  79                         static LocatedToken[] buffer = new LocatedToken[0];
  80                         static int pos;
  81
  82                         private LocatedToken ()
  83                         {
  84                         }
  85
  86                         public static LocatedToken Create (int row, int column)
  87                         {
  88                                 return Create (null, row, column);
  89                         }
  90
  91                         public static LocatedToken Create (string value, Location loc)
  92                         {
  93                                 return Create (value, loc.Row, loc.Column);
  94                         }
  95
  96                         public static LocatedToken Create (string value, int row, int column)
  97                         {
  98                                 //
  99                                 // TODO: I am not very happy about the logic but it's the best
 100                                 // what I could come up with for now.
 101                                 // Ideally we should be using just tiny buffer (256 elements) which
 102                                 // is enough to hold all details for currect stack and recycle elements
 103                                 // poped from the stack but there is a trick needed to recycle
 104                                 // them properly.
 105                                 //
 106                                 LocatedToken entry;
 107                                 if (pos >= buffer.Length) {
 108                                         entry = new LocatedToken ();
 109                                 } else {
 110                                         entry = buffer [pos];
 111                                         if (entry == null) {
 112                                                 entry = new LocatedToken ();
 113                                                 buffer [pos] = entry;
 114                                         }
 115
 116                                         ++pos;
 117                                 }
 118                                 entry.value = value;
 119                                 entry.row = row;
 120                                 entry.column = column;
 121                                 return entry;
 122                         }
 123
 124                         //
 125                         // Used for token not required by expression evaluator
 126                         //
 127                         [Conditional ("FULL_AST")]
 128                         public static void CreateOptional (int row, int col, ref object token)
 129                         {
 130                                 token = Create (row, col);
 131                         }
 132
 133                         public static void Initialize ()
 134                         {
 135 #if !FULL_AST
 136                                 if (buffer.Length == 0)
 137                                         buffer = new LocatedToken [15000];
 138 #endif
 139                                 pos = 0;
 140                         }
 141
 142                         public override string ToString ()
 143                         {
 144                                 return string.Format ("Token '{0}' at {1},{2}", Value, row, column);
 145                         }
 146
 147                         public Location Location {
 148                                 get { return new Location (row, column); }
 149                         }
 150
 151                         public string Value {
 152                                 get { return value; }
 153                         }
 154                 }
 155
 156                 public enum PreprocessorDirective
 157                 {
 158                         Invalid = 0,
 159
 160                         Region = 1,
 161                         Endregion = 2,
 162                         If = 3 | RequiresArgument,
 163                         Endif = 4,
 164                         Elif = 5 | RequiresArgument,
 165                         Else = 6,
 166                         Define = 7 | RequiresArgument,
 167                         Undef = 8 | RequiresArgument,
 168                         Error = 9,
 169                         Warning = 10,
 170                         Pragma = 11 | CustomArgumentsParsing,
 171                         Line = 12 | CustomArgumentsParsing,
 172
 173                         CustomArgumentsParsing = 1 << 10,
 174                         RequiresArgument = 1 << 11
 175                 }
 176
 177                 readonly SeekableStreamReader reader;
 178                 readonly CompilationSourceFile source_file;
 179                 readonly CompilerContext context;
 180
 181                 SourceFile current_source;
 182                 Location hidden_block_start;
 183                 int ref_line = 1;
 184                 int line = 1;
 185                 int col = 0;
 186                 int previous_col;
 187                 int current_token;
 188                 readonly int tab_size;
 189                 bool handle_get_set = false;
 190                 bool handle_remove_add = false;
 191                 bool handle_where = false;
 192                 bool handle_typeof = false;
 193                 bool lambda_arguments_parsing;
 194                 List<Location> escaped_identifiers;
 195                 int parsing_generic_less_than;
 196                 readonly bool doc_processing;
 197
 198                 //
 199                 // Used mainly for parser optimizations. Some expressions for instance
 200                 // can appear only in block (including initializer, base initializer)
 201                 // scope only
 202                 //
 203                 public int parsing_block;
 204                 internal bool query_parsing;
 205
 206                 //
 207                 // When parsing type only, useful for ambiguous nullable types
 208                 //
 209                 public int parsing_type;
 210
 211                 //
 212                 // Set when parsing generic declaration (type or method header)
 213                 //
 214                 public bool parsing_generic_declaration;
 215                 public bool parsing_generic_declaration_doc;
 216
 217                 //
 218                 // The value indicates that we have not reach any declaration or
 219                 // namespace yet
 220                 //
 221                 public int parsing_declaration;
 222
 223                 public bool parsing_attribute_section;
 224
 225                 public bool parsing_modifiers;
 226
 227                 //
 228                 // The special characters to inject on streams to run the unit parser
 229                 // in the special expression mode. Using private characters from
 230                 // Plane Sixteen (U+100000 to U+10FFFD)
 231                 //
 232                 // This character is only tested just before the tokenizer is about to report
 233                 // an error;   So on the regular operation mode, this addition will have no
 234                 // impact on the tokenizer's performance.
 235                 //
 236
 237                 public const int EvalStatementParserCharacter = 0x100000;
 238                 public const int EvalCompilationUnitParserCharacter = 0x100001;
 239                 public const int EvalUsingDeclarationsParserCharacter = 0x100002;
 240                 public const int DocumentationXref = 0x100003;
 241
 242                 //
 243                 // XML documentation buffer. The save point is used to divide
 244                 // comments on types and comments on members.
 245                 //
 246                 StringBuilder xml_comment_buffer;
 247
 248                 //
 249                 // See comment on XmlCommentState enumeration.
 250                 //
 251                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 252
 253                 //
 254                 // Whether tokens have been seen on this line
 255                 //
 256                 bool tokens_seen = false;
 257
 258                 //
 259                 // Set to true once the GENERATE_COMPLETION token has bee
 260                 // returned.   This helps produce one GENERATE_COMPLETION,
 261                 // as many COMPLETE_COMPLETION as necessary to complete the
 262                 // AST tree and one final EOF.
 263                 //
 264                 bool generated;
 265
 266                 //
 267                 // Whether a token has been seen on the file
 268                 // This is needed because `define' is not allowed to be used
 269                 // after a token has been seen.
 270                 //
 271                 bool any_token_seen;
 272
 273                 //
 274                 // Class variables
 275                 //
 276                 static readonly KeywordEntry<int>[][] keywords;
 277                 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 278                 static readonly HashSet<string> keyword_strings;
 279                 static readonly NumberStyles styles;
 280                 static readonly NumberFormatInfo csharp_format_info;
 281
 282                 // Pragma arguments
 283                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 284                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 285                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 286                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 287                 static readonly char[] line_hidden = "hidden".ToCharArray ();
 288                 static readonly char[] line_default = "default".ToCharArray ();
 289
 290                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 291
 292                 public bool PropertyParsing {
 293                         get { return handle_get_set; }
 294                         set { handle_get_set = value; }
 295                 }
 296
 297                 public bool EventParsing {
 298                         get { return handle_remove_add; }
 299                         set { handle_remove_add = value; }
 300                 }
 301
 302                 public bool ConstraintsParsing {
 303                         get { return handle_where; }
 304                         set { handle_where = value; }
 305                 }
 306
 307                 public bool TypeOfParsing {
 308                         get { return handle_typeof; }
 309                         set { handle_typeof = value; }
 310                 }
 311
 312                 public XmlCommentState doc_state {
 313                         get { return xml_doc_state; }
 314                         set {
 315                                 if (value == XmlCommentState.Allowed) {
 316                                         check_incorrect_doc_comment ();
 317                                         reset_doc_comment ();
 318                                 }
 319                                 xml_doc_state = value;
 320                         }
 321                 }
 322
 323                 //
 324                 // This is used to trigger completion generation on the parser
 325                 public bool CompleteOnEOF;
 326
 327                 void AddEscapedIdentifier (Location loc)
 328                 {
 329                         if (escaped_identifiers == null)
 330                                 escaped_identifiers = new List<Location> ();
 331
 332                         escaped_identifiers.Add (loc);
 333                 }
 334
 335                 public bool IsEscapedIdentifier (ATypeNameExpression name)
 336                 {
 337                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 338                 }
 339
 340                 //
 341                 // Values for the associated token returned
 342                 //
 343                 internal int putback_char;      // Used by repl only
 344                 object val;
 345
 346                 //
 347                 // Pre-processor
 348                 //
 349                 const int TAKING        = 1;
 350                 const int ELSE_SEEN     = 4;
 351                 const int PARENT_TAKING = 8;
 352                 const int REGION        = 16;
 353
 354                 //
 355                 // pre-processor if stack state:
 356                 //
 357                 Stack<int> ifstack;
 358
 359                 const int max_id_size = 512;
 360                 const int max_number_size = 512;
 361
 362 #if FULL_AST
 363                 readonly char [] id_builder = new char [max_id_size];
 364
 365                 Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 366
 367                 char [] number_builder = new char [max_number_size];
 368                 int number_pos;
 369
 370                 char[] value_builder = new char[256];
 371 #else
 372                 static readonly char [] id_builder = new char [max_id_size];
 373
 374                 static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 375
 376                 static char [] number_builder = new char [max_number_size];
 377                 static int number_pos;
 378
 379                 static char[] value_builder = new char[256];
 380 #endif
 381
 382                 public int Line {
 383                         get {
 384                                 return ref_line;
 385                         }
 386                 }
 387
 388                 //
 389                 // This is used when the tokenizer needs to save
 390                 // the current position as it needs to do some parsing
 391                 // on its own to deamiguate a token in behalf of the
 392                 // parser.
 393                 //
 394                 Stack<Position> position_stack = new Stack<Position> (2);
 395
 396                 class Position {
 397                         public int position;
 398                         public int line;
 399                         public int ref_line;
 400                         public int col;
 401                         public Location hidden;
 402                         public int putback_char;
 403                         public int previous_col;
 404                         public Stack<int> ifstack;
 405                         public int parsing_generic_less_than;
 406                         public int current_token;
 407                         public object val;
 408
 409                         public Position (Tokenizer t)
 410                         {
 411                                 position = t.reader.Position;
 412                                 line = t.line;
 413                                 ref_line = t.ref_line;
 414                                 col = t.col;
 415                                 hidden = t.hidden_block_start;
 416                                 putback_char = t.putback_char;
 417                                 previous_col = t.previous_col;
 418                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 419                                         // There is no simple way to clone Stack<T> all
 420                                         // methods reverse the order
 421                                         var clone = t.ifstack.ToArray ();
 422                                         Array.Reverse (clone);
 423                                         ifstack = new Stack<int> (clone);
 424                                 }
 425                                 parsing_generic_less_than = t.parsing_generic_less_than;
 426                                 current_token = t.current_token;
 427                                 val = t.val;
 428                         }
 429                 }
 430
 431                 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file)
 432                 {
 433                         this.source_file = file;
 434                         this.context = file.Compiler;
 435                         this.current_source = file.SourceFile;
 436
 437                         reader = input;
 438
 439                         putback_char = -1;
 440
 441                         xml_comment_buffer = new StringBuilder ();
 442                         doc_processing = context.Settings.DocumentationFile != null;
 443
 444                         tab_size = context.Settings.TabSize;
 445
 446                         Mono.CSharp.Location.Push (current_source);
 447                 }
 448
 449                 public void PushPosition ()
 450                 {
 451                         position_stack.Push (new Position (this));
 452                 }
 453
 454                 public void PopPosition ()
 455                 {
 456                         Position p = position_stack.Pop ();
 457
 458                         reader.Position = p.position;
 459                         ref_line = p.ref_line;
 460                         line = p.line;
 461                         col = p.col;
 462                         hidden_block_start = p.hidden;
 463                         putback_char = p.putback_char;
 464                         previous_col = p.previous_col;
 465                         ifstack = p.ifstack;
 466                         parsing_generic_less_than = p.parsing_generic_less_than;
 467                         current_token = p.current_token;
 468                         val = p.val;
 469                 }
 470
 471                 // Do not reset the position, ignore it.
 472                 public void DiscardPosition ()
 473                 {
 474                         position_stack.Pop ();
 475                 }
 476
 477                 static void AddKeyword (string kw, int token)
 478                 {
 479                         keyword_strings.Add (kw);
 480
 481                         AddKeyword (keywords, kw, token);
 482                 }
 483
 484                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 485                 {
 486                         AddKeyword (keywords_preprocessor, kw, directive);
 487                 }
 488
 489                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 490                 {
 491                         int length = kw.Length;
 492                         if (keywords[length] == null) {
 493                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 494                         }
 495
 496                         int char_index = kw[0] - '_';
 497                         var kwe = keywords[length][char_index];
 498                         if (kwe == null) {
 499                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 500                                 return;
 501                         }
 502
 503                         while (kwe.Next != null) {
 504                                 kwe = kwe.Next;
 505                         }
 506
 507                         kwe.Next = new KeywordEntry<T> (kw, token);
 508                 }
 509
 510                 //
 511                 // Class initializer
 512                 //
 513                 static Tokenizer ()
 514                 {
 515                         keyword_strings = new HashSet<string> ();
 516
 517                         // 11 is the length of the longest keyword for now
 518                         keywords = new KeywordEntry<int>[11][];
 519
 520                         AddKeyword ("__arglist", Token.ARGLIST);
 521                         AddKeyword ("__makeref", Token.MAKEREF);
 522                         AddKeyword ("__reftype", Token.REFTYPE);
 523                         AddKeyword ("__refvalue", Token.REFVALUE);
 524                         AddKeyword ("abstract", Token.ABSTRACT);
 525                         AddKeyword ("as", Token.AS);
 526                         AddKeyword ("add", Token.ADD);
 527                         AddKeyword ("base", Token.BASE);
 528                         AddKeyword ("bool", Token.BOOL);
 529                         AddKeyword ("break", Token.BREAK);
 530                         AddKeyword ("byte", Token.BYTE);
 531                         AddKeyword ("case", Token.CASE);
 532                         AddKeyword ("catch", Token.CATCH);
 533                         AddKeyword ("char", Token.CHAR);
 534                         AddKeyword ("checked", Token.CHECKED);
 535                         AddKeyword ("class", Token.CLASS);
 536                         AddKeyword ("const", Token.CONST);
 537                         AddKeyword ("continue", Token.CONTINUE);
 538                         AddKeyword ("decimal", Token.DECIMAL);
 539                         AddKeyword ("default", Token.DEFAULT);
 540                         AddKeyword ("delegate", Token.DELEGATE);
 541                         AddKeyword ("do", Token.DO);
 542                         AddKeyword ("double", Token.DOUBLE);
 543                         AddKeyword ("else", Token.ELSE);
 544                         AddKeyword ("enum", Token.ENUM);
 545                         AddKeyword ("event", Token.EVENT);
 546                         AddKeyword ("explicit", Token.EXPLICIT);
 547                         AddKeyword ("extern", Token.EXTERN);
 548                         AddKeyword ("false", Token.FALSE);
 549                         AddKeyword ("finally", Token.FINALLY);
 550                         AddKeyword ("fixed", Token.FIXED);
 551                         AddKeyword ("float", Token.FLOAT);
 552                         AddKeyword ("for", Token.FOR);
 553                         AddKeyword ("foreach", Token.FOREACH);
 554                         AddKeyword ("goto", Token.GOTO);
 555                         AddKeyword ("get", Token.GET);
 556                         AddKeyword ("if", Token.IF);
 557                         AddKeyword ("implicit", Token.IMPLICIT);
 558                         AddKeyword ("in", Token.IN);
 559                         AddKeyword ("int", Token.INT);
 560                         AddKeyword ("interface", Token.INTERFACE);
 561                         AddKeyword ("internal", Token.INTERNAL);
 562                         AddKeyword ("is", Token.IS);
 563                         AddKeyword ("lock", Token.LOCK);
 564                         AddKeyword ("long", Token.LONG);
 565                         AddKeyword ("namespace", Token.NAMESPACE);
 566                         AddKeyword ("new", Token.NEW);
 567                         AddKeyword ("null", Token.NULL);
 568                         AddKeyword ("object", Token.OBJECT);
 569                         AddKeyword ("operator", Token.OPERATOR);
 570                         AddKeyword ("out", Token.OUT);
 571                         AddKeyword ("override", Token.OVERRIDE);
 572                         AddKeyword ("params", Token.PARAMS);
 573                         AddKeyword ("private", Token.PRIVATE);
 574                         AddKeyword ("protected", Token.PROTECTED);
 575                         AddKeyword ("public", Token.PUBLIC);
 576                         AddKeyword ("readonly", Token.READONLY);
 577                         AddKeyword ("ref", Token.REF);
 578                         AddKeyword ("remove", Token.REMOVE);
 579                         AddKeyword ("return", Token.RETURN);
 580                         AddKeyword ("sbyte", Token.SBYTE);
 581                         AddKeyword ("sealed", Token.SEALED);
 582                         AddKeyword ("set", Token.SET);
 583                         AddKeyword ("short", Token.SHORT);
 584                         AddKeyword ("sizeof", Token.SIZEOF);
 585                         AddKeyword ("stackalloc", Token.STACKALLOC);
 586                         AddKeyword ("static", Token.STATIC);
 587                         AddKeyword ("string", Token.STRING);
 588                         AddKeyword ("struct", Token.STRUCT);
 589                         AddKeyword ("switch", Token.SWITCH);
 590                         AddKeyword ("this", Token.THIS);
 591                         AddKeyword ("throw", Token.THROW);
 592                         AddKeyword ("true", Token.TRUE);
 593                         AddKeyword ("try", Token.TRY);
 594                         AddKeyword ("typeof", Token.TYPEOF);
 595                         AddKeyword ("uint", Token.UINT);
 596                         AddKeyword ("ulong", Token.ULONG);
 597                         AddKeyword ("unchecked", Token.UNCHECKED);
 598                         AddKeyword ("unsafe", Token.UNSAFE);
 599                         AddKeyword ("ushort", Token.USHORT);
 600                         AddKeyword ("using", Token.USING);
 601                         AddKeyword ("virtual", Token.VIRTUAL);
 602                         AddKeyword ("void", Token.VOID);
 603                         AddKeyword ("volatile", Token.VOLATILE);
 604                         AddKeyword ("while", Token.WHILE);
 605                         AddKeyword ("partial", Token.PARTIAL);
 606                         AddKeyword ("where", Token.WHERE);
 607
 608                         // LINQ keywords
 609                         AddKeyword ("from", Token.FROM);
 610                         AddKeyword ("join", Token.JOIN);
 611                         AddKeyword ("on", Token.ON);
 612                         AddKeyword ("equals", Token.EQUALS);
 613                         AddKeyword ("select", Token.SELECT);
 614                         AddKeyword ("group", Token.GROUP);
 615                         AddKeyword ("by", Token.BY);
 616                         AddKeyword ("let", Token.LET);
 617                         AddKeyword ("orderby", Token.ORDERBY);
 618                         AddKeyword ("ascending", Token.ASCENDING);
 619                         AddKeyword ("descending", Token.DESCENDING);
 620                         AddKeyword ("into", Token.INTO);
 621
 622                         // Contextual async keywords
 623                         AddKeyword ("async", Token.ASYNC);
 624                         AddKeyword ("await", Token.AWAIT);
 625
 626                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 627
 628                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 629                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 630                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 631                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 632                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 633                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 634                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 635                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 636                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 637                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 638                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 639                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 640
 641                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 642                         styles = NumberStyles.Float;
 643                 }
 644
 645                 int GetKeyword (char[] id, int id_len)
 646                 {
 647                         //
 648                         // Keywords are stored in an array of arrays grouped by their
 649                         // length and then by the first character
 650                         //
 651                         if (id_len >= keywords.Length || keywords [id_len] == null)
 652                                 return -1;
 653
 654                         int first_index = id [0] - '_';
 655                         if (first_index > 'z' - '_')
 656                                 return -1;
 657
 658                         var kwe = keywords [id_len] [first_index];
 659                         if (kwe == null)
 660                                 return -1;
 661
 662                         int res;
 663                         do {
 664                                 res = kwe.Token;
 665                                 for (int i = 1; i < id_len; ++i) {
 666                                         if (id [i] != kwe.Value [i]) {
 667                                                 res = 0;
 668                                                 kwe = kwe.Next;
 669                                                 break;
 670                                         }
 671                                 }
 672                         } while (res == 0 && kwe != null);
 673
 674                         if (res == 0)
 675                                 return -1;
 676
 677                         int next_token;
 678                         switch (res) {
 679                         case Token.GET:
 680                         case Token.SET:
 681                                 if (!handle_get_set)
 682                                         res = -1;
 683                                 break;
 684                         case Token.REMOVE:
 685                         case Token.ADD:
 686                                 if (!handle_remove_add)
 687                                         res = -1;
 688                                 break;
 689                         case Token.EXTERN:
 690                                 if (parsing_declaration == 0)
 691                                         res = Token.EXTERN_ALIAS;
 692                                 break;
 693                         case Token.DEFAULT:
 694                                 if (peek_token () == Token.COLON) {
 695                                         token ();
 696                                         res = Token.DEFAULT_COLON;
 697                                 }
 698                                 break;
 699                         case Token.WHERE:
 700                                 if (!handle_where && !query_parsing)
 701                                         res = -1;
 702                                 break;
 703                         case Token.FROM:
 704                                 //
 705                                 // A query expression is any expression that starts with `from identifier'
 706                                 // followed by any token except ; , =
 707                                 //
 708                                 if (!query_parsing) {
 709                                         if (lambda_arguments_parsing) {
 710                                                 res = -1;
 711                                                 break;
 712                                         }
 713
 714                                         PushPosition ();
 715                                         // HACK: to disable generics micro-parser, because PushPosition does not
 716                                         // store identifiers array
 717                                         parsing_generic_less_than = 1;
 718                                         switch (xtoken ()) {
 719                                         case Token.IDENTIFIER:
 720                                         case Token.INT:
 721                                         case Token.BOOL:
 722                                         case Token.BYTE:
 723                                         case Token.CHAR:
 724                                         case Token.DECIMAL:
 725                                         case Token.FLOAT:
 726                                         case Token.LONG:
 727                                         case Token.OBJECT:
 728                                         case Token.STRING:
 729                                         case Token.UINT:
 730                                         case Token.ULONG:
 731                                                 next_token = xtoken ();
 732                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 733                                                         goto default;
 734
 735                                                 res = Token.FROM_FIRST;
 736                                                 query_parsing = true;
 737                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 738                                                         Report.FeatureIsNotAvailable (context, Location, "query expressions");
 739                                                 break;
 740                                         case Token.VOID:
 741                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 742                                                 break;
 743                                         default:
 744                                                 PopPosition ();
 745                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 746                                                 // which has been overwritten before we grabbed the identifier
 747                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 748                                                 return -1;
 749                                         }
 750                                         PopPosition ();
 751                                 }
 752                                 break;
 753                         case Token.JOIN:
 754                         case Token.ON:
 755                         case Token.EQUALS:
 756                         case Token.SELECT:
 757                         case Token.GROUP:
 758                         case Token.BY:
 759                         case Token.LET:
 760                         case Token.ORDERBY:
 761                         case Token.ASCENDING:
 762                         case Token.DESCENDING:
 763                         case Token.INTO:
 764                                 if (!query_parsing)
 765                                         res = -1;
 766                                 break;
 767
 768                         case Token.USING:
 769                         case Token.NAMESPACE:
 770                                 // TODO: some explanation needed
 771                                 check_incorrect_doc_comment ();
 772                                 break;
 773
 774                         case Token.PARTIAL:
 775                                 if (parsing_block > 0) {
 776                                         res = -1;
 777                                         break;
 778                                 }
 779
 780                                 // Save current position and parse next token.
 781                                 PushPosition ();
 782
 783                                 next_token = token ();
 784                                 bool ok = (next_token == Token.CLASS) ||
 785                                         (next_token == Token.STRUCT) ||
 786                                         (next_token == Token.INTERFACE) ||
 787                                         (next_token == Token.VOID);
 788
 789                                 PopPosition ();
 790
 791                                 if (ok) {
 792                                         if (next_token == Token.VOID) {
 793                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 794                                                         Report.FeatureIsNotAvailable (context, Location, "partial methods");
 795                                         } else if (context.Settings.Version == LanguageVersion.ISO_1)
 796                                                 Report.FeatureIsNotAvailable (context, Location, "partial types");
 797
 798                                         return res;
 799                                 }
 800
 801                                 if (next_token < Token.LAST_KEYWORD) {
 802                                         Report.Error (267, Location,
 803                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 804                                         return token ();
 805                                 }
 806
 807                                 res = -1;
 808                                 break;
 809
 810                         case Token.ASYNC:
 811                                 if (parsing_modifiers) {
 812                                         //
 813                                         // Skip attributes section or constructor called async
 814                                         //
 815                                         if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) {
 816                                                 res = -1;
 817                                         } else {
 818                                                 // async is keyword
 819                                         }
 820                                 } else if (parsing_block > 0) {
 821                                         switch (peek_token ()) {
 822                                         case Token.DELEGATE:
 823                                         case Token.OPEN_PARENS_LAMBDA:
 824                                                 // async is keyword
 825                                                 break;
 826                                         case Token.IDENTIFIER:
 827                                                 PushPosition ();
 828                                                 xtoken ();
 829                                                 if (xtoken () != Token.ARROW)
 830                                                         res = -1;
 831
 832                                                 PopPosition ();
 833                                                 break;
 834                                         default:
 835                                                 res = -1;
 836                                                 break;
 837                                         }
 838                                 } else {
 839                                         res = -1;
 840                                 }
 841
 842                                 if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) {
 843                                         Report.FeatureIsNotAvailable (context, Location, "asynchronous functions");
 844                                 }
 845
 846                                 break;
 847
 848                         case Token.AWAIT:
 849                                 if (parsing_block == 0)
 850                                         res = -1;
 851
 852                                 break;
 853                         }
 854
 855
 856                         return res;
 857                 }
 858
 859                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 860                 {
 861                         //
 862                         // Keywords are stored in an array of arrays grouped by their
 863                         // length and then by the first character
 864                         //
 865                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 866                                 return PreprocessorDirective.Invalid;
 867
 868                         int first_index = id[0] - '_';
 869                         if (first_index > 'z' - '_')
 870                                 return PreprocessorDirective.Invalid;
 871
 872                         var kwe = keywords_preprocessor[id_len][first_index];
 873                         if (kwe == null)
 874                                 return PreprocessorDirective.Invalid;
 875
 876                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 877                         do {
 878                                 res = kwe.Token;
 879                                 for (int i = 1; i < id_len; ++i) {
 880                                         if (id[i] != kwe.Value[i]) {
 881                                                 res = 0;
 882                                                 kwe = kwe.Next;
 883                                                 break;
 884                                         }
 885                                 }
 886                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 887
 888                         return res;
 889                 }
 890
 891                 public Location Location {
 892                         get {
 893                                 return new Location (ref_line, col);
 894                         }
 895                 }
 896
 897                 static bool is_identifier_start_character (int c)
 898                 {
 899                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 900                 }
 901
 902                 static bool is_identifier_part_character (char c)
 903                 {
 904                         if (c >= 'a' && c <= 'z')
 905                                 return true;
 906
 907                         if (c >= 'A' && c <= 'Z')
 908                                 return true;
 909
 910                         if (c == '_' || (c >= '0' && c <= '9'))
 911                                 return true;
 912
 913                         if (c < 0x80)
 914                                 return false;
 915
 916                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 917                 }
 918
 919                 public static bool IsKeyword (string s)
 920                 {
 921                         return keyword_strings.Contains (s);
 922                 }
 923
 924                 //
 925                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 926                 //
 927                 int TokenizeOpenParens ()
 928                 {
 929                         int ptoken;
 930                         current_token = -1;
 931
 932                         int bracket_level = 0;
 933                         bool is_type = false;
 934                         bool can_be_type = false;
 935
 936                         while (true) {
 937                                 ptoken = current_token;
 938                                 token ();
 939
 940                                 switch (current_token) {
 941                                 case Token.CLOSE_PARENS:
 942                                         token ();
 943
 944                                         //
 945                                         // Expression inside parens is lambda, (int i) =>
 946                                         //
 947                                         if (current_token == Token.ARROW)
 948                                                 return Token.OPEN_PARENS_LAMBDA;
 949
 950                                         //
 951                                         // Expression inside parens is single type, (int[])
 952                                         //
 953                                         if (is_type)
 954                                                 return Token.OPEN_PARENS_CAST;
 955
 956                                         //
 957                                         // Expression is possible cast, look at next token, (T)null
 958                                         //
 959                                         if (can_be_type) {
 960                                                 switch (current_token) {
 961                                                 case Token.OPEN_PARENS:
 962                                                 case Token.BANG:
 963                                                 case Token.TILDE:
 964                                                 case Token.IDENTIFIER:
 965                                                 case Token.LITERAL:
 966                                                 case Token.BASE:
 967                                                 case Token.CHECKED:
 968                                                 case Token.DELEGATE:
 969                                                 case Token.FALSE:
 970                                                 case Token.FIXED:
 971                                                 case Token.NEW:
 972                                                 case Token.NULL:
 973                                                 case Token.SIZEOF:
 974                                                 case Token.THIS:
 975                                                 case Token.THROW:
 976                                                 case Token.TRUE:
 977                                                 case Token.TYPEOF:
 978                                                 case Token.UNCHECKED:
 979                                                 case Token.UNSAFE:
 980                                                 case Token.DEFAULT:
 981                                                 case Token.AWAIT:
 982
 983                                                 //
 984                                                 // These can be part of a member access
 985                                                 //
 986                                                 case Token.INT:
 987                                                 case Token.UINT:
 988                                                 case Token.SHORT:
 989                                                 case Token.USHORT:
 990                                                 case Token.LONG:
 991                                                 case Token.ULONG:
 992                                                 case Token.DOUBLE:
 993                                                 case Token.FLOAT:
 994                                                 case Token.CHAR:
 995                                                 case Token.BYTE:
 996                                                 case Token.DECIMAL:
 997                                                 case Token.BOOL:
 998                                                         return Token.OPEN_PARENS_CAST;
 999                                                 }
1000                                         }
1001                                         return Token.OPEN_PARENS;
1002
1003                                 case Token.DOT:
1004                                 case Token.DOUBLE_COLON:
1005                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
1006                                                 goto default;
1007
1008                                         continue;
1009
1010                                 case Token.IDENTIFIER:
1011                                         switch (ptoken) {
1012                                         case Token.DOT:
1013                                                 if (bracket_level == 0) {
1014                                                         is_type = false;
1015                                                         can_be_type = true;
1016                                                 }
1017
1018                                                 continue;
1019                                         case Token.OP_GENERICS_LT:
1020                                         case Token.COMMA:
1021                                         case Token.DOUBLE_COLON:
1022                                         case -1:
1023                                                 if (bracket_level == 0)
1024                                                         can_be_type = true;
1025                                                 continue;
1026                                         default:
1027                                                 can_be_type = is_type = false;
1028                                                 continue;
1029                                         }
1030
1031                                 case Token.OBJECT:
1032                                 case Token.STRING:
1033                                 case Token.BOOL:
1034                                 case Token.DECIMAL:
1035                                 case Token.FLOAT:
1036                                 case Token.DOUBLE:
1037                                 case Token.SBYTE:
1038                                 case Token.BYTE:
1039                                 case Token.SHORT:
1040                                 case Token.USHORT:
1041                                 case Token.INT:
1042                                 case Token.UINT:
1043                                 case Token.LONG:
1044                                 case Token.ULONG:
1045                                 case Token.CHAR:
1046                                 case Token.VOID:
1047                                         if (bracket_level == 0)
1048                                                 is_type = true;
1049                                         continue;
1050
1051                                 case Token.COMMA:
1052                                         if (bracket_level == 0) {
1053                                                 bracket_level = 100;
1054                                                 can_be_type = is_type = false;
1055                                         }
1056                                         continue;
1057
1058                                 case Token.OP_GENERICS_LT:
1059                                 case Token.OPEN_BRACKET:
1060                                         if (bracket_level++ == 0)
1061                                                 is_type = true;
1062                                         continue;
1063
1064                                 case Token.OP_GENERICS_GT:
1065                                 case Token.CLOSE_BRACKET:
1066                                         --bracket_level;
1067                                         continue;
1068
1069                                 case Token.INTERR_NULLABLE:
1070                                 case Token.STAR:
1071                                         if (bracket_level == 0)
1072                                                 is_type = true;
1073                                         continue;
1074
1075                                 case Token.REF:
1076                                 case Token.OUT:
1077                                         can_be_type = is_type = false;
1078                                         continue;
1079
1080                                 default:
1081                                         return Token.OPEN_PARENS;
1082                                 }
1083                         }
1084                 }
1085
1086                 public static bool IsValidIdentifier (string s)
1087                 {
1088                         if (s == null || s.Length == 0)
1089                                 return false;
1090
1091                         if (!is_identifier_start_character (s [0]))
1092                                 return false;
1093
1094                         for (int i = 1; i < s.Length; i ++)
1095                                 if (! is_identifier_part_character (s [i]))
1096                                         return false;
1097
1098                         return true;
1099                 }
1100
1101                 bool parse_less_than ()
1102                 {
1103                 start:
1104                         int the_token = token ();
1105                         if (the_token == Token.OPEN_BRACKET) {
1106                                 do {
1107                                         the_token = token ();
1108                                 } while (the_token != Token.CLOSE_BRACKET);
1109                                 the_token = token ();
1110                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1111                                 the_token = token ();
1112                         }
1113                         switch (the_token) {
1114                         case Token.IDENTIFIER:
1115                         case Token.OBJECT:
1116                         case Token.STRING:
1117                         case Token.BOOL:
1118                         case Token.DECIMAL:
1119                         case Token.FLOAT:
1120                         case Token.DOUBLE:
1121                         case Token.SBYTE:
1122                         case Token.BYTE:
1123                         case Token.SHORT:
1124                         case Token.USHORT:
1125                         case Token.INT:
1126                         case Token.UINT:
1127                         case Token.LONG:
1128                         case Token.ULONG:
1129                         case Token.CHAR:
1130                         case Token.VOID:
1131                                 break;
1132                         case Token.OP_GENERICS_GT:
1133                         case Token.IN:
1134                         case Token.OUT:
1135                                 return true;
1136
1137                         default:
1138                                 return false;
1139                         }
1140                 again:
1141                         the_token = token ();
1142
1143                         if (the_token == Token.OP_GENERICS_GT)
1144                                 return true;
1145                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1146                                 goto start;
1147                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1148                                 goto again;
1149                         else if (the_token == Token.OP_GENERICS_LT) {
1150                                 if (!parse_less_than ())
1151                                         return false;
1152                                 goto again;
1153                         } else if (the_token == Token.OPEN_BRACKET) {
1154                         rank_specifiers:
1155                                 the_token = token ();
1156                                 if (the_token == Token.CLOSE_BRACKET)
1157                                         goto again;
1158                                 else if (the_token == Token.COMMA)
1159                                         goto rank_specifiers;
1160                                 return false;
1161                         }
1162
1163                         return false;
1164                 }
1165
1166                 bool parse_generic_dimension (out int dimension)
1167                 {
1168                         dimension = 1;
1169
1170                 again:
1171                         int the_token = token ();
1172                         if (the_token == Token.OP_GENERICS_GT)
1173                                 return true;
1174                         else if (the_token == Token.COMMA) {
1175                                 dimension++;
1176                                 goto again;
1177                         }
1178
1179                         return false;
1180                 }
1181
1182                 public int peek_token ()
1183                 {
1184                         int the_token;
1185
1186                         PushPosition ();
1187                         the_token = token ();
1188                         PopPosition ();
1189
1190                         return the_token;
1191                 }
1192
1193                 //
1194                 // Tonizes `?' using custom disambiguous rules to return one
1195                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1196                 //
1197                 // Tricky expression look like:
1198                 //
1199                 // Foo ? a = x ? b : c;
1200                 //
1201                 int TokenizePossibleNullableType ()
1202                 {
1203                         if (parsing_block == 0 || parsing_type > 0)
1204                                 return Token.INTERR_NULLABLE;
1205
1206                         int d = peek_char ();
1207                         if (d == '?') {
1208                                 get_char ();
1209                                 return Token.OP_COALESCING;
1210                         }
1211
1212                         switch (current_token) {
1213                         case Token.CLOSE_PARENS:
1214                         case Token.TRUE:
1215                         case Token.FALSE:
1216                         case Token.NULL:
1217                         case Token.LITERAL:
1218                                 return Token.INTERR;
1219                         }
1220
1221                         if (d != ' ') {
1222                                 if (d == ',' || d == ';' || d == '>')
1223                                         return Token.INTERR_NULLABLE;
1224                                 if (d == '*' || (d >= '0' && d <= '9'))
1225                                         return Token.INTERR;
1226                         }
1227
1228                         PushPosition ();
1229                         current_token = Token.NONE;
1230                         int next_token;
1231                         switch (xtoken ()) {
1232                         case Token.LITERAL:
1233                         case Token.TRUE:
1234                         case Token.FALSE:
1235                         case Token.NULL:
1236                         case Token.THIS:
1237                         case Token.NEW:
1238                                 next_token = Token.INTERR;
1239                                 break;
1240
1241                         case Token.SEMICOLON:
1242                         case Token.COMMA:
1243                         case Token.CLOSE_PARENS:
1244                         case Token.OPEN_BRACKET:
1245                         case Token.OP_GENERICS_GT:
1246                         case Token.INTERR:
1247                                 next_token = Token.INTERR_NULLABLE;
1248                                 break;
1249
1250                         default:
1251                                 next_token = -1;
1252                                 break;
1253                         }
1254
1255                         if (next_token == -1) {
1256                                 switch (xtoken ()) {
1257                                 case Token.COMMA:
1258                                 case Token.SEMICOLON:
1259                                 case Token.OPEN_BRACE:
1260                                 case Token.CLOSE_PARENS:
1261                                 case Token.IN:
1262                                         next_token = Token.INTERR_NULLABLE;
1263                                         break;
1264
1265                                 case Token.COLON:
1266                                         next_token = Token.INTERR;
1267                                         break;
1268
1269                                 default:
1270                                         int ntoken;
1271                                         int interrs = 1;
1272                                         int colons = 0;
1273                                         int braces = 0;
1274                                         //
1275                                         // All shorcuts failed, do it hard way
1276                                         //
1277                                         while ((ntoken = xtoken ()) != Token.EOF) {
1278                                                 if (ntoken == Token.OPEN_BRACE) {
1279                                                         ++braces;
1280                                                         continue;
1281                                                 }
1282
1283                                                 if (ntoken == Token.CLOSE_BRACE) {
1284                                                         --braces;
1285                                                         continue;
1286                                                 }
1287
1288                                                 if (braces != 0)
1289                                                         continue;
1290
1291                                                 if (ntoken == Token.SEMICOLON)
1292                                                         break;
1293
1294                                                 if (ntoken == Token.COLON) {
1295                                                         if (++colons == interrs)
1296                                                                 break;
1297                                                         continue;
1298                                                 }
1299
1300                                                 if (ntoken == Token.INTERR) {
1301                                                         ++interrs;
1302                                                         continue;
1303                                                 }
1304                                         }
1305
1306                                         next_token = colons != interrs && braces == 0 ? Token.INTERR_NULLABLE : Token.INTERR;
1307                                         break;
1308                                 }
1309                         }
1310
1311                         PopPosition ();
1312                         return next_token;
1313                 }
1314
1315                 bool decimal_digits (int c)
1316                 {
1317                         int d;
1318                         bool seen_digits = false;
1319
1320                         if (c != -1){
1321                                 if (number_pos == max_number_size)
1322                                         Error_NumericConstantTooLong ();
1323                                 number_builder [number_pos++] = (char) c;
1324                         }
1325
1326                         //
1327                         // We use peek_char2, because decimal_digits needs to do a
1328                         // 2-character look-ahead (5.ToString for example).
1329                         //
1330                         while ((d = peek_char2 ()) != -1){
1331                                 if (d >= '0' && d <= '9'){
1332                                         if (number_pos == max_number_size)
1333                                                 Error_NumericConstantTooLong ();
1334                                         number_builder [number_pos++] = (char) d;
1335                                         get_char ();
1336                                         seen_digits = true;
1337                                 } else
1338                                         break;
1339                         }
1340
1341                         return seen_digits;
1342                 }
1343
1344                 static bool is_hex (int e)
1345                 {
1346                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1347                 }
1348
1349                 static TypeCode real_type_suffix (int c)
1350                 {
1351                         switch (c){
1352                         case 'F': case 'f':
1353                                 return TypeCode.Single;
1354                         case 'D': case 'd':
1355                                 return TypeCode.Double;
1356                         case 'M': case 'm':
1357                                 return TypeCode.Decimal;
1358                         default:
1359                                 return TypeCode.Empty;
1360                         }
1361                 }
1362
1363                 ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc)
1364                 {
1365                         bool is_unsigned = false;
1366                         bool is_long = false;
1367
1368                         if (c != -1){
1369                                 bool scanning = true;
1370                                 do {
1371                                         switch (c){
1372                                         case 'U': case 'u':
1373                                                 if (is_unsigned)
1374                                                         scanning = false;
1375                                                 is_unsigned = true;
1376                                                 get_char ();
1377                                                 break;
1378
1379                                         case 'l':
1380                                                 if (!is_unsigned){
1381                                                         //
1382                                                         // if we have not seen anything in between
1383                                                         // report this error
1384                                                         //
1385                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1386                                                 }
1387
1388                                                 goto case 'L';
1389
1390                                         case 'L':
1391                                                 if (is_long)
1392                                                         scanning = false;
1393                                                 is_long = true;
1394                                                 get_char ();
1395                                                 break;
1396
1397                                         default:
1398                                                 scanning = false;
1399                                                 break;
1400                                         }
1401                                         c = peek_char ();
1402                                 } while (scanning);
1403                         }
1404
1405                         if (is_long && is_unsigned){
1406                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1407                         }
1408
1409                         if (is_unsigned){
1410                                 // uint if possible, or ulong else.
1411
1412                                 if ((ul & 0xffffffff00000000) == 0)
1413                                         return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc);
1414                                 else
1415                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1416                         } else if (is_long){
1417                                 // long if possible, ulong otherwise
1418                                 if ((ul & 0x8000000000000000) != 0)
1419                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1420                                 else
1421                                         return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1422                         } else {
1423                                 // int, uint, long or ulong in that order
1424                                 if ((ul & 0xffffffff00000000) == 0){
1425                                         uint ui = (uint) ul;
1426
1427                                         if ((ui & 0x80000000) != 0)
1428                                                 return new UIntLiteral (context.BuiltinTypes, ui, loc);
1429                                         else
1430                                                 return new IntLiteral (context.BuiltinTypes, (int) ui, loc);
1431                                 } else {
1432                                         if ((ul & 0x8000000000000000) != 0)
1433                                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1434                                         else
1435                                                 return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1436                                 }
1437                         }
1438                 }
1439
1440                 //
1441                 // given `c' as the next char in the input decide whether
1442                 // we need to convert to a special type, and then choose
1443                 // the best representation for the integer
1444                 //
1445                 ILiteralConstant adjust_int (int c, Location loc)
1446                 {
1447                         try {
1448                                 if (number_pos > 9){
1449                                         ulong ul = (uint) (number_builder [0] - '0');
1450
1451                                         for (int i = 1; i < number_pos; i++){
1452                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1453                                         }
1454
1455                                         return integer_type_suffix (ul, c, loc);
1456                                 } else {
1457                                         uint ui = (uint) (number_builder [0] - '0');
1458
1459                                         for (int i = 1; i < number_pos; i++){
1460                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1461                                         }
1462
1463                                         return integer_type_suffix (ui, c, loc);
1464                                 }
1465                         } catch (OverflowException) {
1466                                 Error_NumericConstantTooLong ();
1467                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1468                         }
1469                         catch (FormatException) {
1470                                 Report.Error (1013, Location, "Invalid number");
1471                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1472                         }
1473                 }
1474
1475                 ILiteralConstant adjust_real (TypeCode t, Location loc)
1476                 {
1477                         string s = new string (number_builder, 0, number_pos);
1478                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1479
1480                         switch (t){
1481                         case TypeCode.Decimal:
1482                                 try {
1483                                         return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc);
1484                                 } catch (OverflowException) {
1485                                         Report.Error (594, Location, error_details, "decimal");
1486                                         return new DecimalLiteral (context.BuiltinTypes, 0, loc);
1487                                 }
1488                         case TypeCode.Single:
1489                                 try {
1490                                         return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc);
1491                                 } catch (OverflowException) {
1492                                         Report.Error (594, Location, error_details, "float");
1493                                         return new FloatLiteral (context.BuiltinTypes, 0, loc);
1494                                 }
1495                         default:
1496                                 try {
1497                                         return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc);
1498                                 } catch (OverflowException) {
1499                                         Report.Error (594, loc, error_details, "double");
1500                                         return new DoubleLiteral (context.BuiltinTypes, 0, loc);
1501                                 }
1502                         }
1503                 }
1504
1505                 ILiteralConstant handle_hex (Location loc)
1506                 {
1507                         int d;
1508                         ulong ul;
1509
1510                         get_char ();
1511                         while ((d = peek_char ()) != -1){
1512                                 if (is_hex (d)){
1513                                         number_builder [number_pos++] = (char) d;
1514                                         get_char ();
1515                                 } else
1516                                         break;
1517                         }
1518
1519                         string s = new String (number_builder, 0, number_pos);
1520
1521                         try {
1522                                 if (number_pos <= 8)
1523                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1524                                 else
1525                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1526
1527                                 return integer_type_suffix (ul, peek_char (), loc);
1528                         } catch (OverflowException){
1529                                 Error_NumericConstantTooLong ();
1530                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1531                         }
1532                         catch (FormatException) {
1533                                 Report.Error (1013, Location, "Invalid number");
1534                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1535                         }
1536                 }
1537
1538                 //
1539                 // Invoked if we know we have .digits or digits
1540                 //
1541                 int is_number (int c)
1542                 {
1543                         ILiteralConstant res;
1544
1545 #if FULL_AST
1546                         int read_start = reader.Position - 1;
1547                         if (c == '.') {
1548                                 //
1549                                 // Caller did peek_char
1550                                 //
1551                                 --read_start;
1552                         }
1553 #endif
1554                         number_pos = 0;
1555                         var loc = Location;
1556
1557                         if (c >= '0' && c <= '9'){
1558                                 if (c == '0'){
1559                                         int peek = peek_char ();
1560
1561                                         if (peek == 'x' || peek == 'X') {
1562                                                 val = res = handle_hex (loc);
1563 #if FULL_AST
1564                                                 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1565 #endif
1566
1567                                                 return Token.LITERAL;
1568                                         }
1569                                 }
1570                                 decimal_digits (c);
1571                                 c = get_char ();
1572                         }
1573
1574                         //
1575                         // We need to handle the case of
1576                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1577                         //
1578                         bool is_real = false;
1579                         if (c == '.'){
1580                                 if (decimal_digits ('.')){
1581                                         is_real = true;
1582                                         c = get_char ();
1583                                 } else {
1584                                         putback ('.');
1585                                         number_pos--;
1586                                         val = res = adjust_int (-1, loc);
1587
1588 #if FULL_AST
1589                                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1590 #endif
1591                                         return Token.LITERAL;
1592                                 }
1593                         }
1594
1595                         if (c == 'e' || c == 'E'){
1596                                 is_real = true;
1597                                 if (number_pos == max_number_size)
1598                                         Error_NumericConstantTooLong ();
1599                                 number_builder [number_pos++] = (char) c;
1600                                 c = get_char ();
1601
1602                                 if (c == '+'){
1603                                         if (number_pos == max_number_size)
1604                                                 Error_NumericConstantTooLong ();
1605                                         number_builder [number_pos++] = '+';
1606                                         c = -1;
1607                                 } else if (c == '-') {
1608                                         if (number_pos == max_number_size)
1609                                                 Error_NumericConstantTooLong ();
1610                                         number_builder [number_pos++] = '-';
1611                                         c = -1;
1612                                 } else {
1613                                         if (number_pos == max_number_size)
1614                                                 Error_NumericConstantTooLong ();
1615                                         number_builder [number_pos++] = '+';
1616                                 }
1617
1618                                 decimal_digits (c);
1619                                 c = get_char ();
1620                         }
1621
1622                         var type = real_type_suffix (c);
1623                         if (type == TypeCode.Empty && !is_real) {
1624                                 putback (c);
1625                                 res = adjust_int (c, loc);
1626                         } else {
1627                                 is_real = true;
1628
1629                                 if (type == TypeCode.Empty) {
1630                                         putback (c);
1631                                 }
1632
1633                                 res = adjust_real (type, loc);
1634                         }
1635
1636                         val = res;
1637
1638 #if FULL_AST
1639                         var chars = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty && c > 0 ? 1 : 0));
1640                         if (chars[chars.Length - 1] == '\r')
1641                                 Array.Resize (ref chars, chars.Length - 1);
1642                         res.ParsedValue = chars;
1643 #endif
1644
1645                         return Token.LITERAL;
1646                 }
1647
1648                 //
1649                 // Accepts exactly count (4 or 8) hex, no more no less
1650                 //
1651                 int getHex (int count, out int surrogate, out bool error)
1652                 {
1653                         int i;
1654                         int total = 0;
1655                         int c;
1656                         int top = count != -1 ? count : 4;
1657
1658                         get_char ();
1659                         error = false;
1660                         surrogate = 0;
1661                         for (i = 0; i < top; i++){
1662                                 c = get_char ();
1663
1664                                 if (c >= '0' && c <= '9')
1665                                         c = (int) c - (int) '0';
1666                                 else if (c >= 'A' && c <= 'F')
1667                                         c = (int) c - (int) 'A' + 10;
1668                                 else if (c >= 'a' && c <= 'f')
1669                                         c = (int) c - (int) 'a' + 10;
1670                                 else {
1671                                         error = true;
1672                                         return 0;
1673                                 }
1674
1675                                 total = (total * 16) + c;
1676                                 if (count == -1){
1677                                         int p = peek_char ();
1678                                         if (p == -1)
1679                                                 break;
1680                                         if (!is_hex ((char)p))
1681                                                 break;
1682                                 }
1683                         }
1684
1685                         if (top == 8) {
1686                                 if (total > 0x0010FFFF) {
1687                                         error = true;
1688                                         return 0;
1689                                 }
1690
1691                                 if (total >= 0x00010000) {
1692                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1693                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1694                                 }
1695                         }
1696
1697                         return total;
1698                 }
1699
1700                 int escape (int c, out int surrogate)
1701                 {
1702                         bool error;
1703                         int d;
1704                         int v;
1705
1706                         d = peek_char ();
1707                         if (c != '\\') {
1708                                 surrogate = 0;
1709                                 return c;
1710                         }
1711
1712                         switch (d){
1713                         case 'a':
1714                                 v = '\a'; break;
1715                         case 'b':
1716                                 v = '\b'; break;
1717                         case 'n':
1718                                 v = '\n'; break;
1719                         case 't':
1720                                 v = '\t'; break;
1721                         case 'v':
1722                                 v = '\v'; break;
1723                         case 'r':
1724                                 v = '\r'; break;
1725                         case '\\':
1726                                 v = '\\'; break;
1727                         case 'f':
1728                                 v = '\f'; break;
1729                         case '0':
1730                                 v = 0; break;
1731                         case '"':
1732                                 v = '"'; break;
1733                         case '\'':
1734                                 v = '\''; break;
1735                         case 'x':
1736                                 v = getHex (-1, out surrogate, out error);
1737                                 if (error)
1738                                         goto default;
1739                                 return v;
1740                         case 'u':
1741                         case 'U':
1742                                 return EscapeUnicode (d, out surrogate);
1743                         default:
1744                                 surrogate = 0;
1745                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1746                                 return d;
1747                         }
1748
1749                         get_char ();
1750                         surrogate = 0;
1751                         return v;
1752                 }
1753
1754                 int EscapeUnicode (int ch, out int surrogate)
1755                 {
1756                         bool error;
1757                         if (ch == 'U') {
1758                                 ch = getHex (8, out surrogate, out error);
1759                         } else {
1760                                 ch = getHex (4, out surrogate, out error);
1761                         }
1762
1763                         if (error)
1764                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1765
1766                         return ch;
1767                 }
1768
1769                 int get_char ()
1770                 {
1771                         int x;
1772                         if (putback_char != -1) {
1773                                 x = putback_char;
1774                                 putback_char = -1;
1775                         } else {
1776                                 x = reader.Read ();
1777                         }
1778
1779                         if (x == '\r') {
1780                                 if (peek_char () == '\n') {
1781                                         putback_char = -1;
1782                                 }
1783
1784                                 x = '\n';
1785                                 advance_line ();
1786                         } else if (x == '\n') {
1787                                 advance_line ();
1788                         } else {
1789                                 col++;
1790                         }
1791                         return x;
1792                 }
1793
1794                 void advance_line ()
1795                 {
1796                         line++;
1797                         ref_line++;
1798                         previous_col = col;
1799                         col = 0;
1800                 }
1801
1802                 int peek_char ()
1803                 {
1804                         if (putback_char == -1)
1805                                 putback_char = reader.Read ();
1806                         return putback_char;
1807                 }
1808
1809                 int peek_char2 ()
1810                 {
1811                         if (putback_char != -1)
1812                                 return putback_char;
1813                         return reader.Peek ();
1814                 }
1815
1816                 public void putback (int c)
1817                 {
1818                         if (putback_char != -1) {
1819                                 throw new InternalErrorException (string.Format ("Secondary putback [{0}] putting back [{1}] is not allowed", (char)putback_char, (char) c), Location);
1820                         }
1821
1822                         if (c == '\n' || col == 0) {
1823                                 // It won't happen though.
1824                                 line--;
1825                                 ref_line--;
1826                                 col = previous_col;
1827                         }
1828                         else
1829                                 col--;
1830                         putback_char = c;
1831                 }
1832
1833                 public bool advance ()
1834                 {
1835                         return peek_char () != -1 || CompleteOnEOF;
1836                 }
1837
1838                 public Object Value {
1839                         get {
1840                                 return val;
1841                         }
1842                 }
1843
1844                 public Object value ()
1845                 {
1846                         return val;
1847                 }
1848
1849                 public int token ()
1850                 {
1851                         current_token = xtoken ();
1852                         return current_token;
1853                 }
1854
1855                 int TokenizePreprocessorIdentifier (out int c)
1856                 {
1857                         // skip over white space
1858                         do {
1859                                 c = get_char ();
1860                         } while (c == ' ' || c == '\t');
1861
1862
1863                         int pos = 0;
1864                         while (c != -1 && c >= 'a' && c <= 'z') {
1865                                 id_builder[pos++] = (char) c;
1866                                 c = get_char ();
1867                                 if (c == '\\') {
1868                                         int peek = peek_char ();
1869                                         if (peek == 'U' || peek == 'u') {
1870                                                 int surrogate;
1871                                                 c = EscapeUnicode (c, out surrogate);
1872                                                 if (surrogate != 0) {
1873                                                         if (is_identifier_part_character ((char) c)) {
1874                                                                 id_builder[pos++] = (char) c;
1875                                                         }
1876                                                         c = surrogate;
1877                                                 }
1878                                         }
1879                                 }
1880                         }
1881
1882                         return pos;
1883                 }
1884
1885                 PreprocessorDirective get_cmd_arg (out string arg)
1886                 {
1887                         int c;
1888
1889                         tokens_seen = false;
1890                         arg = "";
1891
1892                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorIdentifier (out c));
1893
1894                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
1895                                 return cmd;
1896
1897                         // skip over white space
1898                         while (c == ' ' || c == '\t')
1899                                 c = get_char ();
1900
1901                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
1902                         int pos = 0;
1903
1904                         while (c != -1 && c != '\n') {
1905                                 if (c == '\\' && has_identifier_argument >= 0) {
1906                                         if (has_identifier_argument != 0) {
1907                                                 has_identifier_argument = 1;
1908
1909                                                 int peek = peek_char ();
1910                                                 if (peek == 'U' || peek == 'u') {
1911                                                         int surrogate;
1912                                                         c = EscapeUnicode (c, out surrogate);
1913                                                         if (surrogate != 0) {
1914                                                                 if (is_identifier_part_character ((char) c)) {
1915                                                                         if (pos == value_builder.Length)
1916                                                                                 Array.Resize (ref value_builder, pos * 2);
1917
1918                                                                         value_builder[pos++] = (char) c;
1919                                                                 }
1920                                                                 c = surrogate;
1921                                                         }
1922                                                 }
1923                                         } else {
1924                                                 has_identifier_argument = -1;
1925                                         }
1926                                 } else if (c == '/' && peek_char () == '/') {
1927                                         //
1928                                         // Eat single-line comments
1929                                         //
1930                                         get_char ();
1931                                         do {
1932                                                 c = get_char ();
1933                                         } while (c != -1 && c != '\n');
1934
1935                                         break;
1936                                 }
1937
1938                                 if (pos == value_builder.Length)
1939                                         Array.Resize (ref value_builder, pos * 2);
1940
1941                                 value_builder[pos++] = (char) c;
1942                                 c = get_char ();
1943                         }
1944
1945                         if (pos != 0) {
1946                                 if (pos > max_id_size)
1947                                         arg = new string (value_builder, 0, pos);
1948                                 else
1949                                         arg = InternIdentifier (value_builder, pos);
1950
1951                                 // Eat any trailing whitespaces
1952                                 arg = arg.Trim (simple_whitespaces);
1953                         }
1954
1955                         return cmd;
1956                 }
1957
1958                 //
1959                 // Handles the #line directive
1960                 //
1961                 bool PreProcessLine ()
1962                 {
1963                         Location loc = Location;
1964
1965                         int c;
1966
1967                         int length = TokenizePreprocessorIdentifier (out c);
1968                         if (length == line_default.Length) {
1969                                 if (!IsTokenIdentifierEqual (line_default))
1970                                         return false;
1971
1972                                 current_source = source_file.SourceFile;
1973                                 if (!hidden_block_start.IsNull) {
1974                                         current_source.RegisterHiddenScope (hidden_block_start, loc);
1975                                         hidden_block_start = Location.Null;
1976                                 }
1977
1978                                 ref_line = line;
1979                                 Location.Push (current_source);
1980                                 return true;
1981                         }
1982
1983                         if (length == line_hidden.Length) {
1984                                 if (!IsTokenIdentifierEqual (line_hidden))
1985                                         return false;
1986
1987                                 if (hidden_block_start.IsNull)
1988                                         hidden_block_start = loc;
1989
1990                                 return true;
1991                         }
1992
1993                         if (length != 0 || c < '0' || c > '9') {
1994                                 //
1995                                 // Eat any remaining characters to continue parsing on next line
1996                                 //
1997                                 while (c != -1 && c != '\n') {
1998                                         c = get_char ();
1999                                 }
2000
2001                                 return false;
2002                         }
2003
2004                         int new_line = TokenizeNumber (c);
2005                         if (new_line < 1) {
2006                                 //
2007                                 // Eat any remaining characters to continue parsing on next line
2008                                 //
2009                                 while (c != -1 && c != '\n') {
2010                                         c = get_char ();
2011                                 }
2012
2013                                 return new_line != 0;
2014                         }
2015
2016                         c = get_char ();
2017                         if (c == ' ') {
2018                                 // skip over white space
2019                                 do {
2020                                         c = get_char ();
2021                                 } while (c == ' ' || c == '\t');
2022                         } else if (c == '"') {
2023                                 c = 0;
2024                         }
2025
2026                         if (c != '\n' && c != '/' && c != '"') {
2027                                 //
2028                                 // Eat any remaining characters to continue parsing on next line
2029                                 //
2030                                 while (c != -1 && c != '\n') {
2031                                         c = get_char ();
2032                                 }
2033
2034                                 Report.Error (1578, loc, "Filename, single-line comment or end-of-line expected");
2035                                 return true;
2036                         }
2037
2038                         string new_file_name = null;
2039                         if (c == '"') {
2040                                 new_file_name = TokenizeFileName (ref c);
2041
2042                                 // skip over white space
2043                                 while (c == ' ' || c == '\t') {
2044                                         c = get_char ();
2045                                 }
2046                         }
2047
2048                         if (c == '\n') {
2049                         } else if (c == '/') {
2050                                 ReadSingleLineComment ();
2051                         } else {
2052                                 //
2053                                 // Eat any remaining characters to continue parsing on next line
2054                                 //
2055                                 while (c != -1 && c != '\n') {
2056                                         c = get_char ();
2057                                 }
2058
2059                                 Error_EndLineExpected ();
2060                                 return true;
2061                         }
2062
2063                         if (new_file_name != null) {
2064                                 current_source = context.LookupFile (source_file, new_file_name);
2065                                 source_file.AddIncludeFile (current_source);
2066                                 Location.Push (current_source);
2067                         }
2068
2069                         if (!hidden_block_start.IsNull) {
2070                                 current_source.RegisterHiddenScope (hidden_block_start, loc);
2071                                 hidden_block_start = Location.Null;
2072                         }
2073
2074                         ref_line = new_line;
2075                         return true;
2076                 }
2077
2078                 //
2079                 // Handles #define and #undef
2080                 //
2081                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
2082                 {
2083                         if (ident.Length == 0 || ident == "true" || ident == "false"){
2084                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
2085                                 return;
2086                         }
2087
2088                         if (ident.IndexOfAny (simple_whitespaces) != -1){
2089                                 Error_EndLineExpected ();
2090                                 return;
2091                         }
2092
2093                         if (!is_identifier_start_character (ident [0]))
2094                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
2095
2096                         foreach (char c in ident.Substring (1)){
2097                                 if (!is_identifier_part_character (c)){
2098                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
2099                                         return;
2100                                 }
2101                         }
2102
2103                         if (!caller_is_taking)
2104                                 return;
2105
2106                         if (is_define) {
2107                                 //
2108                                 // #define ident
2109                                 //
2110                                 if (context.Settings.IsConditionalSymbolDefined (ident))
2111                                         return;
2112
2113                                 source_file.AddDefine (ident);
2114                         } else {
2115                                 //
2116                                 // #undef ident
2117                                 //
2118                                 source_file.AddUndefine (ident);
2119                         }
2120                 }
2121
2122                 byte read_hex (out bool error)
2123                 {
2124                         int total;
2125                         int c = get_char ();
2126
2127                         if ((c >= '0') && (c <= '9'))
2128                                 total = (int) c - (int) '0';
2129                         else if ((c >= 'A') && (c <= 'F'))
2130                                 total = (int) c - (int) 'A' + 10;
2131                         else if ((c >= 'a') && (c <= 'f'))
2132                                 total = (int) c - (int) 'a' + 10;
2133                         else {
2134                                 error = true;
2135                                 return 0;
2136                         }
2137
2138                         total *= 16;
2139                         c = get_char ();
2140
2141                         if ((c >= '0') && (c <= '9'))
2142                                 total += (int) c - (int) '0';
2143                         else if ((c >= 'A') && (c <= 'F'))
2144                                 total += (int) c - (int) 'A' + 10;
2145                         else if ((c >= 'a') && (c <= 'f'))
2146                                 total += (int) c - (int) 'a' + 10;
2147                         else {
2148                                 error = true;
2149                                 return 0;
2150                         }
2151
2152                         error = false;
2153                         return (byte) total;
2154                 }
2155
2156                 //
2157                 // Parses #pragma checksum
2158                 //
2159                 bool ParsePragmaChecksum ()
2160                 {
2161                         //
2162                         // The syntax is ` "foo.txt" "{guid}" "hash"'
2163                         //
2164                         int c = get_char ();
2165
2166                         if (c != '"')
2167                                 return false;
2168
2169                         string file_name = TokenizeFileName (ref c);
2170
2171                         // TODO: Any white-spaces count
2172                         if (c != ' ')
2173                                 return false;
2174
2175                         SourceFile file = context.LookupFile (source_file, file_name);
2176
2177                         if (get_char () != '"' || get_char () != '{')
2178                                 return false;
2179
2180                         bool error;
2181                         byte[] guid_bytes = new byte [16];
2182                         int i = 0;
2183
2184                         for (; i < 4; i++) {
2185                                 guid_bytes [i] = read_hex (out error);
2186                                 if (error)
2187                                         return false;
2188                         }
2189
2190                         if (get_char () != '-')
2191                                 return false;
2192
2193                         for (; i < 10; i++) {
2194                                 guid_bytes [i] = read_hex (out error);
2195                                 if (error)
2196                                         return false;
2197
2198                                 guid_bytes [i++] = read_hex (out error);
2199                                 if (error)
2200                                         return false;
2201
2202                                 if (get_char () != '-')
2203                                         return false;
2204                         }
2205
2206                         for (; i < 16; i++) {
2207                                 guid_bytes [i] = read_hex (out error);
2208                                 if (error)
2209                                         return false;
2210                         }
2211
2212                         if (get_char () != '}' || get_char () != '"')
2213                                 return false;
2214
2215                         // TODO: Any white-spaces count
2216                         c = get_char ();
2217                         if (c != ' ')
2218                                 return false;
2219
2220                         if (get_char () != '"')
2221                                 return false;
2222
2223                         // Any length of checksum
2224                         List<byte> checksum_bytes = new List<byte> (16);
2225
2226                         c = peek_char ();
2227                         while (c != '"' && c != -1) {
2228                                 checksum_bytes.Add (read_hex (out error));
2229                                 if (error)
2230                                         return false;
2231
2232                                 c = peek_char ();
2233                         }
2234
2235                         if (c == '/') {
2236                                 ReadSingleLineComment ();
2237                         } else if (get_char () != '"') {
2238                                 return false;
2239                         }
2240
2241                         file.SetChecksum (guid_bytes, checksum_bytes.ToArray ());
2242                         current_source.AutoGenerated = true;
2243                         return true;
2244                 }
2245
2246 #if !FULL_AST
2247                 static
2248 #endif
2249                 bool IsTokenIdentifierEqual (char[] identifier)
2250                 {
2251                         for (int i = 0; i < identifier.Length; ++i) {
2252                                 if (identifier[i] != id_builder[i])
2253                                         return false;
2254                         }
2255
2256                         return true;
2257                 }
2258
2259                 int TokenizeNumber (int value)
2260                 {
2261                         number_pos = 0;
2262
2263                         decimal_digits (value);
2264                         uint ui = (uint) (number_builder[0] - '0');
2265
2266                         try {
2267                                 for (int i = 1; i < number_pos; i++) {
2268                                         ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2269                                 }
2270
2271                                 return (int) ui;
2272                         } catch (OverflowException) {
2273                                 Error_NumericConstantTooLong ();
2274                                 return -1;
2275                         }
2276                 }
2277
2278                 string TokenizeFileName (ref int c)
2279                 {
2280                         var string_builder = new StringBuilder ();
2281                         while (c != -1 && c != '\n') {
2282                                 c = get_char ();
2283                                 if (c == '"') {
2284                                         c = get_char ();
2285                                         break;
2286                                 }
2287
2288                                 string_builder.Append ((char) c);
2289                         }
2290
2291                         if (string_builder.Length == 0) {
2292                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
2293                         }
2294
2295
2296                         return string_builder.ToString ();
2297                 }
2298
2299                 int TokenizePragmaNumber (ref int c)
2300                 {
2301                         number_pos = 0;
2302
2303                         int number;
2304
2305                         if (c >= '0' && c <= '9') {
2306                                 number = TokenizeNumber (c);
2307
2308                                 c = get_char ();
2309
2310                                 // skip over white space
2311                                 while (c == ' ' || c == '\t')
2312                                         c = get_char ();
2313
2314                                 if (c == ',') {
2315                                         c = get_char ();
2316                                 }
2317
2318                                 // skip over white space
2319                                 while (c == ' ' || c == '\t')
2320                                         c = get_char ();
2321                         } else {
2322                                 number = -1;
2323                                 if (c == '/') {
2324                                         ReadSingleLineComment ();
2325                                 } else {
2326                                         Report.Warning (1692, 1, Location, "Invalid number");
2327
2328                                         // Read everything till the end of the line or file
2329                                         do {
2330                                                 c = get_char ();
2331                                         } while (c != -1 && c != '\n');
2332                                 }
2333                         }
2334
2335                         return number;
2336                 }
2337
2338                 void ReadSingleLineComment ()
2339                 {
2340                         if (peek_char () != '/')
2341                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2342
2343                         // Read everything till the end of the line or file
2344                         int c;
2345                         do {
2346                                 c = get_char ();
2347                         } while (c != -1 && c != '\n');
2348                 }
2349
2350                 /// <summary>
2351                 /// Handles #pragma directive
2352                 /// </summary>
2353                 void ParsePragmaDirective (string arg)
2354                 {
2355                         int c;
2356                         int length = TokenizePreprocessorIdentifier (out c);
2357                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2358                                 length = TokenizePreprocessorIdentifier (out c);
2359
2360                                 //
2361                                 // #pragma warning disable
2362                                 // #pragma warning restore
2363                                 //
2364                                 if (length == pragma_warning_disable.Length) {
2365                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2366                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2367                                                 // skip over white space
2368                                                 while (c == ' ' || c == '\t')
2369                                                         c = get_char ();
2370
2371                                                 var loc = Location;
2372
2373                                                 if (c == '\n' || c == '/') {
2374                                                         if (c == '/')
2375                                                                 ReadSingleLineComment ();
2376
2377                                                         //
2378                                                         // Disable/Restore all warnings
2379                                                         //
2380                                                         if (disable) {
2381                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2382                                                         } else {
2383                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2384                                                         }
2385                                                 } else {
2386                                                         //
2387                                                         // Disable/Restore a warning or group of warnings
2388                                                         //
2389                                                         int code;
2390                                                         do {
2391                                                                 code = TokenizePragmaNumber (ref c);
2392                                                                 if (code > 0) {
2393                                                                         if (disable) {
2394                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, context.Report);
2395                                                                         } else {
2396                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, context);
2397                                                                         }
2398                                                                 }
2399                                                         } while (code >= 0 && c != '\n' && c != -1);
2400                                                 }
2401
2402                                                 return;
2403                                         }
2404                                 }
2405
2406                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2407                                 return;
2408                         }
2409
2410                         //
2411                         // #pragma checksum
2412                         //
2413                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2414                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2415                                         Report.Warning (1695, 1, Location,
2416                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2417                                 }
2418
2419                                 return;
2420                         }
2421
2422                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2423                 }
2424
2425                 bool eval_val (string s)
2426                 {
2427                         if (s == "true")
2428                                 return true;
2429                         if (s == "false")
2430                                 return false;
2431
2432                         return source_file.IsConditionalDefined (s);
2433                 }
2434
2435                 bool pp_primary (ref string s)
2436                 {
2437                         s = s.Trim ();
2438                         int len = s.Length;
2439
2440                         if (len > 0){
2441                                 char c = s [0];
2442
2443                                 if (c == '('){
2444                                         s = s.Substring (1);
2445                                         bool val = pp_expr (ref s, false);
2446                                         if (s.Length > 0 && s [0] == ')'){
2447                                                 s = s.Substring (1);
2448                                                 return val;
2449                                         }
2450                                         Error_InvalidDirective ();
2451                                         return false;
2452                                 }
2453
2454                                 if (is_identifier_start_character (c)){
2455                                         int j = 1;
2456
2457                                         while (j < len){
2458                                                 c = s [j];
2459
2460                                                 if (is_identifier_part_character (c)){
2461                                                         j++;
2462                                                         continue;
2463                                                 }
2464                                                 bool v = eval_val (s.Substring (0, j));
2465                                                 s = s.Substring (j);
2466                                                 return v;
2467                                         }
2468                                         bool vv = eval_val (s);
2469                                         s = "";
2470                                         return vv;
2471                                 }
2472                         }
2473                         Error_InvalidDirective ();
2474                         return false;
2475                 }
2476
2477                 bool pp_unary (ref string s)
2478                 {
2479                         s = s.Trim ();
2480                         int len = s.Length;
2481
2482                         if (len > 0){
2483                                 if (s [0] == '!'){
2484                                         if (len > 1 && s [1] == '='){
2485                                                 Error_InvalidDirective ();
2486                                                 return false;
2487                                         }
2488                                         s = s.Substring (1);
2489                                         return ! pp_primary (ref s);
2490                                 } else
2491                                         return pp_primary (ref s);
2492                         } else {
2493                                 Error_InvalidDirective ();
2494                                 return false;
2495                         }
2496                 }
2497
2498                 bool pp_eq (ref string s)
2499                 {
2500                         bool va = pp_unary (ref s);
2501
2502                         s = s.Trim ();
2503                         int len = s.Length;
2504                         if (len > 0){
2505                                 if (s [0] == '='){
2506                                         if (len > 2 && s [1] == '='){
2507                                                 s = s.Substring (2);
2508                                                 return va == pp_unary (ref s);
2509                                         } else {
2510                                                 Error_InvalidDirective ();
2511                                                 return false;
2512                                         }
2513                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2514                                         s = s.Substring (2);
2515
2516                                         return va != pp_unary (ref s);
2517
2518                                 }
2519                         }
2520
2521                         return va;
2522
2523                 }
2524
2525                 bool pp_and (ref string s)
2526                 {
2527                         bool va = pp_eq (ref s);
2528
2529                         s = s.Trim ();
2530                         int len = s.Length;
2531                         if (len > 0){
2532                                 if (s [0] == '&'){
2533                                         if (len > 2 && s [1] == '&'){
2534                                                 s = s.Substring (2);
2535                                                 return (va & pp_and (ref s));
2536                                         } else {
2537                                                 Error_InvalidDirective ();
2538                                                 return false;
2539                                         }
2540                                 }
2541                         }
2542                         return va;
2543                 }
2544
2545                 //
2546                 // Evaluates an expression for `#if' or `#elif'
2547                 //
2548                 bool pp_expr (ref string s, bool isTerm)
2549                 {
2550                         bool va = pp_and (ref s);
2551                         s = s.Trim ();
2552                         int len = s.Length;
2553                         if (len > 0){
2554                                 char c = s [0];
2555
2556                                 if (c == '|'){
2557                                         if (len > 2 && s [1] == '|'){
2558                                                 s = s.Substring (2);
2559                                                 return va | pp_expr (ref s, isTerm);
2560                                         } else {
2561                                                 Error_InvalidDirective ();
2562                                                 return false;
2563                                         }
2564                                 }
2565                                 if (isTerm) {
2566                                         Error_EndLineExpected ();
2567                                         return false;
2568                                 }
2569                         }
2570
2571                         return va;
2572                 }
2573
2574                 bool eval (string s)
2575                 {
2576                         bool v = pp_expr (ref s, true);
2577                         s = s.Trim ();
2578                         if (s.Length != 0){
2579                                 return false;
2580                         }
2581
2582                         return v;
2583                 }
2584
2585                 void Error_NumericConstantTooLong ()
2586                 {
2587                         Report.Error (1021, Location, "Integral constant is too large");
2588                 }
2589
2590                 void Error_InvalidDirective ()
2591                 {
2592                         Report.Error (1517, Location, "Invalid preprocessor directive");
2593                 }
2594
2595                 void Error_UnexpectedDirective (string extra)
2596                 {
2597                         Report.Error (
2598                                 1028, Location,
2599                                 "Unexpected processor directive ({0})", extra);
2600                 }
2601
2602                 void Error_TokensSeen ()
2603                 {
2604                         Report.Error (1032, Location,
2605                                 "Cannot define or undefine preprocessor symbols after first token in file");
2606                 }
2607
2608                 void Eror_WrongPreprocessorLocation ()
2609                 {
2610                         Report.Error (1040, Location,
2611                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2612                 }
2613
2614                 void Error_EndLineExpected ()
2615                 {
2616                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2617                 }
2618
2619                 //
2620                 // Raises a warning when tokenizer found documentation comment
2621                 // on unexpected place
2622                 //
2623                 void WarningMisplacedComment (Location loc)
2624                 {
2625                         if (doc_state != XmlCommentState.Error) {
2626                                 doc_state = XmlCommentState.Error;
2627                                 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element");
2628                         }
2629                 }
2630
2631                 //
2632                 // if true, then the code continues processing the code
2633                 // if false, the code stays in a loop until another directive is
2634                 // reached.
2635                 // When caller_is_taking is false we ignore all directives except the ones
2636                 // which can help us to identify where the #if block ends
2637                 bool ParsePreprocessingDirective (bool caller_is_taking)
2638                 {
2639                         string arg;
2640                         bool region_directive = false;
2641
2642                         var directive = get_cmd_arg (out arg);
2643
2644                         //
2645                         // The first group of pre-processing instructions is always processed
2646                         //
2647                         switch (directive) {
2648                         case PreprocessorDirective.Region:
2649                                 region_directive = true;
2650                                 arg = "true";
2651                                 goto case PreprocessorDirective.If;
2652
2653                         case PreprocessorDirective.Endregion:
2654                                 if (ifstack == null || ifstack.Count == 0){
2655                                         Error_UnexpectedDirective ("no #region for this #endregion");
2656                                         return true;
2657                                 }
2658                                 int pop = ifstack.Pop ();
2659
2660                                 if ((pop & REGION) == 0)
2661                                         Report.Error (1027, Location, "Expected `#endif' directive");
2662
2663                                 return caller_is_taking;
2664
2665                         case PreprocessorDirective.If:
2666                                 if (ifstack == null)
2667                                         ifstack = new Stack<int> (2);
2668
2669                                 int flags = region_directive ? REGION : 0;
2670                                 if (ifstack.Count == 0){
2671                                         flags |= PARENT_TAKING;
2672                                 } else {
2673                                         int state = ifstack.Peek ();
2674                                         if ((state & TAKING) != 0) {
2675                                                 flags |= PARENT_TAKING;
2676                                         }
2677                                 }
2678
2679                                 if (eval (arg) && caller_is_taking) {
2680                                         ifstack.Push (flags | TAKING);
2681                                         return true;
2682                                 }
2683                                 ifstack.Push (flags);
2684                                 return false;
2685
2686                         case PreprocessorDirective.Endif:
2687                                 if (ifstack == null || ifstack.Count == 0){
2688                                         Error_UnexpectedDirective ("no #if for this #endif");
2689                                         return true;
2690                                 } else {
2691                                         pop = ifstack.Pop ();
2692
2693                                         if ((pop & REGION) != 0)
2694                                                 Report.Error (1038, Location, "#endregion directive expected");
2695
2696                                         if (arg.Length != 0) {
2697                                                 Error_EndLineExpected ();
2698                                         }
2699
2700                                         if (ifstack.Count == 0)
2701                                                 return true;
2702
2703                                         int state = ifstack.Peek ();
2704                                         return (state & TAKING) != 0;
2705                                 }
2706
2707                         case PreprocessorDirective.Elif:
2708                                 if (ifstack == null || ifstack.Count == 0){
2709                                         Error_UnexpectedDirective ("no #if for this #elif");
2710                                         return true;
2711                                 } else {
2712                                         int state = ifstack.Pop ();
2713
2714                                         if ((state & REGION) != 0) {
2715                                                 Report.Error (1038, Location, "#endregion directive expected");
2716                                                 return true;
2717                                         }
2718
2719                                         if ((state & ELSE_SEEN) != 0){
2720                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2721                                                 return true;
2722                                         }
2723
2724                                         if ((state & TAKING) != 0) {
2725                                                 ifstack.Push (0);
2726                                                 return false;
2727                                         }
2728
2729                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2730                                                 ifstack.Push (state | TAKING);
2731                                                 return true;
2732                                         }
2733
2734                                         ifstack.Push (state);
2735                                         return false;
2736                                 }
2737
2738                         case PreprocessorDirective.Else:
2739                                 if (ifstack == null || ifstack.Count == 0){
2740                                         Error_UnexpectedDirective ("no #if for this #else");
2741                                         return true;
2742                                 } else {
2743                                         int state = ifstack.Peek ();
2744
2745                                         if ((state & REGION) != 0) {
2746                                                 Report.Error (1038, Location, "#endregion directive expected");
2747                                                 return true;
2748                                         }
2749
2750                                         if ((state & ELSE_SEEN) != 0){
2751                                                 Error_UnexpectedDirective ("#else within #else");
2752                                                 return true;
2753                                         }
2754
2755                                         ifstack.Pop ();
2756
2757                                         if (arg.Length != 0) {
2758                                                 Error_EndLineExpected ();
2759                                                 return true;
2760                                         }
2761
2762                                         bool ret = false;
2763                                         if ((state & PARENT_TAKING) != 0) {
2764                                                 ret = (state & TAKING) == 0;
2765
2766                                                 if (ret)
2767                                                         state |= TAKING;
2768                                                 else
2769                                                         state &= ~TAKING;
2770                                         }
2771
2772                                         ifstack.Push (state | ELSE_SEEN);
2773
2774                                         return ret;
2775                                 }
2776                         case PreprocessorDirective.Define:
2777                                 if (any_token_seen){
2778                                         Error_TokensSeen ();
2779                                         return caller_is_taking;
2780                                 }
2781                                 PreProcessDefinition (true, arg, caller_is_taking);
2782                                 return caller_is_taking;
2783
2784                         case PreprocessorDirective.Undef:
2785                                 if (any_token_seen){
2786                                         Error_TokensSeen ();
2787                                         return caller_is_taking;
2788                                 }
2789                                 PreProcessDefinition (false, arg, caller_is_taking);
2790                                 return caller_is_taking;
2791
2792                         case PreprocessorDirective.Invalid:
2793                                 Report.Error (1024, Location, "Wrong preprocessor directive");
2794                                 return true;
2795                         }
2796
2797                         //
2798                         // These are only processed if we are in a `taking' block
2799                         //
2800                         if (!caller_is_taking)
2801                                 return false;
2802
2803                         switch (directive){
2804                         case PreprocessorDirective.Error:
2805                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2806                                 return true;
2807
2808                         case PreprocessorDirective.Warning:
2809                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2810                                 return true;
2811
2812                         case PreprocessorDirective.Pragma:
2813                                 if (context.Settings.Version == LanguageVersion.ISO_1) {
2814                                         Report.FeatureIsNotAvailable (context, Location, "#pragma");
2815                                 }
2816
2817                                 ParsePragmaDirective (arg);
2818                                 return true;
2819
2820                         case PreprocessorDirective.Line:
2821                                 Location loc = Location;
2822                                 if (!PreProcessLine ())
2823                                         Report.Error (1576, loc, "The line number specified for #line directive is missing or invalid");
2824
2825                                 return caller_is_taking;
2826                         }
2827
2828                         throw new NotImplementedException (directive.ToString ());
2829                 }
2830
2831                 private int consume_string (bool quoted)
2832                 {
2833                         int c;
2834                         int pos = 0;
2835                         Location start_location = Location;
2836                         if (quoted)
2837                                 start_location = start_location - 1;
2838
2839 #if FULL_AST
2840                         int reader_pos = reader.Position;
2841 #endif
2842
2843                         while (true){
2844                                 c = get_char ();
2845                                 if (c == '"') {
2846                                         if (quoted && peek_char () == '"') {
2847                                                 if (pos == value_builder.Length)
2848                                                         Array.Resize (ref value_builder, pos * 2);
2849
2850                                                 value_builder[pos++] = (char) c;
2851                                                 get_char ();
2852                                                 continue;
2853                                         }
2854
2855                                         string s;
2856                                         if (pos == 0)
2857                                                 s = string.Empty;
2858                                         else if (pos <= 4)
2859                                                 s = InternIdentifier (value_builder, pos);
2860                                         else
2861                                                 s = new string (value_builder, 0, pos);
2862
2863                                         ILiteralConstant res = new StringLiteral (context.BuiltinTypes, s, start_location);
2864                                         val = res;
2865 #if FULL_AST
2866                                         res.ParsedValue = quoted ?
2867                                                 reader.ReadChars (reader_pos - 2, reader.Position - 1) :
2868                                                 reader.ReadChars (reader_pos - 1, reader.Position);
2869 #endif
2870
2871                                         return Token.LITERAL;
2872                                 }
2873
2874                                 if (c == '\n') {
2875                                         if (!quoted) {
2876                                                 Report.Error (1010, Location, "Newline in constant");
2877                                                 val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location);
2878                                                 return Token.LITERAL;
2879                                         }
2880                                 } else if (c == '\\' && !quoted) {
2881                                         int surrogate;
2882                                         c = escape (c, out surrogate);
2883                                         if (c == -1)
2884                                                 return Token.ERROR;
2885                                         if (surrogate != 0) {
2886                                                 if (pos == value_builder.Length)
2887                                                         Array.Resize (ref value_builder, pos * 2);
2888
2889                                                 value_builder[pos++] = (char) c;
2890                                                 c = surrogate;
2891                                         }
2892                                 } else if (c == -1) {
2893                                         Report.Error (1039, Location, "Unterminated string literal");
2894                                         return Token.EOF;
2895                                 }
2896
2897                                 if (pos == value_builder.Length)
2898                                         Array.Resize (ref value_builder, pos * 2);
2899
2900                                 value_builder[pos++] = (char) c;
2901                         }
2902                 }
2903
2904                 private int consume_identifier (int s)
2905                 {
2906                         int res = consume_identifier (s, false);
2907
2908                         if (doc_state == XmlCommentState.Allowed)
2909                                 doc_state = XmlCommentState.NotAllowed;
2910
2911                         return res;
2912                 }
2913
2914                 int consume_identifier (int c, bool quoted)
2915                 {
2916                         //
2917                         // This method is very performance sensitive. It accounts
2918                         // for approximately 25% of all parser time
2919                         //
2920
2921                         int pos = 0;
2922                         int column = col;
2923                         if (quoted)
2924                                 --column;
2925
2926                         if (c == '\\') {
2927                                 int surrogate;
2928                                 c = escape (c, out surrogate);
2929                                 if (surrogate != 0) {
2930                                         id_builder [pos++] = (char) c;
2931                                         c = surrogate;
2932                                 }
2933                         }
2934
2935                         id_builder [pos++] = (char) c;
2936
2937                         try {
2938                                 while (true) {
2939                                         c = reader.Read ();
2940
2941                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
2942                                                 id_builder [pos++] = (char) c;
2943                                                 continue;
2944                                         }
2945
2946                                         if (c < 0x80) {
2947                                                 if (c == '\\') {
2948                                                         int surrogate;
2949                                                         c = escape (c, out surrogate);
2950                                                         if (is_identifier_part_character ((char) c))
2951                                                                 id_builder[pos++] = (char) c;
2952
2953                                                         if (surrogate != 0) {
2954                                                                 c = surrogate;
2955                                                         }
2956
2957                                                         continue;
2958                                                 }
2959                                         } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
2960                                                 id_builder [pos++] = (char) c;
2961                                                 continue;
2962                                         }
2963
2964                                         putback_char = c;
2965                                         break;
2966                                 }
2967                         } catch (IndexOutOfRangeException) {
2968                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2969                                 --pos;
2970                                 col += pos;
2971                         }
2972
2973                         col += pos - 1;
2974
2975                         //
2976                         // Optimization: avoids doing the keyword lookup
2977                         // on uppercase letters
2978                         //
2979                         if (id_builder [0] >= '_' && !quoted) {
2980                                 int keyword = GetKeyword (id_builder, pos);
2981                                 if (keyword != -1) {
2982                                         val = LocatedToken.Create (keyword == Token.AWAIT ? "await" : null, ref_line, column);
2983                                         return keyword;
2984                                 }
2985                         }
2986
2987                         string s = InternIdentifier (id_builder, pos);
2988                         val = LocatedToken.Create (s, ref_line, column);
2989                         if (quoted && parsing_attribute_section)
2990                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2991
2992                         return Token.IDENTIFIER;
2993                 }
2994
2995 #if !FULL_AST
2996                 static
2997 #endif
2998                 string InternIdentifier (char[] charBuffer, int length)
2999                 {
3000                         //
3001                         // Keep identifiers in an array of hashtables to avoid needless
3002                         // allocations
3003                         //
3004                         var identifiers_group = identifiers[length];
3005                         string s;
3006                         if (identifiers_group != null) {
3007                                 if (identifiers_group.TryGetValue (charBuffer, out s)) {
3008                                         return s;
3009                                 }
3010                         } else {
3011                                 // TODO: this should be number of files dependant
3012                                 // corlib compilation peaks at 1000 and System.Core at 150
3013                                 int capacity = length > 20 ? 10 : 100;
3014                                 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length));
3015                                 identifiers[length] = identifiers_group;
3016                         }
3017
3018                         char[] chars = new char[length];
3019                         Array.Copy (charBuffer, chars, length);
3020
3021                         s = new string (charBuffer, 0, length);
3022                         identifiers_group.Add (chars, s);
3023                         return s;
3024                 }
3025
3026                 public int xtoken ()
3027                 {
3028                         int d, c;
3029
3030                         // Whether we have seen comments on the current line
3031                         bool comments_seen = false;
3032                         while ((c = get_char ()) != -1) {
3033                                 switch (c) {
3034                                 case '\t':
3035                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
3036                                         continue;
3037
3038                                 case ' ':
3039                                 case '\f':
3040                                 case '\v':
3041                                 case 0xa0:
3042                                 case 0:
3043                                 case 0xFEFF:    // Ignore BOM anywhere in the file
3044                                         continue;
3045
3046 /*                              This is required for compatibility with .NET
3047                                 case 0xEF:
3048                                         if (peek_char () == 0xBB) {
3049                                                 PushPosition ();
3050                                                 get_char ();
3051                                                 if (get_char () == 0xBF)
3052                                                         continue;
3053                                                 PopPosition ();
3054                                         }
3055                                         break;
3056 */
3057                                 case '\\':
3058                                         tokens_seen = true;
3059                                         return consume_identifier (c);
3060
3061                                 case '{':
3062                                         val = LocatedToken.Create (ref_line, col);
3063                                         return Token.OPEN_BRACE;
3064                                 case '}':
3065                                         val = LocatedToken.Create (ref_line, col);
3066                                         return Token.CLOSE_BRACE;
3067                                 case '[':
3068                                         // To block doccomment inside attribute declaration.
3069                                         if (doc_state == XmlCommentState.Allowed)
3070                                                 doc_state = XmlCommentState.NotAllowed;
3071
3072                                         val = LocatedToken.Create (ref_line, col);
3073
3074                                         if (parsing_block == 0 || lambda_arguments_parsing)
3075                                                 return Token.OPEN_BRACKET;
3076
3077                                         int next = peek_char ();
3078                                         switch (next) {
3079                                         case ']':
3080                                         case ',':
3081                                                 return Token.OPEN_BRACKET;
3082
3083                                         case ' ':
3084                                         case '\f':
3085                                         case '\v':
3086                                         case '\r':
3087                                         case '\n':
3088                                         case '/':
3089                                                 next = peek_token ();
3090                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
3091                                                         return Token.OPEN_BRACKET;
3092
3093                                                 return Token.OPEN_BRACKET_EXPR;
3094                                         default:
3095                                                 return Token.OPEN_BRACKET_EXPR;
3096                                         }
3097                                 case ']':
3098                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3099                                         return Token.CLOSE_BRACKET;
3100                                 case '(':
3101                                         val = LocatedToken.Create (ref_line, col);
3102                                         //
3103                                         // An expression versions of parens can appear in block context only
3104                                         //
3105                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
3106
3107                                                 //
3108                                                 // Optmize most common case where we know that parens
3109                                                 // is not special
3110                                                 //
3111                                                 switch (current_token) {
3112                                                 case Token.IDENTIFIER:
3113                                                 case Token.IF:
3114                                                 case Token.FOR:
3115                                                 case Token.FOREACH:
3116                                                 case Token.TYPEOF:
3117                                                 case Token.WHILE:
3118                                                 case Token.USING:
3119                                                 case Token.DEFAULT:
3120                                                 case Token.DELEGATE:
3121                                                 case Token.OP_GENERICS_GT:
3122                                                         return Token.OPEN_PARENS;
3123                                                 }
3124
3125                                                 // Optimize using peek
3126                                                 int xx = peek_char ();
3127                                                 switch (xx) {
3128                                                 case '(':
3129                                                 case '\'':
3130                                                 case '"':
3131                                                 case '0':
3132                                                 case '1':
3133                                                         return Token.OPEN_PARENS;
3134                                                 }
3135
3136                                                 lambda_arguments_parsing = true;
3137                                                 PushPosition ();
3138                                                 d = TokenizeOpenParens ();
3139                                                 PopPosition ();
3140                                                 lambda_arguments_parsing = false;
3141                                                 return d;
3142                                         }
3143
3144                                         return Token.OPEN_PARENS;
3145                                 case ')':
3146                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3147                                         return Token.CLOSE_PARENS;
3148                                 case ',':
3149                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3150                                         return Token.COMMA;
3151                                 case ';':
3152                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3153                                         return Token.SEMICOLON;
3154                                 case '~':
3155                                         val = LocatedToken.Create (ref_line, col);
3156                                         return Token.TILDE;
3157                                 case '?':
3158                                         val = LocatedToken.Create (ref_line, col);
3159                                         return TokenizePossibleNullableType ();
3160                                 case '<':
3161                                         val = LocatedToken.Create (ref_line, col);
3162                                         if (parsing_generic_less_than++ > 0)
3163                                                 return Token.OP_GENERICS_LT;
3164
3165                                         return TokenizeLessThan ();
3166
3167                                 case '>':
3168                                         val = LocatedToken.Create (ref_line, col);
3169                                         d = peek_char ();
3170
3171                                         if (d == '='){
3172                                                 get_char ();
3173                                                 return Token.OP_GE;
3174                                         }
3175
3176                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
3177                                                 parsing_generic_less_than--;
3178                                                 return Token.OP_GENERICS_GT;
3179                                         }
3180
3181                                         if (d == '>') {
3182                                                 get_char ();
3183                                                 d = peek_char ();
3184
3185                                                 if (d == '=') {
3186                                                         get_char ();
3187                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
3188                                                 }
3189                                                 return Token.OP_SHIFT_RIGHT;
3190                                         }
3191
3192                                         return Token.OP_GT;
3193
3194                                 case '+':
3195                                         val = LocatedToken.Create (ref_line, col);
3196                                         d = peek_char ();
3197                                         if (d == '+') {
3198                                                 d = Token.OP_INC;
3199                                         } else if (d == '=') {
3200                                                 d = Token.OP_ADD_ASSIGN;
3201                                         } else {
3202                                                 return Token.PLUS;
3203                                         }
3204                                         get_char ();
3205                                         return d;
3206
3207                                 case '-':
3208                                         val = LocatedToken.Create (ref_line, col);
3209                                         d = peek_char ();
3210                                         if (d == '-') {
3211                                                 d = Token.OP_DEC;
3212                                         } else if (d == '=')
3213                                                 d = Token.OP_SUB_ASSIGN;
3214                                         else if (d == '>')
3215                                                 d = Token.OP_PTR;
3216                                         else {
3217                                                 return Token.MINUS;
3218                                         }
3219                                         get_char ();
3220                                         return d;
3221
3222                                 case '!':
3223                                         val = LocatedToken.Create (ref_line, col);
3224                                         if (peek_char () == '='){
3225                                                 get_char ();
3226                                                 return Token.OP_NE;
3227                                         }
3228                                         return Token.BANG;
3229
3230                                 case '=':
3231                                         val = LocatedToken.Create (ref_line, col);
3232                                         d = peek_char ();
3233                                         if (d == '='){
3234                                                 get_char ();
3235                                                 return Token.OP_EQ;
3236                                         }
3237                                         if (d == '>'){
3238                                                 get_char ();
3239                                                 return Token.ARROW;
3240                                         }
3241
3242                                         return Token.ASSIGN;
3243
3244                                 case '&':
3245                                         val = LocatedToken.Create (ref_line, col);
3246                                         d = peek_char ();
3247                                         if (d == '&'){
3248                                                 get_char ();
3249                                                 return Token.OP_AND;
3250                                         }
3251                                         if (d == '='){
3252                                                 get_char ();
3253                                                 return Token.OP_AND_ASSIGN;
3254                                         }
3255                                         return Token.BITWISE_AND;
3256
3257                                 case '|':
3258                                         val = LocatedToken.Create (ref_line, col);
3259                                         d = peek_char ();
3260                                         if (d == '|'){
3261                                                 get_char ();
3262                                                 return Token.OP_OR;
3263                                         }
3264                                         if (d == '='){
3265                                                 get_char ();
3266                                                 return Token.OP_OR_ASSIGN;
3267                                         }
3268                                         return Token.BITWISE_OR;
3269
3270                                 case '*':
3271                                         val = LocatedToken.Create (ref_line, col);
3272                                         if (peek_char () == '='){
3273                                                 get_char ();
3274                                                 return Token.OP_MULT_ASSIGN;
3275                                         }
3276                                         return Token.STAR;
3277
3278                                 case '/':
3279                                         d = peek_char ();
3280                                         if (d == '='){
3281                                                 val = LocatedToken.Create (ref_line, col);
3282                                                 get_char ();
3283                                                 return Token.OP_DIV_ASSIGN;
3284                                         }
3285
3286                                         // Handle double-slash comments.
3287                                         if (d == '/'){
3288                                                 get_char ();
3289                                                 if (doc_processing) {
3290                                                         if (peek_char () == '/') {
3291                                                                 get_char ();
3292                                                                 // Don't allow ////.
3293                                                                 if ((d = peek_char ()) != '/') {
3294                                                                         if (doc_state == XmlCommentState.Allowed)
3295                                                                                 handle_one_line_xml_comment ();
3296                                                                         else if (doc_state == XmlCommentState.NotAllowed)
3297                                                                                 WarningMisplacedComment (Location - 3);
3298                                                                 }
3299                                                         } else {
3300                                                                 if (xml_comment_buffer.Length > 0)
3301                                                                         doc_state = XmlCommentState.NotAllowed;
3302                                                         }
3303                                                 }
3304
3305                                                 while ((d = get_char ()) != -1 && d != '\n');
3306
3307                                                 any_token_seen |= tokens_seen;
3308                                                 tokens_seen = false;
3309                                                 comments_seen = false;
3310                                                 continue;
3311                                         } else if (d == '*'){
3312                                                 get_char ();
3313                                                 bool docAppend = false;
3314                                                 if (doc_processing && peek_char () == '*') {
3315                                                         get_char ();
3316                                                         // But when it is /**/, just do nothing.
3317                                                         if (peek_char () == '/') {
3318                                                                 get_char ();
3319                                                                 continue;
3320                                                         }
3321                                                         if (doc_state == XmlCommentState.Allowed)
3322                                                                 docAppend = true;
3323                                                         else if (doc_state == XmlCommentState.NotAllowed) {
3324                                                                 WarningMisplacedComment (Location - 2);
3325                                                         }
3326                                                 }
3327
3328                                                 int current_comment_start = 0;
3329                                                 if (docAppend) {
3330                                                         current_comment_start = xml_comment_buffer.Length;
3331                                                         xml_comment_buffer.Append (Environment.NewLine);
3332                                                 }
3333
3334                                                 while ((d = get_char ()) != -1){
3335                                                         if (d == '*' && peek_char () == '/'){
3336                                                                 get_char ();
3337                                                                 comments_seen = true;
3338                                                                 break;
3339                                                         }
3340                                                         if (docAppend)
3341                                                                 xml_comment_buffer.Append ((char) d);
3342
3343                                                         if (d == '\n'){
3344                                                                 any_token_seen |= tokens_seen;
3345                                                                 tokens_seen = false;
3346                                                                 //
3347                                                                 // Reset 'comments_seen' just to be consistent.
3348                                                                 // It doesn't matter either way, here.
3349                                                                 //
3350                                                                 comments_seen = false;
3351                                                         }
3352                                                 }
3353                                                 if (!comments_seen)
3354                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3355
3356                                                 if (docAppend)
3357                                                         update_formatted_doc_comment (current_comment_start);
3358                                                 continue;
3359                                         }
3360                                         val = LocatedToken.Create (ref_line, col);
3361                                         return Token.DIV;
3362
3363                                 case '%':
3364                                         val = LocatedToken.Create (ref_line, col);
3365                                         if (peek_char () == '='){
3366                                                 get_char ();
3367                                                 return Token.OP_MOD_ASSIGN;
3368                                         }
3369                                         return Token.PERCENT;
3370
3371                                 case '^':
3372                                         val = LocatedToken.Create (ref_line, col);
3373                                         if (peek_char () == '='){
3374                                                 get_char ();
3375                                                 return Token.OP_XOR_ASSIGN;
3376                                         }
3377                                         return Token.CARRET;
3378
3379                                 case ':':
3380                                         val = LocatedToken.Create (ref_line, col);
3381                                         if (peek_char () == ':') {
3382                                                 get_char ();
3383                                                 return Token.DOUBLE_COLON;
3384                                         }
3385                                         return Token.COLON;
3386
3387                                 case '0': case '1': case '2': case '3': case '4':
3388                                 case '5': case '6': case '7': case '8': case '9':
3389                                         tokens_seen = true;
3390                                         return is_number (c);
3391
3392                                 case '\n': // white space
3393                                         any_token_seen |= tokens_seen;
3394                                         tokens_seen = false;
3395                                         comments_seen = false;
3396                                         continue;
3397
3398                                 case '.':
3399                                         tokens_seen = true;
3400                                         d = peek_char ();
3401                                         if (d >= '0' && d <= '9')
3402                                                 return is_number (c);
3403
3404                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3405                                         return Token.DOT;
3406
3407                                 case '#':
3408                                         if (tokens_seen || comments_seen) {
3409                                                 Eror_WrongPreprocessorLocation ();
3410                                                 return Token.ERROR;
3411                                         }
3412
3413                                         if (ParsePreprocessingDirective (true))
3414                                                 continue;
3415
3416                                         bool directive_expected = false;
3417                                         while ((c = get_char ()) != -1) {
3418                                                 if (col == 1) {
3419                                                         directive_expected = true;
3420                                                 } else if (!directive_expected) {
3421                                                         // TODO: Implement comment support for disabled code and uncomment this code
3422 //                                                      if (c == '#') {
3423 //                                                              Eror_WrongPreprocessorLocation ();
3424 //                                                              return Token.ERROR;
3425 //                                                      }
3426                                                         continue;
3427                                                 }
3428
3429                                                 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' )
3430                                                         continue;
3431
3432                                                 if (c == '#') {
3433                                                         if (ParsePreprocessingDirective (false))
3434                                                                 break;
3435                                                 }
3436                                                 directive_expected = false;
3437                                         }
3438
3439                                         if (c != -1) {
3440                                                 tokens_seen = false;
3441                                                 continue;
3442                                         }
3443
3444                                         return Token.EOF;
3445
3446                                 case '"':
3447                                         return consume_string (false);
3448
3449                                 case '\'':
3450                                         return TokenizeBackslash ();
3451
3452                                 case '@':
3453                                         c = get_char ();
3454                                         if (c == '"') {
3455                                                 tokens_seen = true;
3456                                                 return consume_string (true);
3457                                         }
3458
3459                                         if (is_identifier_start_character (c)){
3460                                                 return consume_identifier (c, true);
3461                                         }
3462
3463                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3464                                         return Token.ERROR;
3465
3466                                 case EvalStatementParserCharacter:
3467                                         return Token.EVAL_STATEMENT_PARSER;
3468                                 case EvalCompilationUnitParserCharacter:
3469                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3470                                 case EvalUsingDeclarationsParserCharacter:
3471                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3472                                 case DocumentationXref:
3473                                         return Token.DOC_SEE;
3474                                 }
3475
3476                                 if (is_identifier_start_character (c)) {
3477                                         tokens_seen = true;
3478                                         return consume_identifier (c);
3479                                 }
3480
3481                                 if (char.IsWhiteSpace ((char) c))
3482                                         continue;
3483
3484                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3485                         }
3486
3487                         if (CompleteOnEOF){
3488                                 if (generated)
3489                                         return Token.COMPLETE_COMPLETION;
3490
3491                                 generated = true;
3492                                 return Token.GENERATE_COMPLETION;
3493                         }
3494
3495
3496                         return Token.EOF;
3497                 }
3498
3499                 int TokenizeBackslash ()
3500                 {
3501 #if FULL_AST
3502                         int read_start = reader.Position;
3503 #endif
3504                         Location start_location = Location;
3505                         int c = get_char ();
3506                         tokens_seen = true;
3507                         if (c == '\'') {
3508                                 val = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3509                                 Report.Error (1011, start_location, "Empty character literal");
3510                                 return Token.LITERAL;
3511                         }
3512
3513                         if (c == '\n') {
3514                                 Report.Error (1010, start_location, "Newline in constant");
3515                                 return Token.ERROR;
3516                         }
3517
3518                         int d;
3519                         c = escape (c, out d);
3520                         if (c == -1)
3521                                 return Token.ERROR;
3522                         if (d != 0)
3523                                 throw new NotImplementedException ();
3524
3525                         ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3526                         val = res;
3527                         c = get_char ();
3528
3529                         if (c != '\'') {
3530                                 Report.Error (1012, start_location, "Too many characters in character literal");
3531
3532                                 // Try to recover, read until newline or next "'"
3533                                 while ((c = get_char ()) != -1) {
3534                                         if (c == '\n' || c == '\'')
3535                                                 break;
3536                                 }
3537                         }
3538
3539 #if FULL_AST
3540                         res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position);
3541 #endif
3542
3543                         return Token.LITERAL;
3544                 }
3545
3546                 int TokenizeLessThan ()
3547                 {
3548                         int d;
3549                         if (handle_typeof) {
3550                                 PushPosition ();
3551                                 if (parse_generic_dimension (out d)) {
3552                                         val = d;
3553                                         DiscardPosition ();
3554                                         return Token.GENERIC_DIMENSION;
3555                                 }
3556                                 PopPosition ();
3557                         }
3558
3559                         // Save current position and parse next token.
3560                         PushPosition ();
3561                         if (parse_less_than ()) {
3562                                 if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) {
3563                                         d = Token.OP_GENERICS_LT_DECL;
3564                                 } else {
3565                                         d = Token.OP_GENERICS_LT;
3566                                 }
3567                                 PopPosition ();
3568                                 return d;
3569                         }
3570
3571                         PopPosition ();
3572                         parsing_generic_less_than = 0;
3573
3574                         d = peek_char ();
3575                         if (d == '<') {
3576                                 get_char ();
3577                                 d = peek_char ();
3578
3579                                 if (d == '=') {
3580                                         get_char ();
3581                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3582                                 }
3583                                 return Token.OP_SHIFT_LEFT;
3584                         }
3585
3586                         if (d == '=') {
3587                                 get_char ();
3588                                 return Token.OP_LE;
3589                         }
3590                         return Token.OP_LT;
3591                 }
3592
3593                 //
3594                 // Handles one line xml comment
3595                 //
3596                 private void handle_one_line_xml_comment ()
3597                 {
3598                         int c;
3599                         while ((c = peek_char ()) == ' ')
3600                                 get_char (); // skip heading whitespaces.
3601                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3602                                 xml_comment_buffer.Append ((char) get_char ());
3603                         }
3604                         if (c == '\r' || c == '\n')
3605                                 xml_comment_buffer.Append (Environment.NewLine);
3606                 }
3607
3608                 //
3609                 // Remove heading "*" in Javadoc-like xml documentation.
3610                 //
3611                 private void update_formatted_doc_comment (int current_comment_start)
3612                 {
3613                         int length = xml_comment_buffer.Length - current_comment_start;
3614                         string [] lines = xml_comment_buffer.ToString (
3615                                 current_comment_start,
3616                                 length).Replace ("\r", "").Split ('\n');
3617
3618                         // The first line starts with /**, thus it is not target
3619                         // for the format check.
3620                         for (int i = 1; i < lines.Length; i++) {
3621                                 string s = lines [i];
3622                                 int idx = s.IndexOf ('*');
3623                                 string head = null;
3624                                 if (idx < 0) {
3625                                         if (i < lines.Length - 1)
3626                                                 return;
3627                                         head = s;
3628                                 } else
3629                                         head = s.Substring (0, idx);
3630                                 foreach (char c in head)
3631                                         if (c != ' ')
3632                                                 return;
3633                                 lines [i] = s.Substring (idx + 1);
3634                         }
3635                         xml_comment_buffer.Remove (current_comment_start, length);
3636                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3637                 }
3638
3639                 //
3640                 // Checks if there was incorrect doc comments and raise
3641                 // warnings.
3642                 //
3643                 public void check_incorrect_doc_comment ()
3644                 {
3645                         if (xml_comment_buffer.Length > 0)
3646                                 WarningMisplacedComment (Location);
3647                 }
3648
3649                 //
3650                 // Consumes the saved xml comment lines (if any)
3651                 // as for current target member or type.
3652                 //
3653                 public string consume_doc_comment ()
3654                 {
3655                         if (xml_comment_buffer.Length > 0) {
3656                                 string ret = xml_comment_buffer.ToString ();
3657                                 reset_doc_comment ();
3658                                 return ret;
3659                         }
3660                         return null;
3661                 }
3662
3663                 Report Report {
3664                         get { return context.Report; }
3665                 }
3666
3667                 void reset_doc_comment ()
3668                 {
3669                         xml_comment_buffer.Length = 0;
3670                 }
3671
3672                 public void cleanup ()
3673                 {
3674                         if (ifstack != null && ifstack.Count >= 1) {
3675                                 int state = ifstack.Pop ();
3676                                 if ((state & REGION) != 0)
3677                                         Report.Error (1038, Location, "#endregion directive expected");
3678                                 else
3679                                         Report.Error (1027, Location, "Expected `#endif' directive");
3680                         }
3681                 }
3682         }
3683
3684         //
3685         // Indicates whether it accepts XML documentation or not.
3686         //
3687         public enum XmlCommentState {
3688                 // comment is allowed in this state.
3689                 Allowed,
3690                 // comment is not allowed in this state.
3691                 NotAllowed,
3692                 // once comments appeared when it is NotAllowed, then the
3693                 // state is changed to it, until the state is changed to
3694                 // .Allowed.
3695                 Error
3696         }
3697 }
3698