mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@gmail.com)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 // Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20
  21 namespace Mono.CSharp
  22 {
  23         /// <summary>
  24         ///    Tokenizer for C# source code.
  25         /// </summary>
  26
  27         public class Tokenizer : yyParser.yyInput
  28         {
  29                 class KeywordEntry<T>
  30                 {
  31                         public readonly T Token;
  32                         public KeywordEntry<T> Next;
  33                         public readonly char[] Value;
  34
  35                         public KeywordEntry (string value, T token)
  36                         {
  37                                 this.Value = value.ToCharArray ();
  38                                 this.Token = token;
  39                         }
  40                 }
  41
  42                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  43                 {
  44                         readonly int length;
  45
  46                         public IdentifiersComparer (int length)
  47                         {
  48                                 this.length = length;
  49                         }
  50
  51                         public bool Equals (char[] x, char[] y)
  52                         {
  53                                 for (int i = 0; i < length; ++i)
  54                                         if (x [i] != y [i])
  55                                                 return false;
  56
  57                                 return true;
  58                         }
  59
  60                         public int GetHashCode (char[] obj)
  61                         {
  62                                 int h = 0;
  63                                 for (int i = 0; i < length; ++i)
  64                                         h = (h << 5) - h + obj [i];
  65
  66                                 return h;
  67                         }
  68                 }
  69
  70                 //
  71                 // This class has to be used in the parser only, it reuses token
  72                 // details after each parse
  73                 //
  74                 public class LocatedToken
  75                 {
  76                         int row, column;
  77                         string value;
  78
  79                         static LocatedToken[] buffer;
  80                         static int pos;
  81
  82                         private LocatedToken ()
  83                         {
  84                         }
  85
  86                         public static LocatedToken Create (int row, int column)
  87                         {
  88                                 return Create (null, row, column);
  89                         }
  90
  91                         public static LocatedToken Create (string value, int row, int column)
  92                         {
  93                                 //
  94                                 // TODO: I am not very happy about the logic but it's the best
  95                                 // what I could come up with for now.
  96                                 // Ideally we should be using just tiny buffer (256 elements) which
  97                                 // is enough to hold all details for currect stack and recycle elements
  98                                 // poped from the stack but there is a trick needed to recycle
  99                                 // them properly.
 100                                 //
 101                                 LocatedToken entry;
 102                                 if (pos >= buffer.Length) {
 103                                         entry = new LocatedToken ();
 104                                 } else {
 105                                         entry = buffer [pos];
 106                                         if (entry == null) {
 107                                                 entry = new LocatedToken ();
 108                                                 buffer [pos] = entry;
 109                                         }
 110
 111                                         ++pos;
 112                                 }
 113                                 entry.value = value;
 114                                 entry.row = row;
 115                                 entry.column = column;
 116                                 return entry;
 117                         }
 118
 119                         //
 120                         // Used for token not required by expression evaluator
 121                         //
 122                         [Conditional ("FULL_AST")]
 123                         public static void CreateOptional (int row, int col, ref object token)
 124                         {
 125                                 token = Create (row, col);
 126                         }
 127
 128                         public static void Initialize ()
 129                         {
 130                                 if (buffer == null)
 131                                         buffer = new LocatedToken [10000];
 132                                 pos = 0;
 133                         }
 134
 135                         public Location Location {
 136                                 get { return new Location (row, column); }
 137                         }
 138
 139                         public string Value {
 140                                 get { return value; }
 141                         }
 142                 }
 143
 144                 enum PreprocessorDirective
 145                 {
 146                         Invalid = 0,
 147
 148                         Region = 1,
 149                         Endregion = 2,
 150                         If = 3 | RequiresArgument,
 151                         Endif = 4,
 152                         Elif = 5 | RequiresArgument,
 153                         Else = 6,
 154                         Define = 7 | RequiresArgument,
 155                         Undef = 8 | RequiresArgument,
 156                         Error = 9,
 157                         Warning = 10,
 158                         Pragma = 11 | CustomArgumentsParsing,
 159                         Line = 12,
 160
 161                         CustomArgumentsParsing = 1 << 10,
 162                         RequiresArgument = 1 << 11
 163                 }
 164
 165                 SeekableStreamReader reader;
 166                 SourceFile ref_name;
 167                 CompilationSourceFile file_name;
 168                 CompilerContext context;
 169                 bool hidden = false;
 170                 int ref_line = 1;
 171                 int line = 1;
 172                 int col = 0;
 173                 int previous_col;
 174                 int current_token;
 175                 int tab_size;
 176                 bool handle_get_set = false;
 177                 bool handle_remove_add = false;
 178                 bool handle_where = false;
 179                 bool handle_typeof = false;
 180                 bool lambda_arguments_parsing;
 181                 List<Location> escaped_identifiers;
 182                 int parsing_generic_less_than;
 183                 readonly bool doc_processing;
 184
 185                 //
 186                 // Used mainly for parser optimizations. Some expressions for instance
 187                 // can appear only in block (including initializer, base initializer)
 188                 // scope only
 189                 //
 190                 public int parsing_block;
 191                 internal bool query_parsing;
 192
 193                 //
 194                 // When parsing type only, useful for ambiguous nullable types
 195                 //
 196                 public int parsing_type;
 197
 198                 //
 199                 // Set when parsing generic declaration (type or method header)
 200                 //
 201                 public bool parsing_generic_declaration;
 202                 public bool parsing_generic_declaration_doc;
 203
 204                 //
 205                 // The value indicates that we have not reach any declaration or
 206                 // namespace yet
 207                 //
 208                 public int parsing_declaration;
 209
 210                 public bool parsing_attribute_section;
 211
 212                 public bool parsing_modifiers;
 213
 214                 public bool async_block;
 215
 216                 //
 217                 // The special characters to inject on streams to run the unit parser
 218                 // in the special expression mode. Using private characters from
 219                 // Plane Sixteen (U+100000 to U+10FFFD)
 220                 //
 221                 // This character is only tested just before the tokenizer is about to report
 222                 // an error;   So on the regular operation mode, this addition will have no
 223                 // impact on the tokenizer's performance.
 224                 //
 225
 226                 public const int EvalStatementParserCharacter = 0x100000;
 227                 public const int EvalCompilationUnitParserCharacter = 0x100001;
 228                 public const int EvalUsingDeclarationsParserCharacter = 0x100002;
 229                 public const int DocumentationXref = 0x100003;
 230
 231                 //
 232                 // XML documentation buffer. The save point is used to divide
 233                 // comments on types and comments on members.
 234                 //
 235                 StringBuilder xml_comment_buffer;
 236
 237                 //
 238                 // See comment on XmlCommentState enumeration.
 239                 //
 240                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 241
 242                 //
 243                 // Whether tokens have been seen on this line
 244                 //
 245                 bool tokens_seen = false;
 246
 247                 //
 248                 // Set to true once the GENERATE_COMPLETION token has bee
 249                 // returned.   This helps produce one GENERATE_COMPLETION,
 250                 // as many COMPLETE_COMPLETION as necessary to complete the
 251                 // AST tree and one final EOF.
 252                 //
 253                 bool generated;
 254
 255                 //
 256                 // Whether a token has been seen on the file
 257                 // This is needed because `define' is not allowed to be used
 258                 // after a token has been seen.
 259                 //
 260                 bool any_token_seen;
 261
 262                 //
 263                 // Class variables
 264                 //
 265                 static readonly KeywordEntry<int>[][] keywords;
 266                 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 267                 static readonly Dictionary<string, object> keyword_strings;             // TODO: HashSet
 268                 static readonly NumberStyles styles;
 269                 static readonly NumberFormatInfo csharp_format_info;
 270
 271                 // Pragma arguments
 272                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 273                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 274                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 275                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 276
 277                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 278
 279                 public bool PropertyParsing {
 280                         get { return handle_get_set; }
 281                         set { handle_get_set = value; }
 282                 }
 283
 284                 public bool EventParsing {
 285                         get { return handle_remove_add; }
 286                         set { handle_remove_add = value; }
 287                 }
 288
 289                 public bool ConstraintsParsing {
 290                         get { return handle_where; }
 291                         set { handle_where = value; }
 292                 }
 293
 294                 public bool TypeOfParsing {
 295                         get { return handle_typeof; }
 296                         set { handle_typeof = value; }
 297                 }
 298
 299                 public int TabSize {
 300                         get { return tab_size; }
 301                         set { tab_size = value; }
 302                 }
 303
 304                 public XmlCommentState doc_state {
 305                         get { return xml_doc_state; }
 306                         set {
 307                                 if (value == XmlCommentState.Allowed) {
 308                                         check_incorrect_doc_comment ();
 309                                         reset_doc_comment ();
 310                                 }
 311                                 xml_doc_state = value;
 312                         }
 313                 }
 314
 315                 //
 316                 // This is used to trigger completion generation on the parser
 317                 public bool CompleteOnEOF;
 318
 319                 void AddEscapedIdentifier (Location loc)
 320                 {
 321                         if (escaped_identifiers == null)
 322                                 escaped_identifiers = new List<Location> ();
 323
 324                         escaped_identifiers.Add (loc);
 325                 }
 326
 327                 public bool IsEscapedIdentifier (MemberName name)
 328                 {
 329                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 330                 }
 331
 332                 //
 333                 // Values for the associated token returned
 334                 //
 335                 internal int putback_char;      // Used by repl only
 336                 object val;
 337
 338                 //
 339                 // Pre-processor
 340                 //
 341                 const int TAKING        = 1;
 342                 const int ELSE_SEEN     = 4;
 343                 const int PARENT_TAKING = 8;
 344                 const int REGION        = 16;
 345
 346                 //
 347                 // pre-processor if stack state:
 348                 //
 349                 Stack<int> ifstack;
 350
 351                 static System.Text.StringBuilder string_builder;
 352
 353                 const int max_id_size = 512;
 354                 static readonly char [] id_builder = new char [max_id_size];
 355
 356                 public static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 357
 358                 const int max_number_size = 512;
 359                 static char [] number_builder = new char [max_number_size];
 360                 static int number_pos;
 361
 362                 static char[] value_builder = new char[256];
 363
 364                 public int Line {
 365                         get {
 366                                 return ref_line;
 367                         }
 368                 }
 369
 370                 //
 371                 // This is used when the tokenizer needs to save
 372                 // the current position as it needs to do some parsing
 373                 // on its own to deamiguate a token in behalf of the
 374                 // parser.
 375                 //
 376                 Stack<Position> position_stack = new Stack<Position> (2);
 377
 378                 class Position {
 379                         public int position;
 380                         public int line;
 381                         public int ref_line;
 382                         public int col;
 383                         public bool hidden;
 384                         public int putback_char;
 385                         public int previous_col;
 386                         public Stack<int> ifstack;
 387                         public int parsing_generic_less_than;
 388                         public int current_token;
 389                         public object val;
 390
 391                         public Position (Tokenizer t)
 392                         {
 393                                 position = t.reader.Position;
 394                                 line = t.line;
 395                                 ref_line = t.ref_line;
 396                                 col = t.col;
 397                                 hidden = t.hidden;
 398                                 putback_char = t.putback_char;
 399                                 previous_col = t.previous_col;
 400                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 401                                         // There is no simple way to clone Stack<T> all
 402                                         // methods reverse the order
 403                                         var clone = t.ifstack.ToArray ();
 404                                         Array.Reverse (clone);
 405                                         ifstack = new Stack<int> (clone);
 406                                 }
 407                                 parsing_generic_less_than = t.parsing_generic_less_than;
 408                                 current_token = t.current_token;
 409                                 val = t.val;
 410                         }
 411                 }
 412
 413                 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, CompilerContext ctx)
 414                 {
 415                         this.ref_name = file;
 416                         this.file_name = file;
 417                         this.context = ctx;
 418                         reader = input;
 419
 420                         putback_char = -1;
 421
 422                         xml_comment_buffer = new StringBuilder ();
 423                         doc_processing = ctx.Settings.DocumentationFile != null;
 424
 425                         if (Environment.OSVersion.Platform == PlatformID.Win32NT)
 426                                 tab_size = 4;
 427                         else
 428                                 tab_size = 8;
 429
 430                         //
 431                         // FIXME: This could be `Location.Push' but we have to
 432                         // find out why the MS compiler allows this
 433                         //
 434                         Mono.CSharp.Location.Push (file, file);
 435                 }
 436
 437                 public void PushPosition ()
 438                 {
 439                         position_stack.Push (new Position (this));
 440                 }
 441
 442                 public void PopPosition ()
 443                 {
 444                         Position p = position_stack.Pop ();
 445
 446                         reader.Position = p.position;
 447                         ref_line = p.ref_line;
 448                         line = p.line;
 449                         col = p.col;
 450                         hidden = p.hidden;
 451                         putback_char = p.putback_char;
 452                         previous_col = p.previous_col;
 453                         ifstack = p.ifstack;
 454                         parsing_generic_less_than = p.parsing_generic_less_than;
 455                         current_token = p.current_token;
 456                         val = p.val;
 457                 }
 458
 459                 // Do not reset the position, ignore it.
 460                 public void DiscardPosition ()
 461                 {
 462                         position_stack.Pop ();
 463                 }
 464
 465                 static void AddKeyword (string kw, int token)
 466                 {
 467                         keyword_strings.Add (kw, null);
 468
 469                         AddKeyword (keywords, kw, token);
 470                 }
 471
 472                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 473                 {
 474                         AddKeyword (keywords_preprocessor, kw, directive);
 475                 }
 476
 477                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 478                 {
 479                         int length = kw.Length;
 480                         if (keywords[length] == null) {
 481                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 482                         }
 483
 484                         int char_index = kw[0] - '_';
 485                         var kwe = keywords[length][char_index];
 486                         if (kwe == null) {
 487                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 488                                 return;
 489                         }
 490
 491                         while (kwe.Next != null) {
 492                                 kwe = kwe.Next;
 493                         }
 494
 495                         kwe.Next = new KeywordEntry<T> (kw, token);
 496                 }
 497
 498                 //
 499                 // Class initializer
 500                 //
 501                 static Tokenizer ()
 502                 {
 503                         keyword_strings = new Dictionary<string, object> ();
 504
 505                         // 11 is the length of the longest keyword for now
 506                         keywords = new KeywordEntry<int>[11][];
 507
 508                         AddKeyword ("__arglist", Token.ARGLIST);
 509                         AddKeyword ("__makeref", Token.MAKEREF);
 510                         AddKeyword ("__reftype", Token.REFTYPE);
 511                         AddKeyword ("__refvalue", Token.REFVALUE);
 512                         AddKeyword ("abstract", Token.ABSTRACT);
 513                         AddKeyword ("as", Token.AS);
 514                         AddKeyword ("add", Token.ADD);
 515                         AddKeyword ("base", Token.BASE);
 516                         AddKeyword ("bool", Token.BOOL);
 517                         AddKeyword ("break", Token.BREAK);
 518                         AddKeyword ("byte", Token.BYTE);
 519                         AddKeyword ("case", Token.CASE);
 520                         AddKeyword ("catch", Token.CATCH);
 521                         AddKeyword ("char", Token.CHAR);
 522                         AddKeyword ("checked", Token.CHECKED);
 523                         AddKeyword ("class", Token.CLASS);
 524                         AddKeyword ("const", Token.CONST);
 525                         AddKeyword ("continue", Token.CONTINUE);
 526                         AddKeyword ("decimal", Token.DECIMAL);
 527                         AddKeyword ("default", Token.DEFAULT);
 528                         AddKeyword ("delegate", Token.DELEGATE);
 529                         AddKeyword ("do", Token.DO);
 530                         AddKeyword ("double", Token.DOUBLE);
 531                         AddKeyword ("else", Token.ELSE);
 532                         AddKeyword ("enum", Token.ENUM);
 533                         AddKeyword ("event", Token.EVENT);
 534                         AddKeyword ("explicit", Token.EXPLICIT);
 535                         AddKeyword ("extern", Token.EXTERN);
 536                         AddKeyword ("false", Token.FALSE);
 537                         AddKeyword ("finally", Token.FINALLY);
 538                         AddKeyword ("fixed", Token.FIXED);
 539                         AddKeyword ("float", Token.FLOAT);
 540                         AddKeyword ("for", Token.FOR);
 541                         AddKeyword ("foreach", Token.FOREACH);
 542                         AddKeyword ("goto", Token.GOTO);
 543                         AddKeyword ("get", Token.GET);
 544                         AddKeyword ("if", Token.IF);
 545                         AddKeyword ("implicit", Token.IMPLICIT);
 546                         AddKeyword ("in", Token.IN);
 547                         AddKeyword ("int", Token.INT);
 548                         AddKeyword ("interface", Token.INTERFACE);
 549                         AddKeyword ("internal", Token.INTERNAL);
 550                         AddKeyword ("is", Token.IS);
 551                         AddKeyword ("lock", Token.LOCK);
 552                         AddKeyword ("long", Token.LONG);
 553                         AddKeyword ("namespace", Token.NAMESPACE);
 554                         AddKeyword ("new", Token.NEW);
 555                         AddKeyword ("null", Token.NULL);
 556                         AddKeyword ("object", Token.OBJECT);
 557                         AddKeyword ("operator", Token.OPERATOR);
 558                         AddKeyword ("out", Token.OUT);
 559                         AddKeyword ("override", Token.OVERRIDE);
 560                         AddKeyword ("params", Token.PARAMS);
 561                         AddKeyword ("private", Token.PRIVATE);
 562                         AddKeyword ("protected", Token.PROTECTED);
 563                         AddKeyword ("public", Token.PUBLIC);
 564                         AddKeyword ("readonly", Token.READONLY);
 565                         AddKeyword ("ref", Token.REF);
 566                         AddKeyword ("remove", Token.REMOVE);
 567                         AddKeyword ("return", Token.RETURN);
 568                         AddKeyword ("sbyte", Token.SBYTE);
 569                         AddKeyword ("sealed", Token.SEALED);
 570                         AddKeyword ("set", Token.SET);
 571                         AddKeyword ("short", Token.SHORT);
 572                         AddKeyword ("sizeof", Token.SIZEOF);
 573                         AddKeyword ("stackalloc", Token.STACKALLOC);
 574                         AddKeyword ("static", Token.STATIC);
 575                         AddKeyword ("string", Token.STRING);
 576                         AddKeyword ("struct", Token.STRUCT);
 577                         AddKeyword ("switch", Token.SWITCH);
 578                         AddKeyword ("this", Token.THIS);
 579                         AddKeyword ("throw", Token.THROW);
 580                         AddKeyword ("true", Token.TRUE);
 581                         AddKeyword ("try", Token.TRY);
 582                         AddKeyword ("typeof", Token.TYPEOF);
 583                         AddKeyword ("uint", Token.UINT);
 584                         AddKeyword ("ulong", Token.ULONG);
 585                         AddKeyword ("unchecked", Token.UNCHECKED);
 586                         AddKeyword ("unsafe", Token.UNSAFE);
 587                         AddKeyword ("ushort", Token.USHORT);
 588                         AddKeyword ("using", Token.USING);
 589                         AddKeyword ("virtual", Token.VIRTUAL);
 590                         AddKeyword ("void", Token.VOID);
 591                         AddKeyword ("volatile", Token.VOLATILE);
 592                         AddKeyword ("while", Token.WHILE);
 593                         AddKeyword ("partial", Token.PARTIAL);
 594                         AddKeyword ("where", Token.WHERE);
 595
 596                         // LINQ keywords
 597                         AddKeyword ("from", Token.FROM);
 598                         AddKeyword ("join", Token.JOIN);
 599                         AddKeyword ("on", Token.ON);
 600                         AddKeyword ("equals", Token.EQUALS);
 601                         AddKeyword ("select", Token.SELECT);
 602                         AddKeyword ("group", Token.GROUP);
 603                         AddKeyword ("by", Token.BY);
 604                         AddKeyword ("let", Token.LET);
 605                         AddKeyword ("orderby", Token.ORDERBY);
 606                         AddKeyword ("ascending", Token.ASCENDING);
 607                         AddKeyword ("descending", Token.DESCENDING);
 608                         AddKeyword ("into", Token.INTO);
 609
 610                         // Contextual async keywords
 611                         AddKeyword ("async", Token.ASYNC);
 612                         AddKeyword ("await", Token.AWAIT);
 613
 614                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 615
 616                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 617                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 618                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 619                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 620                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 621                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 622                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 623                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 624                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 625                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 626                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 627                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 628
 629                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 630                         styles = NumberStyles.Float;
 631
 632                         string_builder = new System.Text.StringBuilder ();
 633                 }
 634
 635                 int GetKeyword (char[] id, int id_len)
 636                 {
 637                         //
 638                         // Keywords are stored in an array of arrays grouped by their
 639                         // length and then by the first character
 640                         //
 641                         if (id_len >= keywords.Length || keywords [id_len] == null)
 642                                 return -1;
 643
 644                         int first_index = id [0] - '_';
 645                         if (first_index > 'z' - '_')
 646                                 return -1;
 647
 648                         var kwe = keywords [id_len] [first_index];
 649                         if (kwe == null)
 650                                 return -1;
 651
 652                         int res;
 653                         do {
 654                                 res = kwe.Token;
 655                                 for (int i = 1; i < id_len; ++i) {
 656                                         if (id [i] != kwe.Value [i]) {
 657                                                 res = 0;
 658                                                 kwe = kwe.Next;
 659                                                 break;
 660                                         }
 661                                 }
 662                         } while (res == 0 && kwe != null);
 663
 664                         if (res == 0)
 665                                 return -1;
 666
 667                         int next_token;
 668                         switch (res) {
 669                         case Token.GET:
 670                         case Token.SET:
 671                                 if (!handle_get_set)
 672                                         res = -1;
 673                                 break;
 674                         case Token.REMOVE:
 675                         case Token.ADD:
 676                                 if (!handle_remove_add)
 677                                         res = -1;
 678                                 break;
 679                         case Token.EXTERN:
 680                                 if (parsing_declaration == 0)
 681                                         res = Token.EXTERN_ALIAS;
 682                                 break;
 683                         case Token.DEFAULT:
 684                                 if (peek_token () == Token.COLON) {
 685                                         token ();
 686                                         res = Token.DEFAULT_COLON;
 687                                 }
 688                                 break;
 689                         case Token.WHERE:
 690                                 if (!handle_where && !query_parsing)
 691                                         res = -1;
 692                                 break;
 693                         case Token.FROM:
 694                                 //
 695                                 // A query expression is any expression that starts with `from identifier'
 696                                 // followed by any token except ; , =
 697                                 //
 698                                 if (!query_parsing) {
 699                                         if (lambda_arguments_parsing) {
 700                                                 res = -1;
 701                                                 break;
 702                                         }
 703
 704                                         PushPosition ();
 705                                         // HACK: to disable generics micro-parser, because PushPosition does not
 706                                         // store identifiers array
 707                                         parsing_generic_less_than = 1;
 708                                         switch (xtoken ()) {
 709                                         case Token.IDENTIFIER:
 710                                         case Token.INT:
 711                                         case Token.BOOL:
 712                                         case Token.BYTE:
 713                                         case Token.CHAR:
 714                                         case Token.DECIMAL:
 715                                         case Token.FLOAT:
 716                                         case Token.LONG:
 717                                         case Token.OBJECT:
 718                                         case Token.STRING:
 719                                         case Token.UINT:
 720                                         case Token.ULONG:
 721                                                 next_token = xtoken ();
 722                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 723                                                         goto default;
 724
 725                                                 res = Token.FROM_FIRST;
 726                                                 query_parsing = true;
 727                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 728                                                         Report.FeatureIsNotAvailable (context, Location, "query expressions");
 729                                                 break;
 730                                         case Token.VOID:
 731                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 732                                                 break;
 733                                         default:
 734                                                 PopPosition ();
 735                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 736                                                 // which has been overwritten before we grabbed the identifier
 737                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 738                                                 return -1;
 739                                         }
 740                                         PopPosition ();
 741                                 }
 742                                 break;
 743                         case Token.JOIN:
 744                         case Token.ON:
 745                         case Token.EQUALS:
 746                         case Token.SELECT:
 747                         case Token.GROUP:
 748                         case Token.BY:
 749                         case Token.LET:
 750                         case Token.ORDERBY:
 751                         case Token.ASCENDING:
 752                         case Token.DESCENDING:
 753                         case Token.INTO:
 754                                 if (!query_parsing)
 755                                         res = -1;
 756                                 break;
 757
 758                         case Token.USING:
 759                         case Token.NAMESPACE:
 760                                 // TODO: some explanation needed
 761                                 check_incorrect_doc_comment ();
 762                                 break;
 763
 764                         case Token.PARTIAL:
 765                                 if (parsing_block > 0) {
 766                                         res = -1;
 767                                         break;
 768                                 }
 769
 770                                 // Save current position and parse next token.
 771                                 PushPosition ();
 772
 773                                 next_token = token ();
 774                                 bool ok = (next_token == Token.CLASS) ||
 775                                         (next_token == Token.STRUCT) ||
 776                                         (next_token == Token.INTERFACE) ||
 777                                         (next_token == Token.VOID);
 778
 779                                 PopPosition ();
 780
 781                                 if (ok) {
 782                                         if (next_token == Token.VOID) {
 783                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 784                                                         Report.FeatureIsNotAvailable (context, Location, "partial methods");
 785                                         } else if (context.Settings.Version == LanguageVersion.ISO_1)
 786                                                 Report.FeatureIsNotAvailable (context, Location, "partial types");
 787
 788                                         return res;
 789                                 }
 790
 791                                 if (next_token < Token.LAST_KEYWORD) {
 792                                         Report.Error (267, Location,
 793                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 794                                         return token ();
 795                                 }
 796
 797                                 res = -1;
 798                                 break;
 799
 800                         case Token.ASYNC:
 801                                 if (parsing_modifiers) {
 802                                         //
 803                                         // Skip attributes section or constructor called async
 804                                         //
 805                                         if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) {
 806                                                 res = -1;
 807                                         } else {
 808                                                 // async is keyword
 809                                         }
 810                                 } else if (parsing_block > 0) {
 811                                         switch (peek_token ()) {
 812                                         case Token.DELEGATE:
 813                                         case Token.OPEN_PARENS_LAMBDA:
 814                                                 // async is keyword
 815                                                 break;
 816                                         case Token.IDENTIFIER:
 817                                                 PushPosition ();
 818                                                 xtoken ();
 819                                                 if (xtoken () != Token.ARROW)
 820                                                         res = -1;
 821
 822                                                 PopPosition ();
 823                                                 break;
 824                                         default:
 825                                                 res = -1;
 826                                                 break;
 827                                         }
 828                                 } else {
 829                                         res = -1;
 830                                 }
 831
 832                                 if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) {
 833                                         Report.FeatureIsNotAvailable (context, Location, "asynchronous functions");
 834                                 }
 835
 836                                 break;
 837
 838                         case Token.AWAIT:
 839                                 if (!async_block)
 840                                         res = -1;
 841
 842                                 break;
 843                         }
 844
 845
 846                         return res;
 847                 }
 848
 849                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 850                 {
 851                         //
 852                         // Keywords are stored in an array of arrays grouped by their
 853                         // length and then by the first character
 854                         //
 855                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 856                                 return PreprocessorDirective.Invalid;
 857
 858                         int first_index = id[0] - '_';
 859                         if (first_index > 'z' - '_')
 860                                 return PreprocessorDirective.Invalid;
 861
 862                         var kwe = keywords_preprocessor[id_len][first_index];
 863                         if (kwe == null)
 864                                 return PreprocessorDirective.Invalid;
 865
 866                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 867                         do {
 868                                 res = kwe.Token;
 869                                 for (int i = 1; i < id_len; ++i) {
 870                                         if (id[i] != kwe.Value[i]) {
 871                                                 res = 0;
 872                                                 kwe = kwe.Next;
 873                                                 break;
 874                                         }
 875                                 }
 876                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 877
 878                         return res;
 879                 }
 880
 881                 public Location Location {
 882                         get {
 883                                 return new Location (ref_line, hidden ? -1 : col);
 884                         }
 885                 }
 886
 887                 static bool is_identifier_start_character (int c)
 888                 {
 889                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 890                 }
 891
 892                 static bool is_identifier_part_character (char c)
 893                 {
 894                         if (c >= 'a' && c <= 'z')
 895                                 return true;
 896
 897                         if (c >= 'A' && c <= 'Z')
 898                                 return true;
 899
 900                         if (c == '_' || (c >= '0' && c <= '9'))
 901                                 return true;
 902
 903                         if (c < 0x80)
 904                                 return false;
 905
 906                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 907                 }
 908
 909                 public static bool IsKeyword (string s)
 910                 {
 911                         return keyword_strings.ContainsKey (s);
 912                 }
 913
 914                 //
 915                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 916                 //
 917                 int TokenizeOpenParens ()
 918                 {
 919                         int ptoken;
 920                         current_token = -1;
 921
 922                         int bracket_level = 0;
 923                         bool is_type = false;
 924                         bool can_be_type = false;
 925
 926                         while (true) {
 927                                 ptoken = current_token;
 928                                 token ();
 929
 930                                 switch (current_token) {
 931                                 case Token.CLOSE_PARENS:
 932                                         token ();
 933
 934                                         //
 935                                         // Expression inside parens is lambda, (int i) =>
 936                                         //
 937                                         if (current_token == Token.ARROW)
 938                                                 return Token.OPEN_PARENS_LAMBDA;
 939
 940                                         //
 941                                         // Expression inside parens is single type, (int[])
 942                                         //
 943                                         if (is_type)
 944                                                 return Token.OPEN_PARENS_CAST;
 945
 946                                         //
 947                                         // Expression is possible cast, look at next token, (T)null
 948                                         //
 949                                         if (can_be_type) {
 950                                                 switch (current_token) {
 951                                                 case Token.OPEN_PARENS:
 952                                                 case Token.BANG:
 953                                                 case Token.TILDE:
 954                                                 case Token.IDENTIFIER:
 955                                                 case Token.LITERAL:
 956                                                 case Token.BASE:
 957                                                 case Token.CHECKED:
 958                                                 case Token.DELEGATE:
 959                                                 case Token.FALSE:
 960                                                 case Token.FIXED:
 961                                                 case Token.NEW:
 962                                                 case Token.NULL:
 963                                                 case Token.SIZEOF:
 964                                                 case Token.THIS:
 965                                                 case Token.THROW:
 966                                                 case Token.TRUE:
 967                                                 case Token.TYPEOF:
 968                                                 case Token.UNCHECKED:
 969                                                 case Token.UNSAFE:
 970                                                 case Token.DEFAULT:
 971
 972                                                 //
 973                                                 // These can be part of a member access
 974                                                 //
 975                                                 case Token.INT:
 976                                                 case Token.UINT:
 977                                                 case Token.SHORT:
 978                                                 case Token.USHORT:
 979                                                 case Token.LONG:
 980                                                 case Token.ULONG:
 981                                                 case Token.DOUBLE:
 982                                                 case Token.FLOAT:
 983                                                 case Token.CHAR:
 984                                                 case Token.BYTE:
 985                                                 case Token.DECIMAL:
 986                                                 case Token.BOOL:
 987                                                         return Token.OPEN_PARENS_CAST;
 988                                                 }
 989                                         }
 990                                         return Token.OPEN_PARENS;
 991
 992                                 case Token.DOT:
 993                                 case Token.DOUBLE_COLON:
 994                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
 995                                                 goto default;
 996
 997                                         continue;
 998
 999                                 case Token.IDENTIFIER:
1000                                         switch (ptoken) {
1001                                         case Token.DOT:
1002                                                 if (bracket_level == 0) {
1003                                                         is_type = false;
1004                                                         can_be_type = true;
1005                                                 }
1006
1007                                                 continue;
1008                                         case Token.OP_GENERICS_LT:
1009                                         case Token.COMMA:
1010                                         case Token.DOUBLE_COLON:
1011                                         case -1:
1012                                                 if (bracket_level == 0)
1013                                                         can_be_type = true;
1014                                                 continue;
1015                                         default:
1016                                                 can_be_type = is_type = false;
1017                                                 continue;
1018                                         }
1019
1020                                 case Token.OBJECT:
1021                                 case Token.STRING:
1022                                 case Token.BOOL:
1023                                 case Token.DECIMAL:
1024                                 case Token.FLOAT:
1025                                 case Token.DOUBLE:
1026                                 case Token.SBYTE:
1027                                 case Token.BYTE:
1028                                 case Token.SHORT:
1029                                 case Token.USHORT:
1030                                 case Token.INT:
1031                                 case Token.UINT:
1032                                 case Token.LONG:
1033                                 case Token.ULONG:
1034                                 case Token.CHAR:
1035                                 case Token.VOID:
1036                                         if (bracket_level == 0)
1037                                                 is_type = true;
1038                                         continue;
1039
1040                                 case Token.COMMA:
1041                                         if (bracket_level == 0) {
1042                                                 bracket_level = 100;
1043                                                 can_be_type = is_type = false;
1044                                         }
1045                                         continue;
1046
1047                                 case Token.OP_GENERICS_LT:
1048                                 case Token.OPEN_BRACKET:
1049                                         if (bracket_level++ == 0)
1050                                                 is_type = true;
1051                                         continue;
1052
1053                                 case Token.OP_GENERICS_GT:
1054                                 case Token.CLOSE_BRACKET:
1055                                         --bracket_level;
1056                                         continue;
1057
1058                                 case Token.INTERR_NULLABLE:
1059                                 case Token.STAR:
1060                                         if (bracket_level == 0)
1061                                                 is_type = true;
1062                                         continue;
1063
1064                                 case Token.REF:
1065                                 case Token.OUT:
1066                                         can_be_type = is_type = false;
1067                                         continue;
1068
1069                                 default:
1070                                         return Token.OPEN_PARENS;
1071                                 }
1072                         }
1073                 }
1074
1075                 public static bool IsValidIdentifier (string s)
1076                 {
1077                         if (s == null || s.Length == 0)
1078                                 return false;
1079
1080                         if (!is_identifier_start_character (s [0]))
1081                                 return false;
1082
1083                         for (int i = 1; i < s.Length; i ++)
1084                                 if (! is_identifier_part_character (s [i]))
1085                                         return false;
1086
1087                         return true;
1088                 }
1089
1090                 bool parse_less_than ()
1091                 {
1092                 start:
1093                         int the_token = token ();
1094                         if (the_token == Token.OPEN_BRACKET) {
1095                                 do {
1096                                         the_token = token ();
1097                                 } while (the_token != Token.CLOSE_BRACKET);
1098                                 the_token = token ();
1099                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1100                                 the_token = token ();
1101                         }
1102                         switch (the_token) {
1103                         case Token.IDENTIFIER:
1104                         case Token.OBJECT:
1105                         case Token.STRING:
1106                         case Token.BOOL:
1107                         case Token.DECIMAL:
1108                         case Token.FLOAT:
1109                         case Token.DOUBLE:
1110                         case Token.SBYTE:
1111                         case Token.BYTE:
1112                         case Token.SHORT:
1113                         case Token.USHORT:
1114                         case Token.INT:
1115                         case Token.UINT:
1116                         case Token.LONG:
1117                         case Token.ULONG:
1118                         case Token.CHAR:
1119                         case Token.VOID:
1120                                 break;
1121                         case Token.OP_GENERICS_GT:
1122                         case Token.IN:
1123                         case Token.OUT:
1124                                 return true;
1125
1126                         default:
1127                                 return false;
1128                         }
1129                 again:
1130                         the_token = token ();
1131
1132                         if (the_token == Token.OP_GENERICS_GT)
1133                                 return true;
1134                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1135                                 goto start;
1136                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1137                                 goto again;
1138                         else if (the_token == Token.OP_GENERICS_LT) {
1139                                 if (!parse_less_than ())
1140                                         return false;
1141                                 goto again;
1142                         } else if (the_token == Token.OPEN_BRACKET) {
1143                         rank_specifiers:
1144                                 the_token = token ();
1145                                 if (the_token == Token.CLOSE_BRACKET)
1146                                         goto again;
1147                                 else if (the_token == Token.COMMA)
1148                                         goto rank_specifiers;
1149                                 return false;
1150                         }
1151
1152                         return false;
1153                 }
1154
1155                 bool parse_generic_dimension (out int dimension)
1156                 {
1157                         dimension = 1;
1158
1159                 again:
1160                         int the_token = token ();
1161                         if (the_token == Token.OP_GENERICS_GT)
1162                                 return true;
1163                         else if (the_token == Token.COMMA) {
1164                                 dimension++;
1165                                 goto again;
1166                         }
1167
1168                         return false;
1169                 }
1170
1171                 public int peek_token ()
1172                 {
1173                         int the_token;
1174
1175                         PushPosition ();
1176                         the_token = token ();
1177                         PopPosition ();
1178
1179                         return the_token;
1180                 }
1181
1182                 //
1183                 // Tonizes `?' using custom disambiguous rules to return one
1184                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1185                 //
1186                 // Tricky expression look like:
1187                 //
1188                 // Foo ? a = x ? b : c;
1189                 //
1190                 int TokenizePossibleNullableType ()
1191                 {
1192                         if (parsing_block == 0 || parsing_type > 0)
1193                                 return Token.INTERR_NULLABLE;
1194
1195                         int d = peek_char ();
1196                         if (d == '?') {
1197                                 get_char ();
1198                                 return Token.OP_COALESCING;
1199                         }
1200
1201                         switch (current_token) {
1202                         case Token.CLOSE_PARENS:
1203                         case Token.TRUE:
1204                         case Token.FALSE:
1205                         case Token.NULL:
1206                         case Token.LITERAL:
1207                                 return Token.INTERR;
1208                         }
1209
1210                         if (d != ' ') {
1211                                 if (d == ',' || d == ';' || d == '>')
1212                                         return Token.INTERR_NULLABLE;
1213                                 if (d == '*' || (d >= '0' && d <= '9'))
1214                                         return Token.INTERR;
1215                         }
1216
1217                         PushPosition ();
1218                         current_token = Token.NONE;
1219                         int next_token;
1220                         switch (xtoken ()) {
1221                         case Token.LITERAL:
1222                         case Token.TRUE:
1223                         case Token.FALSE:
1224                         case Token.NULL:
1225                         case Token.THIS:
1226                         case Token.NEW:
1227                                 next_token = Token.INTERR;
1228                                 break;
1229
1230                         case Token.SEMICOLON:
1231                         case Token.COMMA:
1232                         case Token.CLOSE_PARENS:
1233                         case Token.OPEN_BRACKET:
1234                         case Token.OP_GENERICS_GT:
1235                         case Token.INTERR:
1236                                 next_token = Token.INTERR_NULLABLE;
1237                                 break;
1238
1239                         default:
1240                                 next_token = -1;
1241                                 break;
1242                         }
1243
1244                         if (next_token == -1) {
1245                                 switch (xtoken ()) {
1246                                 case Token.COMMA:
1247                                 case Token.SEMICOLON:
1248                                 case Token.OPEN_BRACE:
1249                                 case Token.CLOSE_PARENS:
1250                                 case Token.IN:
1251                                         next_token = Token.INTERR_NULLABLE;
1252                                         break;
1253
1254                                 case Token.COLON:
1255                                         next_token = Token.INTERR;
1256                                         break;
1257
1258                                 default:
1259                                         int ntoken;
1260                                         int interrs = 1;
1261                                         int colons = 0;
1262                                         //
1263                                         // All shorcuts failed, do it hard way
1264                                         //
1265                                         while ((ntoken = xtoken ()) != Token.EOF) {
1266                                                 if (ntoken == Token.SEMICOLON)
1267                                                         break;
1268
1269                                                 if (ntoken == Token.COLON) {
1270                                                         if (++colons == interrs)
1271                                                                 break;
1272                                                         continue;
1273                                                 }
1274
1275                                                 if (ntoken == Token.INTERR) {
1276                                                         ++interrs;
1277                                                         continue;
1278                                                 }
1279                                         }
1280
1281                                         next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1282                                         break;
1283                                 }
1284                         }
1285
1286                         PopPosition ();
1287                         return next_token;
1288                 }
1289
1290                 bool decimal_digits (int c)
1291                 {
1292                         int d;
1293                         bool seen_digits = false;
1294
1295                         if (c != -1){
1296                                 if (number_pos == max_number_size)
1297                                         Error_NumericConstantTooLong ();
1298                                 number_builder [number_pos++] = (char) c;
1299                         }
1300
1301                         //
1302                         // We use peek_char2, because decimal_digits needs to do a
1303                         // 2-character look-ahead (5.ToString for example).
1304                         //
1305                         while ((d = peek_char2 ()) != -1){
1306                                 if (d >= '0' && d <= '9'){
1307                                         if (number_pos == max_number_size)
1308                                                 Error_NumericConstantTooLong ();
1309                                         number_builder [number_pos++] = (char) d;
1310                                         get_char ();
1311                                         seen_digits = true;
1312                                 } else
1313                                         break;
1314                         }
1315
1316                         return seen_digits;
1317                 }
1318
1319                 static bool is_hex (int e)
1320                 {
1321                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1322                 }
1323
1324                 static TypeCode real_type_suffix (int c)
1325                 {
1326                         switch (c){
1327                         case 'F': case 'f':
1328                                 return TypeCode.Single;
1329                         case 'D': case 'd':
1330                                 return TypeCode.Double;
1331                         case 'M': case 'm':
1332                                 return TypeCode.Decimal;
1333                         default:
1334                                 return TypeCode.Empty;
1335                         }
1336                 }
1337
1338                 ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc)
1339                 {
1340                         bool is_unsigned = false;
1341                         bool is_long = false;
1342
1343                         if (c != -1){
1344                                 bool scanning = true;
1345                                 do {
1346                                         switch (c){
1347                                         case 'U': case 'u':
1348                                                 if (is_unsigned)
1349                                                         scanning = false;
1350                                                 is_unsigned = true;
1351                                                 get_char ();
1352                                                 break;
1353
1354                                         case 'l':
1355                                                 if (!is_unsigned){
1356                                                         //
1357                                                         // if we have not seen anything in between
1358                                                         // report this error
1359                                                         //
1360                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1361                                                 }
1362
1363                                                 goto case 'L';
1364
1365                                         case 'L':
1366                                                 if (is_long)
1367                                                         scanning = false;
1368                                                 is_long = true;
1369                                                 get_char ();
1370                                                 break;
1371
1372                                         default:
1373                                                 scanning = false;
1374                                                 break;
1375                                         }
1376                                         c = peek_char ();
1377                                 } while (scanning);
1378                         }
1379
1380                         if (is_long && is_unsigned){
1381                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1382                         }
1383
1384                         if (is_unsigned){
1385                                 // uint if possible, or ulong else.
1386
1387                                 if ((ul & 0xffffffff00000000) == 0)
1388                                         return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc);
1389                                 else
1390                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1391                         } else if (is_long){
1392                                 // long if possible, ulong otherwise
1393                                 if ((ul & 0x8000000000000000) != 0)
1394                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1395                                 else
1396                                         return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1397                         } else {
1398                                 // int, uint, long or ulong in that order
1399                                 if ((ul & 0xffffffff00000000) == 0){
1400                                         uint ui = (uint) ul;
1401
1402                                         if ((ui & 0x80000000) != 0)
1403                                                 return new UIntLiteral (context.BuiltinTypes, ui, loc);
1404                                         else
1405                                                 return new IntLiteral (context.BuiltinTypes, (int) ui, loc);
1406                                 } else {
1407                                         if ((ul & 0x8000000000000000) != 0)
1408                                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1409                                         else
1410                                                 return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1411                                 }
1412                         }
1413                 }
1414
1415                 //
1416                 // given `c' as the next char in the input decide whether
1417                 // we need to convert to a special type, and then choose
1418                 // the best representation for the integer
1419                 //
1420                 ILiteralConstant adjust_int (int c, Location loc)
1421                 {
1422                         try {
1423                                 if (number_pos > 9){
1424                                         ulong ul = (uint) (number_builder [0] - '0');
1425
1426                                         for (int i = 1; i < number_pos; i++){
1427                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1428                                         }
1429
1430                                         return integer_type_suffix (ul, c, loc);
1431                                 } else {
1432                                         uint ui = (uint) (number_builder [0] - '0');
1433
1434                                         for (int i = 1; i < number_pos; i++){
1435                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1436                                         }
1437
1438                                         return integer_type_suffix (ui, c, loc);
1439                                 }
1440                         } catch (OverflowException) {
1441                                 Error_NumericConstantTooLong ();
1442                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1443                         }
1444                         catch (FormatException) {
1445                                 Report.Error (1013, Location, "Invalid number");
1446                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1447                         }
1448                 }
1449
1450                 ILiteralConstant adjust_real (TypeCode t, Location loc)
1451                 {
1452                         string s = new string (number_builder, 0, number_pos);
1453                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1454
1455                         switch (t){
1456                         case TypeCode.Decimal:
1457                                 try {
1458                                         return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc);
1459                                 } catch (OverflowException) {
1460                                         Report.Error (594, Location, error_details, "decimal");
1461                                         return new DecimalLiteral (context.BuiltinTypes, 0, loc);
1462                                 }
1463                         case TypeCode.Single:
1464                                 try {
1465                                         return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc);
1466                                 } catch (OverflowException) {
1467                                         Report.Error (594, Location, error_details, "float");
1468                                         return new FloatLiteral (context.BuiltinTypes, 0, loc);
1469                                 }
1470                         default:
1471                                 try {
1472                                         return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc);
1473                                 } catch (OverflowException) {
1474                                         Report.Error (594, loc, error_details, "double");
1475                                         return new DoubleLiteral (context.BuiltinTypes, 0, loc);
1476                                 }
1477                         }
1478                 }
1479
1480                 ILiteralConstant handle_hex (Location loc)
1481                 {
1482                         int d;
1483                         ulong ul;
1484
1485                         get_char ();
1486                         while ((d = peek_char ()) != -1){
1487                                 if (is_hex (d)){
1488                                         number_builder [number_pos++] = (char) d;
1489                                         get_char ();
1490                                 } else
1491                                         break;
1492                         }
1493
1494                         string s = new String (number_builder, 0, number_pos);
1495
1496                         try {
1497                                 if (number_pos <= 8)
1498                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1499                                 else
1500                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1501
1502                                 return integer_type_suffix (ul, peek_char (), loc);
1503                         } catch (OverflowException){
1504                                 Error_NumericConstantTooLong ();
1505                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1506                         }
1507                         catch (FormatException) {
1508                                 Report.Error (1013, Location, "Invalid number");
1509                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1510                         }
1511                 }
1512
1513                 //
1514                 // Invoked if we know we have .digits or digits
1515                 //
1516                 int is_number (int c)
1517                 {
1518                         ILiteralConstant res;
1519
1520 #if FULL_AST
1521                         int read_start = reader.Position - 1;
1522 #endif
1523                         number_pos = 0;
1524                         var loc = Location;
1525
1526                         if (c >= '0' && c <= '9'){
1527                                 if (c == '0'){
1528                                         int peek = peek_char ();
1529
1530                                         if (peek == 'x' || peek == 'X') {
1531                                                 val = res = handle_hex (loc);
1532 #if FULL_AST
1533                                                 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1534 #endif
1535
1536                                                 return Token.LITERAL;
1537                                         }
1538                                 }
1539                                 decimal_digits (c);
1540                                 c = get_char ();
1541                         }
1542
1543                         //
1544                         // We need to handle the case of
1545                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1546                         //
1547                         bool is_real = false;
1548                         if (c == '.'){
1549                                 if (decimal_digits ('.')){
1550                                         is_real = true;
1551                                         c = get_char ();
1552                                 } else {
1553                                         putback ('.');
1554                                         number_pos--;
1555                                         val = res = adjust_int (-1, loc);
1556
1557 #if FULL_AST
1558                                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1559 #endif
1560                                         return Token.LITERAL;
1561                                 }
1562                         }
1563
1564                         if (c == 'e' || c == 'E'){
1565                                 is_real = true;
1566                                 if (number_pos == max_number_size)
1567                                         Error_NumericConstantTooLong ();
1568                                 number_builder [number_pos++] = (char) c;
1569                                 c = get_char ();
1570
1571                                 if (c == '+'){
1572                                         if (number_pos == max_number_size)
1573                                                 Error_NumericConstantTooLong ();
1574                                         number_builder [number_pos++] = '+';
1575                                         c = -1;
1576                                 } else if (c == '-') {
1577                                         if (number_pos == max_number_size)
1578                                                 Error_NumericConstantTooLong ();
1579                                         number_builder [number_pos++] = '-';
1580                                         c = -1;
1581                                 } else {
1582                                         if (number_pos == max_number_size)
1583                                                 Error_NumericConstantTooLong ();
1584                                         number_builder [number_pos++] = '+';
1585                                 }
1586
1587                                 decimal_digits (c);
1588                                 c = get_char ();
1589                         }
1590
1591                         var type = real_type_suffix (c);
1592                         if (type == TypeCode.Empty && !is_real) {
1593                                 putback (c);
1594                                 res = adjust_int (c, loc);
1595                         } else {
1596                                 is_real = true;
1597
1598                                 if (type == TypeCode.Empty) {
1599                                         putback (c);
1600                                 }
1601
1602                                 res = adjust_real (type, loc);
1603                         }
1604
1605                         val = res;
1606
1607 #if FULL_AST
1608                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty ? 1 : 0));
1609 #endif
1610
1611                         return Token.LITERAL;
1612                 }
1613
1614                 //
1615                 // Accepts exactly count (4 or 8) hex, no more no less
1616                 //
1617                 int getHex (int count, out int surrogate, out bool error)
1618                 {
1619                         int i;
1620                         int total = 0;
1621                         int c;
1622                         int top = count != -1 ? count : 4;
1623
1624                         get_char ();
1625                         error = false;
1626                         surrogate = 0;
1627                         for (i = 0; i < top; i++){
1628                                 c = get_char ();
1629
1630                                 if (c >= '0' && c <= '9')
1631                                         c = (int) c - (int) '0';
1632                                 else if (c >= 'A' && c <= 'F')
1633                                         c = (int) c - (int) 'A' + 10;
1634                                 else if (c >= 'a' && c <= 'f')
1635                                         c = (int) c - (int) 'a' + 10;
1636                                 else {
1637                                         error = true;
1638                                         return 0;
1639                                 }
1640
1641                                 total = (total * 16) + c;
1642                                 if (count == -1){
1643                                         int p = peek_char ();
1644                                         if (p == -1)
1645                                                 break;
1646                                         if (!is_hex ((char)p))
1647                                                 break;
1648                                 }
1649                         }
1650
1651                         if (top == 8) {
1652                                 if (total > 0x0010FFFF) {
1653                                         error = true;
1654                                         return 0;
1655                                 }
1656
1657                                 if (total >= 0x00010000) {
1658                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1659                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1660                                 }
1661                         }
1662
1663                         return total;
1664                 }
1665
1666                 int escape (int c, out int surrogate)
1667                 {
1668                         bool error;
1669                         int d;
1670                         int v;
1671
1672                         d = peek_char ();
1673                         if (c != '\\') {
1674                                 surrogate = 0;
1675                                 return c;
1676                         }
1677
1678                         switch (d){
1679                         case 'a':
1680                                 v = '\a'; break;
1681                         case 'b':
1682                                 v = '\b'; break;
1683                         case 'n':
1684                                 v = '\n'; break;
1685                         case 't':
1686                                 v = '\t'; break;
1687                         case 'v':
1688                                 v = '\v'; break;
1689                         case 'r':
1690                                 v = '\r'; break;
1691                         case '\\':
1692                                 v = '\\'; break;
1693                         case 'f':
1694                                 v = '\f'; break;
1695                         case '0':
1696                                 v = 0; break;
1697                         case '"':
1698                                 v = '"'; break;
1699                         case '\'':
1700                                 v = '\''; break;
1701                         case 'x':
1702                                 v = getHex (-1, out surrogate, out error);
1703                                 if (error)
1704                                         goto default;
1705                                 return v;
1706                         case 'u':
1707                         case 'U':
1708                                 return EscapeUnicode (d, out surrogate);
1709                         default:
1710                                 surrogate = 0;
1711                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1712                                 return d;
1713                         }
1714
1715                         get_char ();
1716                         surrogate = 0;
1717                         return v;
1718                 }
1719
1720                 int EscapeUnicode (int ch, out int surrogate)
1721                 {
1722                         bool error;
1723                         if (ch == 'U') {
1724                                 ch = getHex (8, out surrogate, out error);
1725                         } else {
1726                                 ch = getHex (4, out surrogate, out error);
1727                         }
1728
1729                         if (error)
1730                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1731
1732                         return ch;
1733                 }
1734
1735                 int get_char ()
1736                 {
1737                         int x;
1738                         if (putback_char != -1) {
1739                                 x = putback_char;
1740                                 putback_char = -1;
1741                         } else {
1742                                 x = reader.Read ();
1743                         }
1744
1745                         if (x == '\r') {
1746                                 if (peek_char () == '\n') {
1747                                         putback_char = -1;
1748                                 }
1749
1750                                 x = '\n';
1751                                 advance_line ();
1752                         } else if (x == '\n') {
1753                                 advance_line ();
1754                         } else {
1755                                 col++;
1756                         }
1757                         return x;
1758                 }
1759
1760                 void advance_line ()
1761                 {
1762                         line++;
1763                         ref_line++;
1764                         previous_col = col;
1765                         col = 0;
1766                 }
1767
1768                 int peek_char ()
1769                 {
1770                         if (putback_char == -1)
1771                                 putback_char = reader.Read ();
1772                         return putback_char;
1773                 }
1774
1775                 int peek_char2 ()
1776                 {
1777                         if (putback_char != -1)
1778                                 return putback_char;
1779                         return reader.Peek ();
1780                 }
1781
1782                 void putback (int c)
1783                 {
1784                         if (putback_char != -1){
1785                                 Console.WriteLine ("Col: " + col);
1786                                 Console.WriteLine ("Row: " + line);
1787                                 Console.WriteLine ("Name: " + ref_name.Name);
1788                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1789                                 throw new Exception ("This should not happen putback on putback");
1790                         }
1791                         if (c == '\n' || col == 0) {
1792                                 // It won't happen though.
1793                                 line--;
1794                                 ref_line--;
1795                                 col = previous_col;
1796                         }
1797                         else
1798                                 col--;
1799                         putback_char = c;
1800                 }
1801
1802                 public bool advance ()
1803                 {
1804                         return peek_char () != -1 || CompleteOnEOF;
1805                 }
1806
1807                 public Object Value {
1808                         get {
1809                                 return val;
1810                         }
1811                 }
1812
1813                 public Object value ()
1814                 {
1815                         return val;
1816                 }
1817
1818                 public int token ()
1819                 {
1820                         current_token = xtoken ();
1821                         return current_token;
1822                 }
1823
1824                 int TokenizePreprocessorIdentifier (out int c)
1825                 {
1826                         // skip over white space
1827                         do {
1828                                 c = get_char ();
1829                         } while (c == ' ' || c == '\t');
1830
1831
1832                         int pos = 0;
1833                         while (c != -1 && c >= 'a' && c <= 'z') {
1834                                 id_builder[pos++] = (char) c;
1835                                 c = get_char ();
1836                                 if (c == '\\') {
1837                                         int peek = peek_char ();
1838                                         if (peek == 'U' || peek == 'u') {
1839                                                 int surrogate;
1840                                                 c = EscapeUnicode (c, out surrogate);
1841                                                 if (surrogate != 0) {
1842                                                         if (is_identifier_part_character ((char) c)) {
1843                                                                 id_builder[pos++] = (char) c;
1844                                                         }
1845                                                         c = surrogate;
1846                                                 }
1847                                         }
1848                                 }
1849                         }
1850
1851                         return pos;
1852                 }
1853
1854                 PreprocessorDirective get_cmd_arg (out string arg)
1855                 {
1856                         int c;
1857
1858                         tokens_seen = false;
1859                         arg = "";
1860
1861                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorIdentifier (out c));
1862
1863                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
1864                                 return cmd;
1865
1866                         // skip over white space
1867                         while (c == ' ' || c == '\t')
1868                                 c = get_char ();
1869
1870                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
1871                         int pos = 0;
1872
1873                         while (c != -1 && c != '\n') {
1874                                 if (c == '\\' && has_identifier_argument >= 0) {
1875                                         if (has_identifier_argument != 0) {
1876                                                 has_identifier_argument = 1;
1877
1878                                                 int peek = peek_char ();
1879                                                 if (peek == 'U' || peek == 'u') {
1880                                                         int surrogate;
1881                                                         c = EscapeUnicode (c, out surrogate);
1882                                                         if (surrogate != 0) {
1883                                                                 if (is_identifier_part_character ((char) c)) {
1884                                                                         if (pos == value_builder.Length)
1885                                                                                 Array.Resize (ref value_builder, pos * 2);
1886
1887                                                                         value_builder[pos++] = (char) c;
1888                                                                 }
1889                                                                 c = surrogate;
1890                                                         }
1891                                                 }
1892                                         } else {
1893                                                 has_identifier_argument = -1;
1894                                         }
1895                                 } else if (c == '/' && peek_char () == '/') {
1896                                         //
1897                                         // Eat single-line comments
1898                                         //
1899                                         get_char ();
1900                                         do {
1901                                                 c = get_char ();
1902                                         } while (c != -1 && c != '\n');
1903
1904                                         break;
1905                                 }
1906
1907                                 if (pos == value_builder.Length)
1908                                         Array.Resize (ref value_builder, pos * 2);
1909
1910                                 value_builder[pos++] = (char) c;
1911                                 c = get_char ();
1912                         }
1913
1914                         if (pos != 0) {
1915                                 if (pos > max_id_size)
1916                                         arg = new string (value_builder, 0, pos);
1917                                 else
1918                                         arg = InternIdentifier (value_builder, pos);
1919
1920                                 // Eat any trailing whitespaces
1921                                 arg = arg.Trim (simple_whitespaces);
1922                         }
1923
1924                         return cmd;
1925                 }
1926
1927                 //
1928                 // Handles the #line directive
1929                 //
1930                 bool PreProcessLine (string arg)
1931                 {
1932                         if (arg.Length == 0)
1933                                 return false;
1934
1935                         if (arg == "default"){
1936                                 ref_line = line;
1937                                 ref_name = file_name;
1938                                 hidden = false;
1939                                 Location.Push (file_name, ref_name);
1940                                 return true;
1941                         } else if (arg == "hidden"){
1942                                 hidden = true;
1943                                 return true;
1944                         }
1945
1946                         try {
1947                                 int pos;
1948
1949                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1950                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1951                                         pos++;
1952
1953                                         char [] quotes = { '\"' };
1954
1955                                         string name = arg.Substring (pos). Trim (quotes);
1956                                         ref_name = context.LookupFile (file_name, name);
1957                                         file_name.AddIncludeFile (ref_name);
1958                                         hidden = false;
1959                                         Location.Push (file_name, ref_name);
1960                                 } else {
1961                                         ref_line = System.Int32.Parse (arg);
1962                                         hidden = false;
1963                                 }
1964                         } catch {
1965                                 return false;
1966                         }
1967
1968                         return true;
1969                 }
1970
1971                 //
1972                 // Handles #define and #undef
1973                 //
1974                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
1975                 {
1976                         if (ident.Length == 0 || ident == "true" || ident == "false"){
1977                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
1978                                 return;
1979                         }
1980
1981                         if (ident.IndexOfAny (simple_whitespaces) != -1){
1982                                 Error_EndLineExpected ();
1983                                 return;
1984                         }
1985
1986                         if (!is_identifier_start_character (ident [0]))
1987                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
1988
1989                         foreach (char c in ident.Substring (1)){
1990                                 if (!is_identifier_part_character (c)){
1991                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
1992                                         return;
1993                                 }
1994                         }
1995
1996                         if (!caller_is_taking)
1997                                 return;
1998
1999                         if (is_define) {
2000                                 //
2001                                 // #define ident
2002                                 //
2003                                 if (context.Settings.IsConditionalSymbolDefined (ident))
2004                                         return;
2005
2006                                 file_name.AddDefine (ident);
2007                         } else {
2008                                 //
2009                                 // #undef ident
2010                                 //
2011                                 file_name.AddUndefine (ident);
2012                         }
2013                 }
2014
2015                 byte read_hex (out bool error)
2016                 {
2017                         int total;
2018                         int c = get_char ();
2019
2020                         if ((c >= '0') && (c <= '9'))
2021                                 total = (int) c - (int) '0';
2022                         else if ((c >= 'A') && (c <= 'F'))
2023                                 total = (int) c - (int) 'A' + 10;
2024                         else if ((c >= 'a') && (c <= 'f'))
2025                                 total = (int) c - (int) 'a' + 10;
2026                         else {
2027                                 error = true;
2028                                 return 0;
2029                         }
2030
2031                         total *= 16;
2032                         c = get_char ();
2033
2034                         if ((c >= '0') && (c <= '9'))
2035                                 total += (int) c - (int) '0';
2036                         else if ((c >= 'A') && (c <= 'F'))
2037                                 total += (int) c - (int) 'A' + 10;
2038                         else if ((c >= 'a') && (c <= 'f'))
2039                                 total += (int) c - (int) 'a' + 10;
2040                         else {
2041                                 error = true;
2042                                 return 0;
2043                         }
2044
2045                         error = false;
2046                         return (byte) total;
2047                 }
2048
2049                 //
2050                 // Parses #pragma checksum
2051                 //
2052                 bool ParsePragmaChecksum ()
2053                 {
2054                         //
2055                         // The syntax is ` "foo.txt" "{guid}" "hash"'
2056                         //
2057                         int c = get_char ();
2058
2059                         if (c != '"')
2060                                 return false;
2061
2062                         string_builder.Length = 0;
2063                         while (c != -1 && c != '\n') {
2064                                 c = get_char ();
2065                                 if (c == '"') {
2066                                         c = get_char ();
2067                                         break;
2068                                 }
2069
2070                                 string_builder.Append ((char) c);
2071                         }
2072
2073                         if (string_builder.Length == 0) {
2074                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
2075                         }
2076
2077                         // TODO: Any white-spaces count
2078                         if (c != ' ')
2079                                 return false;
2080
2081                         SourceFile file = context.LookupFile (file_name, string_builder.ToString ());
2082
2083                         if (get_char () != '"' || get_char () != '{')
2084                                 return false;
2085
2086                         bool error;
2087                         byte[] guid_bytes = new byte [16];
2088                         int i = 0;
2089
2090                         for (; i < 4; i++) {
2091                                 guid_bytes [i] = read_hex (out error);
2092                                 if (error)
2093                                         return false;
2094                         }
2095
2096                         if (get_char () != '-')
2097                                 return false;
2098
2099                         for (; i < 10; i++) {
2100                                 guid_bytes [i] = read_hex (out error);
2101                                 if (error)
2102                                         return false;
2103
2104                                 guid_bytes [i++] = read_hex (out error);
2105                                 if (error)
2106                                         return false;
2107
2108                                 if (get_char () != '-')
2109                                         return false;
2110                         }
2111
2112                         for (; i < 16; i++) {
2113                                 guid_bytes [i] = read_hex (out error);
2114                                 if (error)
2115                                         return false;
2116                         }
2117
2118                         if (get_char () != '}' || get_char () != '"')
2119                                 return false;
2120
2121                         // TODO: Any white-spaces count
2122                         c = get_char ();
2123                         if (c != ' ')
2124                                 return false;
2125
2126                         if (get_char () != '"')
2127                                 return false;
2128
2129                         // Any length of checksum
2130                         List<byte> checksum_bytes = new List<byte> (16);
2131
2132                         c = peek_char ();
2133                         while (c != '"' && c != -1) {
2134                                 checksum_bytes.Add (read_hex (out error));
2135                                 if (error)
2136                                         return false;
2137
2138                                 c = peek_char ();
2139                         }
2140
2141                         if (c == '/') {
2142                                 ReadSingleLineComment ();
2143                         } else if (get_char () != '"') {
2144                                 return false;
2145                         }
2146
2147                         file.SetChecksum (guid_bytes, checksum_bytes.ToArray ());
2148                         ref_name.AutoGenerated = true;
2149                         return true;
2150                 }
2151
2152                 bool IsTokenIdentifierEqual (char[] identifier)
2153                 {
2154                         for (int i = 0; i < identifier.Length; ++i) {
2155                                 if (identifier[i] != id_builder[i])
2156                                         return false;
2157                         }
2158
2159                         return true;
2160                 }
2161
2162                 int TokenizePragmaNumber (ref int c)
2163                 {
2164                         number_pos = 0;
2165
2166                         int number;
2167
2168                         if (c >= '0' && c <= '9') {
2169                                 decimal_digits (c);
2170                                 uint ui = (uint) (number_builder[0] - '0');
2171
2172                                 try {
2173                                         for (int i = 1; i < number_pos; i++) {
2174                                                 ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2175                                         }
2176
2177                                         number = (int) ui;
2178                                 } catch (OverflowException) {
2179                                         Error_NumericConstantTooLong ();
2180                                         number = -1;
2181                                 }
2182
2183
2184                                 c = get_char ();
2185
2186                                 // skip over white space
2187                                 while (c == ' ' || c == '\t')
2188                                         c = get_char ();
2189
2190                                 if (c == ',') {
2191                                         c = get_char ();
2192                                 }
2193
2194                                 // skip over white space
2195                                 while (c == ' ' || c == '\t')
2196                                         c = get_char ();
2197                         } else {
2198                                 number = -1;
2199                                 if (c == '/') {
2200                                         ReadSingleLineComment ();
2201                                 } else {
2202                                         Report.Warning (1692, 1, Location, "Invalid number");
2203
2204                                         // Read everything till the end of the line or file
2205                                         do {
2206                                                 c = get_char ();
2207                                         } while (c != -1 && c != '\n');
2208                                 }
2209                         }
2210
2211                         return number;
2212                 }
2213
2214                 void ReadSingleLineComment ()
2215                 {
2216                         if (peek_char () != '/')
2217                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2218
2219                         // Read everything till the end of the line or file
2220                         int c;
2221                         do {
2222                                 c = get_char ();
2223                         } while (c != -1 && c != '\n');
2224                 }
2225
2226                 /// <summary>
2227                 /// Handles #pragma directive
2228                 /// </summary>
2229                 void ParsePragmaDirective (string arg)
2230                 {
2231                         int c;
2232                         int length = TokenizePreprocessorIdentifier (out c);
2233                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2234                                 length = TokenizePreprocessorIdentifier (out c);
2235
2236                                 //
2237                                 // #pragma warning disable
2238                                 // #pragma warning restore
2239                                 //
2240                                 if (length == pragma_warning_disable.Length) {
2241                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2242                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2243                                                 // skip over white space
2244                                                 while (c == ' ' || c == '\t')
2245                                                         c = get_char ();
2246
2247                                                 var loc = Location;
2248
2249                                                 if (c == '\n' || c == '/') {
2250                                                         if (c == '/')
2251                                                                 ReadSingleLineComment ();
2252
2253                                                         //
2254                                                         // Disable/Restore all warnings
2255                                                         //
2256                                                         if (disable) {
2257                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2258                                                         } else {
2259                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2260                                                         }
2261                                                 } else {
2262                                                         //
2263                                                         // Disable/Restore a warning or group of warnings
2264                                                         //
2265                                                         int code;
2266                                                         do {
2267                                                                 code = TokenizePragmaNumber (ref c);
2268                                                                 if (code > 0) {
2269                                                                         if (disable) {
2270                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, Report);
2271                                                                         } else {
2272                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, Report);
2273                                                                         }
2274                                                                 }
2275                                                         } while (code >= 0 && c != '\n' && c != -1);
2276                                                 }
2277
2278                                                 return;
2279                                         }
2280                                 }
2281
2282                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2283                                 return;
2284                         }
2285
2286                         //
2287                         // #pragma checksum
2288                         //
2289                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2290                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2291                                         Report.Warning (1695, 1, Location,
2292                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2293                                 }
2294
2295                                 return;
2296                         }
2297
2298                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2299                 }
2300
2301                 bool eval_val (string s)
2302                 {
2303                         if (s == "true")
2304                                 return true;
2305                         if (s == "false")
2306                                 return false;
2307
2308                         return file_name.IsConditionalDefined (context, s);
2309                 }
2310
2311                 bool pp_primary (ref string s)
2312                 {
2313                         s = s.Trim ();
2314                         int len = s.Length;
2315
2316                         if (len > 0){
2317                                 char c = s [0];
2318
2319                                 if (c == '('){
2320                                         s = s.Substring (1);
2321                                         bool val = pp_expr (ref s, false);
2322                                         if (s.Length > 0 && s [0] == ')'){
2323                                                 s = s.Substring (1);
2324                                                 return val;
2325                                         }
2326                                         Error_InvalidDirective ();
2327                                         return false;
2328                                 }
2329
2330                                 if (is_identifier_start_character (c)){
2331                                         int j = 1;
2332
2333                                         while (j < len){
2334                                                 c = s [j];
2335
2336                                                 if (is_identifier_part_character (c)){
2337                                                         j++;
2338                                                         continue;
2339                                                 }
2340                                                 bool v = eval_val (s.Substring (0, j));
2341                                                 s = s.Substring (j);
2342                                                 return v;
2343                                         }
2344                                         bool vv = eval_val (s);
2345                                         s = "";
2346                                         return vv;
2347                                 }
2348                         }
2349                         Error_InvalidDirective ();
2350                         return false;
2351                 }
2352
2353                 bool pp_unary (ref string s)
2354                 {
2355                         s = s.Trim ();
2356                         int len = s.Length;
2357
2358                         if (len > 0){
2359                                 if (s [0] == '!'){
2360                                         if (len > 1 && s [1] == '='){
2361                                                 Error_InvalidDirective ();
2362                                                 return false;
2363                                         }
2364                                         s = s.Substring (1);
2365                                         return ! pp_primary (ref s);
2366                                 } else
2367                                         return pp_primary (ref s);
2368                         } else {
2369                                 Error_InvalidDirective ();
2370                                 return false;
2371                         }
2372                 }
2373
2374                 bool pp_eq (ref string s)
2375                 {
2376                         bool va = pp_unary (ref s);
2377
2378                         s = s.Trim ();
2379                         int len = s.Length;
2380                         if (len > 0){
2381                                 if (s [0] == '='){
2382                                         if (len > 2 && s [1] == '='){
2383                                                 s = s.Substring (2);
2384                                                 return va == pp_unary (ref s);
2385                                         } else {
2386                                                 Error_InvalidDirective ();
2387                                                 return false;
2388                                         }
2389                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2390                                         s = s.Substring (2);
2391
2392                                         return va != pp_unary (ref s);
2393
2394                                 }
2395                         }
2396
2397                         return va;
2398
2399                 }
2400
2401                 bool pp_and (ref string s)
2402                 {
2403                         bool va = pp_eq (ref s);
2404
2405                         s = s.Trim ();
2406                         int len = s.Length;
2407                         if (len > 0){
2408                                 if (s [0] == '&'){
2409                                         if (len > 2 && s [1] == '&'){
2410                                                 s = s.Substring (2);
2411                                                 return (va & pp_and (ref s));
2412                                         } else {
2413                                                 Error_InvalidDirective ();
2414                                                 return false;
2415                                         }
2416                                 }
2417                         }
2418                         return va;
2419                 }
2420
2421                 //
2422                 // Evaluates an expression for `#if' or `#elif'
2423                 //
2424                 bool pp_expr (ref string s, bool isTerm)
2425                 {
2426                         bool va = pp_and (ref s);
2427                         s = s.Trim ();
2428                         int len = s.Length;
2429                         if (len > 0){
2430                                 char c = s [0];
2431
2432                                 if (c == '|'){
2433                                         if (len > 2 && s [1] == '|'){
2434                                                 s = s.Substring (2);
2435                                                 return va | pp_expr (ref s, isTerm);
2436                                         } else {
2437                                                 Error_InvalidDirective ();
2438                                                 return false;
2439                                         }
2440                                 }
2441                                 if (isTerm) {
2442                                         Error_EndLineExpected ();
2443                                         return false;
2444                                 }
2445                         }
2446
2447                         return va;
2448                 }
2449
2450                 bool eval (string s)
2451                 {
2452                         bool v = pp_expr (ref s, true);
2453                         s = s.Trim ();
2454                         if (s.Length != 0){
2455                                 return false;
2456                         }
2457
2458                         return v;
2459                 }
2460
2461                 void Error_NumericConstantTooLong ()
2462                 {
2463                         Report.Error (1021, Location, "Integral constant is too large");
2464                 }
2465
2466                 void Error_InvalidDirective ()
2467                 {
2468                         Report.Error (1517, Location, "Invalid preprocessor directive");
2469                 }
2470
2471                 void Error_UnexpectedDirective (string extra)
2472                 {
2473                         Report.Error (
2474                                 1028, Location,
2475                                 "Unexpected processor directive ({0})", extra);
2476                 }
2477
2478                 void Error_TokensSeen ()
2479                 {
2480                         Report.Error (1032, Location,
2481                                 "Cannot define or undefine preprocessor symbols after first token in file");
2482                 }
2483
2484                 void Eror_WrongPreprocessorLocation ()
2485                 {
2486                         Report.Error (1040, Location,
2487                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2488                 }
2489
2490                 void Error_EndLineExpected ()
2491                 {
2492                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2493                 }
2494
2495                 //
2496                 // Raises a warning when tokenizer found documentation comment
2497                 // on unexpected place
2498                 //
2499                 void WarningMisplacedComment (Location loc)
2500                 {
2501                         if (doc_state != XmlCommentState.Error) {
2502                                 doc_state = XmlCommentState.Error;
2503                                 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element");
2504                         }
2505                 }
2506
2507                 //
2508                 // if true, then the code continues processing the code
2509                 // if false, the code stays in a loop until another directive is
2510                 // reached.
2511                 // When caller_is_taking is false we ignore all directives except the ones
2512                 // which can help us to identify where the #if block ends
2513                 bool ParsePreprocessingDirective (bool caller_is_taking)
2514                 {
2515                         string arg;
2516                         bool region_directive = false;
2517
2518                         var directive = get_cmd_arg (out arg);
2519
2520                         //
2521                         // The first group of pre-processing instructions is always processed
2522                         //
2523                         switch (directive) {
2524                         case PreprocessorDirective.Region:
2525                                 region_directive = true;
2526                                 arg = "true";
2527                                 goto case PreprocessorDirective.If;
2528
2529                         case PreprocessorDirective.Endregion:
2530                                 if (ifstack == null || ifstack.Count == 0){
2531                                         Error_UnexpectedDirective ("no #region for this #endregion");
2532                                         return true;
2533                                 }
2534                                 int pop = ifstack.Pop ();
2535
2536                                 if ((pop & REGION) == 0)
2537                                         Report.Error (1027, Location, "Expected `#endif' directive");
2538
2539                                 return caller_is_taking;
2540
2541                         case PreprocessorDirective.If:
2542                                 if (ifstack == null)
2543                                         ifstack = new Stack<int> (2);
2544
2545                                 int flags = region_directive ? REGION : 0;
2546                                 if (ifstack.Count == 0){
2547                                         flags |= PARENT_TAKING;
2548                                 } else {
2549                                         int state = ifstack.Peek ();
2550                                         if ((state & TAKING) != 0) {
2551                                                 flags |= PARENT_TAKING;
2552                                         }
2553                                 }
2554
2555                                 if (eval (arg) && caller_is_taking) {
2556                                         ifstack.Push (flags | TAKING);
2557                                         return true;
2558                                 }
2559                                 ifstack.Push (flags);
2560                                 return false;
2561
2562                         case PreprocessorDirective.Endif:
2563                                 if (ifstack == null || ifstack.Count == 0){
2564                                         Error_UnexpectedDirective ("no #if for this #endif");
2565                                         return true;
2566                                 } else {
2567                                         pop = ifstack.Pop ();
2568
2569                                         if ((pop & REGION) != 0)
2570                                                 Report.Error (1038, Location, "#endregion directive expected");
2571
2572                                         if (arg.Length != 0) {
2573                                                 Error_EndLineExpected ();
2574                                         }
2575
2576                                         if (ifstack.Count == 0)
2577                                                 return true;
2578
2579                                         int state = ifstack.Peek ();
2580                                         return (state & TAKING) != 0;
2581                                 }
2582
2583                         case PreprocessorDirective.Elif:
2584                                 if (ifstack == null || ifstack.Count == 0){
2585                                         Error_UnexpectedDirective ("no #if for this #elif");
2586                                         return true;
2587                                 } else {
2588                                         int state = ifstack.Pop ();
2589
2590                                         if ((state & REGION) != 0) {
2591                                                 Report.Error (1038, Location, "#endregion directive expected");
2592                                                 return true;
2593                                         }
2594
2595                                         if ((state & ELSE_SEEN) != 0){
2596                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2597                                                 return true;
2598                                         }
2599
2600                                         if ((state & TAKING) != 0) {
2601                                                 ifstack.Push (0);
2602                                                 return false;
2603                                         }
2604
2605                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2606                                                 ifstack.Push (state | TAKING);
2607                                                 return true;
2608                                         }
2609
2610                                         ifstack.Push (state);
2611                                         return false;
2612                                 }
2613
2614                         case PreprocessorDirective.Else:
2615                                 if (ifstack == null || ifstack.Count == 0){
2616                                         Error_UnexpectedDirective ("no #if for this #else");
2617                                         return true;
2618                                 } else {
2619                                         int state = ifstack.Peek ();
2620
2621                                         if ((state & REGION) != 0) {
2622                                                 Report.Error (1038, Location, "#endregion directive expected");
2623                                                 return true;
2624                                         }
2625
2626                                         if ((state & ELSE_SEEN) != 0){
2627                                                 Error_UnexpectedDirective ("#else within #else");
2628                                                 return true;
2629                                         }
2630
2631                                         ifstack.Pop ();
2632
2633                                         if (arg.Length != 0) {
2634                                                 Error_EndLineExpected ();
2635                                                 return true;
2636                                         }
2637
2638                                         bool ret = false;
2639                                         if ((state & PARENT_TAKING) != 0) {
2640                                                 ret = (state & TAKING) == 0;
2641
2642                                                 if (ret)
2643                                                         state |= TAKING;
2644                                                 else
2645                                                         state &= ~TAKING;
2646                                         }
2647
2648                                         ifstack.Push (state | ELSE_SEEN);
2649
2650                                         return ret;
2651                                 }
2652                         case PreprocessorDirective.Define:
2653                                 if (any_token_seen){
2654                                         Error_TokensSeen ();
2655                                         return caller_is_taking;
2656                                 }
2657                                 PreProcessDefinition (true, arg, caller_is_taking);
2658                                 return caller_is_taking;
2659
2660                         case PreprocessorDirective.Undef:
2661                                 if (any_token_seen){
2662                                         Error_TokensSeen ();
2663                                         return caller_is_taking;
2664                                 }
2665                                 PreProcessDefinition (false, arg, caller_is_taking);
2666                                 return caller_is_taking;
2667
2668                         case PreprocessorDirective.Invalid:
2669                                 Report.Error (1024, Location, "Wrong preprocessor directive");
2670                                 return true;
2671                         }
2672
2673                         //
2674                         // These are only processed if we are in a `taking' block
2675                         //
2676                         if (!caller_is_taking)
2677                                 return false;
2678
2679                         switch (directive){
2680                         case PreprocessorDirective.Error:
2681                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2682                                 return true;
2683
2684                         case PreprocessorDirective.Warning:
2685                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2686                                 return true;
2687
2688                         case PreprocessorDirective.Pragma:
2689                                 if (context.Settings.Version == LanguageVersion.ISO_1) {
2690                                         Report.FeatureIsNotAvailable (context, Location, "#pragma");
2691                                 }
2692
2693                                 ParsePragmaDirective (arg);
2694                                 return true;
2695
2696                         case PreprocessorDirective.Line:
2697                                 if (!PreProcessLine (arg))
2698                                         Report.Error (
2699                                                 1576, Location,
2700                                                 "The line number specified for #line directive is missing or invalid");
2701                                 return caller_is_taking;
2702                         }
2703
2704                         throw new NotImplementedException (directive.ToString ());
2705                 }
2706
2707                 private int consume_string (bool quoted)
2708                 {
2709                         int c;
2710                         int pos = 0;
2711                         Location start_location = Location;
2712                         if (quoted)
2713                                 start_location = start_location - 1;
2714
2715 #if FULL_AST
2716                         int reader_pos = reader.Position;
2717 #endif
2718
2719                         while (true){
2720                                 c = get_char ();
2721                                 if (c == '"') {
2722                                         if (quoted && peek_char () == '"') {
2723                                                 if (pos == value_builder.Length)
2724                                                         Array.Resize (ref value_builder, pos * 2);
2725
2726                                                 value_builder[pos++] = (char) c;
2727                                                 get_char ();
2728                                                 continue;
2729                                         }
2730
2731                                         string s;
2732                                         if (pos == 0)
2733                                                 s = string.Empty;
2734                                         else if (pos <= 4)
2735                                                 s = InternIdentifier (value_builder, pos);
2736                                         else
2737                                                 s = new string (value_builder, 0, pos);
2738
2739                                         ILiteralConstant res = new StringLiteral (context.BuiltinTypes, s, start_location);
2740                                         val = res;
2741 #if FULL_AST
2742                                         res.ParsedValue = quoted ?
2743                                                 reader.ReadChars (reader_pos - 2, reader.Position - 1) :
2744                                                 reader.ReadChars (reader_pos - 1, reader.Position);
2745 #endif
2746
2747                                         return Token.LITERAL;
2748                                 }
2749
2750                                 if (c == '\n') {
2751                                         if (!quoted) {
2752                                                 Report.Error (1010, Location, "Newline in constant");
2753                                                 val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location);
2754                                                 return Token.LITERAL;
2755                                         }
2756                                 } else if (c == '\\' && !quoted) {
2757                                         int surrogate;
2758                                         c = escape (c, out surrogate);
2759                                         if (c == -1)
2760                                                 return Token.ERROR;
2761                                         if (surrogate != 0) {
2762                                                 if (pos == value_builder.Length)
2763                                                         Array.Resize (ref value_builder, pos * 2);
2764
2765                                                 value_builder[pos++] = (char) c;
2766                                                 c = surrogate;
2767                                         }
2768                                 } else if (c == -1) {
2769                                         Report.Error (1039, Location, "Unterminated string literal");
2770                                         return Token.EOF;
2771                                 }
2772
2773                                 if (pos == value_builder.Length)
2774                                         Array.Resize (ref value_builder, pos * 2);
2775
2776                                 value_builder[pos++] = (char) c;
2777                         }
2778                 }
2779
2780                 private int consume_identifier (int s)
2781                 {
2782                         int res = consume_identifier (s, false);
2783
2784                         if (doc_state == XmlCommentState.Allowed)
2785                                 doc_state = XmlCommentState.NotAllowed;
2786
2787                         return res;
2788                 }
2789
2790                 int consume_identifier (int c, bool quoted)
2791                 {
2792                         //
2793                         // This method is very performance sensitive. It accounts
2794                         // for approximately 25% of all parser time
2795                         //
2796
2797                         int pos = 0;
2798                         int column = col;
2799                         if (quoted)
2800                                 --column;
2801
2802                         if (c == '\\') {
2803                                 int surrogate;
2804                                 c = escape (c, out surrogate);
2805                                 if (surrogate != 0) {
2806                                         id_builder [pos++] = (char) c;
2807                                         c = surrogate;
2808                                 }
2809                         }
2810
2811                         id_builder [pos++] = (char) c;
2812
2813                         try {
2814                                 while (true) {
2815                                         c = reader.Read ();
2816
2817                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
2818                                                 id_builder [pos++] = (char) c;
2819                                                 continue;
2820                                         }
2821
2822                                         if (c < 0x80) {
2823                                                 if (c == '\\') {
2824                                                         int surrogate;
2825                                                         c = escape (c, out surrogate);
2826                                                         if (is_identifier_part_character ((char) c))
2827                                                                 id_builder[pos++] = (char) c;
2828
2829                                                         if (surrogate != 0) {
2830                                                                 c = surrogate;
2831                                                         }
2832
2833                                                         continue;
2834                                                 }
2835                                         } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
2836                                                 id_builder [pos++] = (char) c;
2837                                                 continue;
2838                                         }
2839
2840                                         putback_char = c;
2841                                         break;
2842                                 }
2843                         } catch (IndexOutOfRangeException) {
2844                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2845                                 --pos;
2846                                 col += pos;
2847                         }
2848
2849                         col += pos - 1;
2850
2851                         //
2852                         // Optimization: avoids doing the keyword lookup
2853                         // on uppercase letters
2854                         //
2855                         if (id_builder [0] >= '_' && !quoted) {
2856                                 int keyword = GetKeyword (id_builder, pos);
2857                                 if (keyword != -1) {
2858                                         val = LocatedToken.Create (null, ref_line, column);
2859                                         return keyword;
2860                                 }
2861                         }
2862
2863                         string s = InternIdentifier (id_builder, pos);
2864                         val = LocatedToken.Create (s, ref_line, column);
2865                         if (quoted && parsing_attribute_section)
2866                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2867
2868                         return Token.IDENTIFIER;
2869                 }
2870
2871                 static string InternIdentifier (char[] charBuffer, int length)
2872                 {
2873                         //
2874                         // Keep identifiers in an array of hashtables to avoid needless
2875                         // allocations
2876                         //
2877                         var identifiers_group = identifiers[length];
2878                         string s;
2879                         if (identifiers_group != null) {
2880                                 if (identifiers_group.TryGetValue (charBuffer, out s)) {
2881                                         return s;
2882                                 }
2883                         } else {
2884                                 // TODO: this should be number of files dependant
2885                                 // corlib compilation peaks at 1000 and System.Core at 150
2886                                 int capacity = length > 20 ? 10 : 100;
2887                                 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length));
2888                                 identifiers[length] = identifiers_group;
2889                         }
2890
2891                         char[] chars = new char[length];
2892                         Array.Copy (charBuffer, chars, length);
2893
2894                         s = new string (charBuffer, 0, length);
2895                         identifiers_group.Add (chars, s);
2896                         return s;
2897                 }
2898
2899                 public int xtoken ()
2900                 {
2901                         int d, c;
2902
2903                         // Whether we have seen comments on the current line
2904                         bool comments_seen = false;
2905                         while ((c = get_char ()) != -1) {
2906                                 switch (c) {
2907                                 case '\t':
2908                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
2909                                         continue;
2910
2911                                 case ' ':
2912                                 case '\f':
2913                                 case '\v':
2914                                 case 0xa0:
2915                                 case 0:
2916                                 case 0xFEFF:    // Ignore BOM anywhere in the file
2917                                         continue;
2918
2919 /*                              This is required for compatibility with .NET
2920                                 case 0xEF:
2921                                         if (peek_char () == 0xBB) {
2922                                                 PushPosition ();
2923                                                 get_char ();
2924                                                 if (get_char () == 0xBF)
2925                                                         continue;
2926                                                 PopPosition ();
2927                                         }
2928                                         break;
2929 */
2930                                 case '\\':
2931                                         tokens_seen = true;
2932                                         return consume_identifier (c);
2933
2934                                 case '{':
2935                                         val = LocatedToken.Create (ref_line, col);
2936                                         return Token.OPEN_BRACE;
2937                                 case '}':
2938                                         val = LocatedToken.Create (ref_line, col);
2939                                         return Token.CLOSE_BRACE;
2940                                 case '[':
2941                                         // To block doccomment inside attribute declaration.
2942                                         if (doc_state == XmlCommentState.Allowed)
2943                                                 doc_state = XmlCommentState.NotAllowed;
2944
2945                                         val = LocatedToken.Create (ref_line, col);
2946
2947                                         if (parsing_block == 0 || lambda_arguments_parsing)
2948                                                 return Token.OPEN_BRACKET;
2949
2950                                         int next = peek_char ();
2951                                         switch (next) {
2952                                         case ']':
2953                                         case ',':
2954                                                 return Token.OPEN_BRACKET;
2955
2956                                         case ' ':
2957                                         case '\f':
2958                                         case '\v':
2959                                         case '\r':
2960                                         case '\n':
2961                                         case '/':
2962                                                 next = peek_token ();
2963                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
2964                                                         return Token.OPEN_BRACKET;
2965
2966                                                 return Token.OPEN_BRACKET_EXPR;
2967                                         default:
2968                                                 return Token.OPEN_BRACKET_EXPR;
2969                                         }
2970                                 case ']':
2971                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2972                                         return Token.CLOSE_BRACKET;
2973                                 case '(':
2974                                         val = LocatedToken.Create (ref_line, col);
2975                                         //
2976                                         // An expression versions of parens can appear in block context only
2977                                         //
2978                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
2979
2980                                                 //
2981                                                 // Optmize most common case where we know that parens
2982                                                 // is not special
2983                                                 //
2984                                                 switch (current_token) {
2985                                                 case Token.IDENTIFIER:
2986                                                 case Token.IF:
2987                                                 case Token.FOR:
2988                                                 case Token.FOREACH:
2989                                                 case Token.TYPEOF:
2990                                                 case Token.WHILE:
2991                                                 case Token.USING:
2992                                                 case Token.DEFAULT:
2993                                                 case Token.DELEGATE:
2994                                                 case Token.OP_GENERICS_GT:
2995                                                         return Token.OPEN_PARENS;
2996                                                 }
2997
2998                                                 // Optimize using peek
2999                                                 int xx = peek_char ();
3000                                                 switch (xx) {
3001                                                 case '(':
3002                                                 case '\'':
3003                                                 case '"':
3004                                                 case '0':
3005                                                 case '1':
3006                                                         return Token.OPEN_PARENS;
3007                                                 }
3008
3009                                                 lambda_arguments_parsing = true;
3010                                                 PushPosition ();
3011                                                 d = TokenizeOpenParens ();
3012                                                 PopPosition ();
3013                                                 lambda_arguments_parsing = false;
3014                                                 return d;
3015                                         }
3016
3017                                         return Token.OPEN_PARENS;
3018                                 case ')':
3019                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3020                                         return Token.CLOSE_PARENS;
3021                                 case ',':
3022                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3023                                         return Token.COMMA;
3024                                 case ';':
3025                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3026                                         return Token.SEMICOLON;
3027                                 case '~':
3028                                         val = LocatedToken.Create (ref_line, col);
3029                                         return Token.TILDE;
3030                                 case '?':
3031                                         val = LocatedToken.Create (ref_line, col);
3032                                         return TokenizePossibleNullableType ();
3033                                 case '<':
3034                                         val = LocatedToken.Create (ref_line, col);
3035                                         if (parsing_generic_less_than++ > 0)
3036                                                 return Token.OP_GENERICS_LT;
3037
3038                                         return TokenizeLessThan ();
3039
3040                                 case '>':
3041                                         val = LocatedToken.Create (ref_line, col);
3042                                         d = peek_char ();
3043
3044                                         if (d == '='){
3045                                                 get_char ();
3046                                                 return Token.OP_GE;
3047                                         }
3048
3049                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
3050                                                 parsing_generic_less_than--;
3051                                                 return Token.OP_GENERICS_GT;
3052                                         }
3053
3054                                         if (d == '>') {
3055                                                 get_char ();
3056                                                 d = peek_char ();
3057
3058                                                 if (d == '=') {
3059                                                         get_char ();
3060                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
3061                                                 }
3062                                                 return Token.OP_SHIFT_RIGHT;
3063                                         }
3064
3065                                         return Token.OP_GT;
3066
3067                                 case '+':
3068                                         val = LocatedToken.Create (ref_line, col);
3069                                         d = peek_char ();
3070                                         if (d == '+') {
3071                                                 d = Token.OP_INC;
3072                                         } else if (d == '=') {
3073                                                 d = Token.OP_ADD_ASSIGN;
3074                                         } else {
3075                                                 return Token.PLUS;
3076                                         }
3077                                         get_char ();
3078                                         return d;
3079
3080                                 case '-':
3081                                         val = LocatedToken.Create (ref_line, col);
3082                                         d = peek_char ();
3083                                         if (d == '-') {
3084                                                 d = Token.OP_DEC;
3085                                         } else if (d == '=')
3086                                                 d = Token.OP_SUB_ASSIGN;
3087                                         else if (d == '>')
3088                                                 d = Token.OP_PTR;
3089                                         else {
3090                                                 return Token.MINUS;
3091                                         }
3092                                         get_char ();
3093                                         return d;
3094
3095                                 case '!':
3096                                         val = LocatedToken.Create (ref_line, col);
3097                                         if (peek_char () == '='){
3098                                                 get_char ();
3099                                                 return Token.OP_NE;
3100                                         }
3101                                         return Token.BANG;
3102
3103                                 case '=':
3104                                         val = LocatedToken.Create (ref_line, col);
3105                                         d = peek_char ();
3106                                         if (d == '='){
3107                                                 get_char ();
3108                                                 return Token.OP_EQ;
3109                                         }
3110                                         if (d == '>'){
3111                                                 get_char ();
3112                                                 return Token.ARROW;
3113                                         }
3114
3115                                         return Token.ASSIGN;
3116
3117                                 case '&':
3118                                         val = LocatedToken.Create (ref_line, col);
3119                                         d = peek_char ();
3120                                         if (d == '&'){
3121                                                 get_char ();
3122                                                 return Token.OP_AND;
3123                                         }
3124                                         if (d == '='){
3125                                                 get_char ();
3126                                                 return Token.OP_AND_ASSIGN;
3127                                         }
3128                                         return Token.BITWISE_AND;
3129
3130                                 case '|':
3131                                         val = LocatedToken.Create (ref_line, col);
3132                                         d = peek_char ();
3133                                         if (d == '|'){
3134                                                 get_char ();
3135                                                 return Token.OP_OR;
3136                                         }
3137                                         if (d == '='){
3138                                                 get_char ();
3139                                                 return Token.OP_OR_ASSIGN;
3140                                         }
3141                                         return Token.BITWISE_OR;
3142
3143                                 case '*':
3144                                         val = LocatedToken.Create (ref_line, col);
3145                                         if (peek_char () == '='){
3146                                                 get_char ();
3147                                                 return Token.OP_MULT_ASSIGN;
3148                                         }
3149                                         return Token.STAR;
3150
3151                                 case '/':
3152                                         d = peek_char ();
3153                                         if (d == '='){
3154                                                 val = LocatedToken.Create (ref_line, col);
3155                                                 get_char ();
3156                                                 return Token.OP_DIV_ASSIGN;
3157                                         }
3158
3159                                         // Handle double-slash comments.
3160                                         if (d == '/'){
3161                                                 get_char ();
3162                                                 if (doc_processing) {
3163                                                         if (peek_char () == '/') {
3164                                                                 get_char ();
3165                                                                 // Don't allow ////.
3166                                                                 if ((d = peek_char ()) != '/') {
3167                                                                         if (doc_state == XmlCommentState.Allowed)
3168                                                                                 handle_one_line_xml_comment ();
3169                                                                         else if (doc_state == XmlCommentState.NotAllowed)
3170                                                                                 WarningMisplacedComment (Location - 3);
3171                                                                 }
3172                                                         } else {
3173                                                                 if (xml_comment_buffer.Length > 0)
3174                                                                         doc_state = XmlCommentState.NotAllowed;
3175                                                         }
3176                                                 }
3177
3178                                                 while ((d = get_char ()) != -1 && d != '\n');
3179
3180                                                 any_token_seen |= tokens_seen;
3181                                                 tokens_seen = false;
3182                                                 comments_seen = false;
3183                                                 continue;
3184                                         } else if (d == '*'){
3185                                                 get_char ();
3186                                                 bool docAppend = false;
3187                                                 if (doc_processing && peek_char () == '*') {
3188                                                         get_char ();
3189                                                         // But when it is /**/, just do nothing.
3190                                                         if (peek_char () == '/') {
3191                                                                 get_char ();
3192                                                                 continue;
3193                                                         }
3194                                                         if (doc_state == XmlCommentState.Allowed)
3195                                                                 docAppend = true;
3196                                                         else if (doc_state == XmlCommentState.NotAllowed) {
3197                                                                 WarningMisplacedComment (Location - 2);
3198                                                         }
3199                                                 }
3200
3201                                                 int current_comment_start = 0;
3202                                                 if (docAppend) {
3203                                                         current_comment_start = xml_comment_buffer.Length;
3204                                                         xml_comment_buffer.Append (Environment.NewLine);
3205                                                 }
3206
3207                                                 while ((d = get_char ()) != -1){
3208                                                         if (d == '*' && peek_char () == '/'){
3209                                                                 get_char ();
3210                                                                 comments_seen = true;
3211                                                                 break;
3212                                                         }
3213                                                         if (docAppend)
3214                                                                 xml_comment_buffer.Append ((char) d);
3215
3216                                                         if (d == '\n'){
3217                                                                 any_token_seen |= tokens_seen;
3218                                                                 tokens_seen = false;
3219                                                                 //
3220                                                                 // Reset 'comments_seen' just to be consistent.
3221                                                                 // It doesn't matter either way, here.
3222                                                                 //
3223                                                                 comments_seen = false;
3224                                                         }
3225                                                 }
3226                                                 if (!comments_seen)
3227                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3228
3229                                                 if (docAppend)
3230                                                         update_formatted_doc_comment (current_comment_start);
3231                                                 continue;
3232                                         }
3233                                         val = LocatedToken.Create (ref_line, col);
3234                                         return Token.DIV;
3235
3236                                 case '%':
3237                                         val = LocatedToken.Create (ref_line, col);
3238                                         if (peek_char () == '='){
3239                                                 get_char ();
3240                                                 return Token.OP_MOD_ASSIGN;
3241                                         }
3242                                         return Token.PERCENT;
3243
3244                                 case '^':
3245                                         val = LocatedToken.Create (ref_line, col);
3246                                         if (peek_char () == '='){
3247                                                 get_char ();
3248                                                 return Token.OP_XOR_ASSIGN;
3249                                         }
3250                                         return Token.CARRET;
3251
3252                                 case ':':
3253                                         val = LocatedToken.Create (ref_line, col);
3254                                         if (peek_char () == ':') {
3255                                                 get_char ();
3256                                                 return Token.DOUBLE_COLON;
3257                                         }
3258                                         return Token.COLON;
3259
3260                                 case '0': case '1': case '2': case '3': case '4':
3261                                 case '5': case '6': case '7': case '8': case '9':
3262                                         tokens_seen = true;
3263                                         return is_number (c);
3264
3265                                 case '\n': // white space
3266                                         any_token_seen |= tokens_seen;
3267                                         tokens_seen = false;
3268                                         comments_seen = false;
3269                                         continue;
3270
3271                                 case '.':
3272                                         tokens_seen = true;
3273                                         d = peek_char ();
3274                                         if (d >= '0' && d <= '9')
3275                                                 return is_number (c);
3276
3277                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3278                                         return Token.DOT;
3279
3280                                 case '#':
3281                                         if (tokens_seen || comments_seen) {
3282                                                 Eror_WrongPreprocessorLocation ();
3283                                                 return Token.ERROR;
3284                                         }
3285
3286                                         if (ParsePreprocessingDirective (true))
3287                                                 continue;
3288
3289                                         bool directive_expected = false;
3290                                         while ((c = get_char ()) != -1) {
3291                                                 if (col == 1) {
3292                                                         directive_expected = true;
3293                                                 } else if (!directive_expected) {
3294                                                         // TODO: Implement comment support for disabled code and uncomment this code
3295 //                                                      if (c == '#') {
3296 //                                                              Eror_WrongPreprocessorLocation ();
3297 //                                                              return Token.ERROR;
3298 //                                                      }
3299                                                         continue;
3300                                                 }
3301
3302                                                 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' )
3303                                                         continue;
3304
3305                                                 if (c == '#') {
3306                                                         if (ParsePreprocessingDirective (false))
3307                                                                 break;
3308                                                 }
3309                                                 directive_expected = false;
3310                                         }
3311
3312                                         if (c != -1) {
3313                                                 tokens_seen = false;
3314                                                 continue;
3315                                         }
3316
3317                                         return Token.EOF;
3318
3319                                 case '"':
3320                                         return consume_string (false);
3321
3322                                 case '\'':
3323                                         return TokenizeBackslash ();
3324
3325                                 case '@':
3326                                         c = get_char ();
3327                                         if (c == '"') {
3328                                                 tokens_seen = true;
3329                                                 return consume_string (true);
3330                                         }
3331
3332                                         if (is_identifier_start_character (c)){
3333                                                 return consume_identifier (c, true);
3334                                         }
3335
3336                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3337                                         return Token.ERROR;
3338
3339                                 case EvalStatementParserCharacter:
3340                                         return Token.EVAL_STATEMENT_PARSER;
3341                                 case EvalCompilationUnitParserCharacter:
3342                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3343                                 case EvalUsingDeclarationsParserCharacter:
3344                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3345                                 case DocumentationXref:
3346                                         return Token.DOC_SEE;
3347                                 }
3348
3349                                 if (is_identifier_start_character (c)) {
3350                                         tokens_seen = true;
3351                                         return consume_identifier (c);
3352                                 }
3353
3354                                 if (char.IsWhiteSpace ((char) c))
3355                                         continue;
3356
3357                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3358                         }
3359
3360                         if (CompleteOnEOF){
3361                                 if (generated)
3362                                         return Token.COMPLETE_COMPLETION;
3363
3364                                 generated = true;
3365                                 return Token.GENERATE_COMPLETION;
3366                         }
3367
3368
3369                         return Token.EOF;
3370                 }
3371
3372                 int TokenizeBackslash ()
3373                 {
3374                         int c = get_char ();
3375                         tokens_seen = true;
3376                         if (c == '\'') {
3377                                 val = new CharLiteral (context.BuiltinTypes, (char) c, Location);
3378                                 Report.Error (1011, Location, "Empty character literal");
3379                                 return Token.LITERAL;
3380                         }
3381
3382                         if (c == '\n') {
3383                                 Report.Error (1010, Location, "Newline in constant");
3384                                 return Token.ERROR;
3385                         }
3386
3387                         int d;
3388                         c = escape (c, out d);
3389                         if (c == -1)
3390                                 return Token.ERROR;
3391                         if (d != 0)
3392                                 throw new NotImplementedException ();
3393
3394                         val = new CharLiteral (context.BuiltinTypes, (char) c, Location);
3395                         c = get_char ();
3396
3397                         if (c != '\'') {
3398                                 Report.Error (1012, Location, "Too many characters in character literal");
3399
3400                                 // Try to recover, read until newline or next "'"
3401                                 while ((c = get_char ()) != -1) {
3402                                         if (c == '\n' || c == '\'')
3403                                                 break;
3404                                 }
3405                         }
3406
3407                         return Token.LITERAL;
3408                 }
3409
3410                 int TokenizeLessThan ()
3411                 {
3412                         int d;
3413                         if (handle_typeof) {
3414                                 PushPosition ();
3415                                 if (parse_generic_dimension (out d)) {
3416                                         val = d;
3417                                         DiscardPosition ();
3418                                         return Token.GENERIC_DIMENSION;
3419                                 }
3420                                 PopPosition ();
3421                         }
3422
3423                         // Save current position and parse next token.
3424                         PushPosition ();
3425                         if (parse_less_than ()) {
3426                                 if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) {
3427                                         d = Token.OP_GENERICS_LT_DECL;
3428                                 } else {
3429                                         d = Token.OP_GENERICS_LT;
3430                                 }
3431                                 PopPosition ();
3432                                 return d;
3433                         }
3434
3435                         PopPosition ();
3436                         parsing_generic_less_than = 0;
3437
3438                         d = peek_char ();
3439                         if (d == '<') {
3440                                 get_char ();
3441                                 d = peek_char ();
3442
3443                                 if (d == '=') {
3444                                         get_char ();
3445                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3446                                 }
3447                                 return Token.OP_SHIFT_LEFT;
3448                         }
3449
3450                         if (d == '=') {
3451                                 get_char ();
3452                                 return Token.OP_LE;
3453                         }
3454                         return Token.OP_LT;
3455                 }
3456
3457                 //
3458                 // Handles one line xml comment
3459                 //
3460                 private void handle_one_line_xml_comment ()
3461                 {
3462                         int c;
3463                         while ((c = peek_char ()) == ' ')
3464                                 get_char (); // skip heading whitespaces.
3465                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3466                                 xml_comment_buffer.Append ((char) get_char ());
3467                         }
3468                         if (c == '\r' || c == '\n')
3469                                 xml_comment_buffer.Append (Environment.NewLine);
3470                 }
3471
3472                 //
3473                 // Remove heading "*" in Javadoc-like xml documentation.
3474                 //
3475                 private void update_formatted_doc_comment (int current_comment_start)
3476                 {
3477                         int length = xml_comment_buffer.Length - current_comment_start;
3478                         string [] lines = xml_comment_buffer.ToString (
3479                                 current_comment_start,
3480                                 length).Replace ("\r", "").Split ('\n');
3481
3482                         // The first line starts with /**, thus it is not target
3483                         // for the format check.
3484                         for (int i = 1; i < lines.Length; i++) {
3485                                 string s = lines [i];
3486                                 int idx = s.IndexOf ('*');
3487                                 string head = null;
3488                                 if (idx < 0) {
3489                                         if (i < lines.Length - 1)
3490                                                 return;
3491                                         head = s;
3492                                 } else
3493                                         head = s.Substring (0, idx);
3494                                 foreach (char c in head)
3495                                         if (c != ' ')
3496                                                 return;
3497                                 lines [i] = s.Substring (idx + 1);
3498                         }
3499                         xml_comment_buffer.Remove (current_comment_start, length);
3500                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3501                 }
3502
3503                 //
3504                 // Checks if there was incorrect doc comments and raise
3505                 // warnings.
3506                 //
3507                 public void check_incorrect_doc_comment ()
3508                 {
3509                         if (xml_comment_buffer.Length > 0)
3510                                 WarningMisplacedComment (Location);
3511                 }
3512
3513                 //
3514                 // Consumes the saved xml comment lines (if any)
3515                 // as for current target member or type.
3516                 //
3517                 public string consume_doc_comment ()
3518                 {
3519                         if (xml_comment_buffer.Length > 0) {
3520                                 string ret = xml_comment_buffer.ToString ();
3521                                 reset_doc_comment ();
3522                                 return ret;
3523                         }
3524                         return null;
3525                 }
3526
3527                 Report Report {
3528                         get { return context.Report; }
3529                 }
3530
3531                 void reset_doc_comment ()
3532                 {
3533                         xml_comment_buffer.Length = 0;
3534                 }
3535
3536                 public void cleanup ()
3537                 {
3538                         if (ifstack != null && ifstack.Count >= 1) {
3539                                 int state = ifstack.Pop ();
3540                                 if ((state & REGION) != 0)
3541                                         Report.Error (1038, Location, "#endregion directive expected");
3542                                 else
3543                                         Report.Error (1027, Location, "Expected `#endif' directive");
3544                         }
3545                 }
3546         }
3547
3548         //
3549         // Indicates whether it accepts XML documentation or not.
3550         //
3551         public enum XmlCommentState {
3552                 // comment is allowed in this state.
3553                 Allowed,
3554                 // comment is not allowed in this state.
3555                 NotAllowed,
3556                 // once comments appeared when it is NotAllowed, then the
3557                 // state is changed to it, until the state is changed to
3558                 // .Allowed.
3559                 Error
3560         }
3561 }
3562