mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@gmail.com)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 // Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20
  21 namespace Mono.CSharp
  22 {
  23         /// <summary>
  24         ///    Tokenizer for C# source code.
  25         /// </summary>
  26
  27         public class Tokenizer : yyParser.yyInput
  28         {
  29                 class KeywordEntry<T>
  30                 {
  31                         public readonly T Token;
  32                         public KeywordEntry<T> Next;
  33                         public readonly char[] Value;
  34
  35                         public KeywordEntry (string value, T token)
  36                         {
  37                                 this.Value = value.ToCharArray ();
  38                                 this.Token = token;
  39                         }
  40                 }
  41
  42                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  43                 {
  44                         readonly int length;
  45
  46                         public IdentifiersComparer (int length)
  47                         {
  48                                 this.length = length;
  49                         }
  50
  51                         public bool Equals (char[] x, char[] y)
  52                         {
  53                                 for (int i = 0; i < length; ++i)
  54                                         if (x [i] != y [i])
  55                                                 return false;
  56
  57                                 return true;
  58                         }
  59
  60                         public int GetHashCode (char[] obj)
  61                         {
  62                                 int h = 0;
  63                                 for (int i = 0; i < length; ++i)
  64                                         h = (h << 5) - h + obj [i];
  65
  66                                 return h;
  67                         }
  68                 }
  69
  70                 //
  71                 // This class has to be used in the parser only, it reuses token
  72                 // details after each parse
  73                 //
  74                 public class LocatedToken
  75                 {
  76                         int row, column;
  77                         string value;
  78
  79                         static LocatedToken[] buffer;
  80                         static int pos;
  81
  82                         private LocatedToken ()
  83                         {
  84                         }
  85
  86                         public static LocatedToken Create (int row, int column)
  87                         {
  88                                 return Create (null, row, column);
  89                         }
  90
  91                         public static LocatedToken Create (string value, Location loc)
  92                         {
  93                                 return Create (value, loc.Row, loc.Column);
  94                         }
  95
  96                         public static LocatedToken Create (string value, int row, int column)
  97                         {
  98                                 //
  99                                 // TODO: I am not very happy about the logic but it's the best
 100                                 // what I could come up with for now.
 101                                 // Ideally we should be using just tiny buffer (256 elements) which
 102                                 // is enough to hold all details for currect stack and recycle elements
 103                                 // poped from the stack but there is a trick needed to recycle
 104                                 // them properly.
 105                                 //
 106                                 LocatedToken entry;
 107                                 if (pos >= buffer.Length) {
 108                                         entry = new LocatedToken ();
 109                                 } else {
 110                                         entry = buffer [pos];
 111                                         if (entry == null) {
 112                                                 entry = new LocatedToken ();
 113                                                 buffer [pos] = entry;
 114                                         }
 115
 116                                         ++pos;
 117                                 }
 118                                 entry.value = value;
 119                                 entry.row = row;
 120                                 entry.column = column;
 121                                 return entry;
 122                         }
 123
 124                         //
 125                         // Used for token not required by expression evaluator
 126                         //
 127                         [Conditional ("FULL_AST")]
 128                         public static void CreateOptional (int row, int col, ref object token)
 129                         {
 130                                 token = Create (row, col);
 131                         }
 132
 133                         public static void Initialize ()
 134                         {
 135                                 if (buffer == null)
 136                                         buffer = new LocatedToken [10000];
 137                                 pos = 0;
 138                         }
 139
 140                         public Location Location {
 141                                 get { return new Location (row, column); }
 142                         }
 143
 144                         public string Value {
 145                                 get { return value; }
 146                         }
 147                 }
 148
 149                 public enum PreprocessorDirective
 150                 {
 151                         Invalid = 0,
 152
 153                         Region = 1,
 154                         Endregion = 2,
 155                         If = 3 | RequiresArgument,
 156                         Endif = 4,
 157                         Elif = 5 | RequiresArgument,
 158                         Else = 6,
 159                         Define = 7 | RequiresArgument,
 160                         Undef = 8 | RequiresArgument,
 161                         Error = 9,
 162                         Warning = 10,
 163                         Pragma = 11 | CustomArgumentsParsing,
 164                         Line = 12,
 165
 166                         CustomArgumentsParsing = 1 << 10,
 167                         RequiresArgument = 1 << 11
 168                 }
 169
 170                 SeekableStreamReader reader;
 171                 SourceFile ref_name;
 172                 CompilationSourceFile file_name;
 173                 CompilerContext context;
 174                 bool hidden = false;
 175                 int ref_line = 1;
 176                 int line = 1;
 177                 int col = 0;
 178                 int previous_col;
 179                 int current_token;
 180                 int tab_size;
 181                 bool handle_get_set = false;
 182                 bool handle_remove_add = false;
 183                 bool handle_where = false;
 184                 bool handle_typeof = false;
 185                 bool lambda_arguments_parsing;
 186                 List<Location> escaped_identifiers;
 187                 int parsing_generic_less_than;
 188                 readonly bool doc_processing;
 189
 190                 //
 191                 // Used mainly for parser optimizations. Some expressions for instance
 192                 // can appear only in block (including initializer, base initializer)
 193                 // scope only
 194                 //
 195                 public int parsing_block;
 196                 internal bool query_parsing;
 197
 198                 //
 199                 // When parsing type only, useful for ambiguous nullable types
 200                 //
 201                 public int parsing_type;
 202
 203                 //
 204                 // Set when parsing generic declaration (type or method header)
 205                 //
 206                 public bool parsing_generic_declaration;
 207                 public bool parsing_generic_declaration_doc;
 208
 209                 //
 210                 // The value indicates that we have not reach any declaration or
 211                 // namespace yet
 212                 //
 213                 public int parsing_declaration;
 214
 215                 public bool parsing_attribute_section;
 216
 217                 public bool parsing_modifiers;
 218
 219                 //
 220                 // The special characters to inject on streams to run the unit parser
 221                 // in the special expression mode. Using private characters from
 222                 // Plane Sixteen (U+100000 to U+10FFFD)
 223                 //
 224                 // This character is only tested just before the tokenizer is about to report
 225                 // an error;   So on the regular operation mode, this addition will have no
 226                 // impact on the tokenizer's performance.
 227                 //
 228
 229                 public const int EvalStatementParserCharacter = 0x100000;
 230                 public const int EvalCompilationUnitParserCharacter = 0x100001;
 231                 public const int EvalUsingDeclarationsParserCharacter = 0x100002;
 232                 public const int DocumentationXref = 0x100003;
 233
 234                 //
 235                 // XML documentation buffer. The save point is used to divide
 236                 // comments on types and comments on members.
 237                 //
 238                 StringBuilder xml_comment_buffer;
 239
 240                 //
 241                 // See comment on XmlCommentState enumeration.
 242                 //
 243                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 244
 245                 //
 246                 // Whether tokens have been seen on this line
 247                 //
 248                 bool tokens_seen = false;
 249
 250                 //
 251                 // Set to true once the GENERATE_COMPLETION token has bee
 252                 // returned.   This helps produce one GENERATE_COMPLETION,
 253                 // as many COMPLETE_COMPLETION as necessary to complete the
 254                 // AST tree and one final EOF.
 255                 //
 256                 bool generated;
 257
 258                 //
 259                 // Whether a token has been seen on the file
 260                 // This is needed because `define' is not allowed to be used
 261                 // after a token has been seen.
 262                 //
 263                 bool any_token_seen;
 264
 265                 //
 266                 // Class variables
 267                 //
 268                 static readonly KeywordEntry<int>[][] keywords;
 269                 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 270                 static readonly Dictionary<string, object> keyword_strings;             // TODO: HashSet
 271                 static readonly NumberStyles styles;
 272                 static readonly NumberFormatInfo csharp_format_info;
 273
 274                 // Pragma arguments
 275                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 276                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 277                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 278                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 279
 280                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 281
 282                 public bool PropertyParsing {
 283                         get { return handle_get_set; }
 284                         set { handle_get_set = value; }
 285                 }
 286
 287                 public bool EventParsing {
 288                         get { return handle_remove_add; }
 289                         set { handle_remove_add = value; }
 290                 }
 291
 292                 public bool ConstraintsParsing {
 293                         get { return handle_where; }
 294                         set { handle_where = value; }
 295                 }
 296
 297                 public bool TypeOfParsing {
 298                         get { return handle_typeof; }
 299                         set { handle_typeof = value; }
 300                 }
 301
 302                 public int TabSize {
 303                         get { return tab_size; }
 304                         set { tab_size = value; }
 305                 }
 306
 307                 public XmlCommentState doc_state {
 308                         get { return xml_doc_state; }
 309                         set {
 310                                 if (value == XmlCommentState.Allowed) {
 311                                         check_incorrect_doc_comment ();
 312                                         reset_doc_comment ();
 313                                 }
 314                                 xml_doc_state = value;
 315                         }
 316                 }
 317
 318                 //
 319                 // This is used to trigger completion generation on the parser
 320                 public bool CompleteOnEOF;
 321
 322                 void AddEscapedIdentifier (Location loc)
 323                 {
 324                         if (escaped_identifiers == null)
 325                                 escaped_identifiers = new List<Location> ();
 326
 327                         escaped_identifiers.Add (loc);
 328                 }
 329
 330                 public bool IsEscapedIdentifier (ATypeNameExpression name)
 331                 {
 332                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 333                 }
 334
 335                 //
 336                 // Values for the associated token returned
 337                 //
 338                 internal int putback_char;      // Used by repl only
 339                 object val;
 340
 341                 //
 342                 // Pre-processor
 343                 //
 344                 const int TAKING        = 1;
 345                 const int ELSE_SEEN     = 4;
 346                 const int PARENT_TAKING = 8;
 347                 const int REGION        = 16;
 348
 349                 //
 350                 // pre-processor if stack state:
 351                 //
 352                 Stack<int> ifstack;
 353
 354                 static System.Text.StringBuilder string_builder;
 355
 356                 const int max_id_size = 512;
 357                 static readonly char [] id_builder = new char [max_id_size];
 358
 359                 public static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 360
 361                 const int max_number_size = 512;
 362                 static char [] number_builder = new char [max_number_size];
 363                 static int number_pos;
 364
 365                 static char[] value_builder = new char[256];
 366
 367                 public int Line {
 368                         get {
 369                                 return ref_line;
 370                         }
 371                 }
 372
 373                 //
 374                 // This is used when the tokenizer needs to save
 375                 // the current position as it needs to do some parsing
 376                 // on its own to deamiguate a token in behalf of the
 377                 // parser.
 378                 //
 379                 Stack<Position> position_stack = new Stack<Position> (2);
 380
 381                 class Position {
 382                         public int position;
 383                         public int line;
 384                         public int ref_line;
 385                         public int col;
 386                         public bool hidden;
 387                         public int putback_char;
 388                         public int previous_col;
 389                         public Stack<int> ifstack;
 390                         public int parsing_generic_less_than;
 391                         public int current_token;
 392                         public object val;
 393
 394                         public Position (Tokenizer t)
 395                         {
 396                                 position = t.reader.Position;
 397                                 line = t.line;
 398                                 ref_line = t.ref_line;
 399                                 col = t.col;
 400                                 hidden = t.hidden;
 401                                 putback_char = t.putback_char;
 402                                 previous_col = t.previous_col;
 403                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 404                                         // There is no simple way to clone Stack<T> all
 405                                         // methods reverse the order
 406                                         var clone = t.ifstack.ToArray ();
 407                                         Array.Reverse (clone);
 408                                         ifstack = new Stack<int> (clone);
 409                                 }
 410                                 parsing_generic_less_than = t.parsing_generic_less_than;
 411                                 current_token = t.current_token;
 412                                 val = t.val;
 413                         }
 414                 }
 415
 416                 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, CompilerContext ctx)
 417                 {
 418                         this.ref_name = file;
 419                         this.file_name = file;
 420                         this.context = ctx;
 421                         reader = input;
 422
 423                         putback_char = -1;
 424
 425                         xml_comment_buffer = new StringBuilder ();
 426                         doc_processing = ctx.Settings.DocumentationFile != null;
 427
 428                         if (Environment.OSVersion.Platform == PlatformID.Win32NT)
 429                                 tab_size = 4;
 430                         else
 431                                 tab_size = 8;
 432
 433                         Mono.CSharp.Location.Push (file, file);
 434                 }
 435
 436                 public void PushPosition ()
 437                 {
 438                         position_stack.Push (new Position (this));
 439                 }
 440
 441                 public void PopPosition ()
 442                 {
 443                         Position p = position_stack.Pop ();
 444
 445                         reader.Position = p.position;
 446                         ref_line = p.ref_line;
 447                         line = p.line;
 448                         col = p.col;
 449                         hidden = p.hidden;
 450                         putback_char = p.putback_char;
 451                         previous_col = p.previous_col;
 452                         ifstack = p.ifstack;
 453                         parsing_generic_less_than = p.parsing_generic_less_than;
 454                         current_token = p.current_token;
 455                         val = p.val;
 456                 }
 457
 458                 // Do not reset the position, ignore it.
 459                 public void DiscardPosition ()
 460                 {
 461                         position_stack.Pop ();
 462                 }
 463
 464                 static void AddKeyword (string kw, int token)
 465                 {
 466                         keyword_strings.Add (kw, null);
 467
 468                         AddKeyword (keywords, kw, token);
 469                 }
 470
 471                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 472                 {
 473                         AddKeyword (keywords_preprocessor, kw, directive);
 474                 }
 475
 476                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 477                 {
 478                         int length = kw.Length;
 479                         if (keywords[length] == null) {
 480                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 481                         }
 482
 483                         int char_index = kw[0] - '_';
 484                         var kwe = keywords[length][char_index];
 485                         if (kwe == null) {
 486                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 487                                 return;
 488                         }
 489
 490                         while (kwe.Next != null) {
 491                                 kwe = kwe.Next;
 492                         }
 493
 494                         kwe.Next = new KeywordEntry<T> (kw, token);
 495                 }
 496
 497                 //
 498                 // Class initializer
 499                 //
 500                 static Tokenizer ()
 501                 {
 502                         keyword_strings = new Dictionary<string, object> ();
 503
 504                         // 11 is the length of the longest keyword for now
 505                         keywords = new KeywordEntry<int>[11][];
 506
 507                         AddKeyword ("__arglist", Token.ARGLIST);
 508                         AddKeyword ("__makeref", Token.MAKEREF);
 509                         AddKeyword ("__reftype", Token.REFTYPE);
 510                         AddKeyword ("__refvalue", Token.REFVALUE);
 511                         AddKeyword ("abstract", Token.ABSTRACT);
 512                         AddKeyword ("as", Token.AS);
 513                         AddKeyword ("add", Token.ADD);
 514                         AddKeyword ("base", Token.BASE);
 515                         AddKeyword ("bool", Token.BOOL);
 516                         AddKeyword ("break", Token.BREAK);
 517                         AddKeyword ("byte", Token.BYTE);
 518                         AddKeyword ("case", Token.CASE);
 519                         AddKeyword ("catch", Token.CATCH);
 520                         AddKeyword ("char", Token.CHAR);
 521                         AddKeyword ("checked", Token.CHECKED);
 522                         AddKeyword ("class", Token.CLASS);
 523                         AddKeyword ("const", Token.CONST);
 524                         AddKeyword ("continue", Token.CONTINUE);
 525                         AddKeyword ("decimal", Token.DECIMAL);
 526                         AddKeyword ("default", Token.DEFAULT);
 527                         AddKeyword ("delegate", Token.DELEGATE);
 528                         AddKeyword ("do", Token.DO);
 529                         AddKeyword ("double", Token.DOUBLE);
 530                         AddKeyword ("else", Token.ELSE);
 531                         AddKeyword ("enum", Token.ENUM);
 532                         AddKeyword ("event", Token.EVENT);
 533                         AddKeyword ("explicit", Token.EXPLICIT);
 534                         AddKeyword ("extern", Token.EXTERN);
 535                         AddKeyword ("false", Token.FALSE);
 536                         AddKeyword ("finally", Token.FINALLY);
 537                         AddKeyword ("fixed", Token.FIXED);
 538                         AddKeyword ("float", Token.FLOAT);
 539                         AddKeyword ("for", Token.FOR);
 540                         AddKeyword ("foreach", Token.FOREACH);
 541                         AddKeyword ("goto", Token.GOTO);
 542                         AddKeyword ("get", Token.GET);
 543                         AddKeyword ("if", Token.IF);
 544                         AddKeyword ("implicit", Token.IMPLICIT);
 545                         AddKeyword ("in", Token.IN);
 546                         AddKeyword ("int", Token.INT);
 547                         AddKeyword ("interface", Token.INTERFACE);
 548                         AddKeyword ("internal", Token.INTERNAL);
 549                         AddKeyword ("is", Token.IS);
 550                         AddKeyword ("lock", Token.LOCK);
 551                         AddKeyword ("long", Token.LONG);
 552                         AddKeyword ("namespace", Token.NAMESPACE);
 553                         AddKeyword ("new", Token.NEW);
 554                         AddKeyword ("null", Token.NULL);
 555                         AddKeyword ("object", Token.OBJECT);
 556                         AddKeyword ("operator", Token.OPERATOR);
 557                         AddKeyword ("out", Token.OUT);
 558                         AddKeyword ("override", Token.OVERRIDE);
 559                         AddKeyword ("params", Token.PARAMS);
 560                         AddKeyword ("private", Token.PRIVATE);
 561                         AddKeyword ("protected", Token.PROTECTED);
 562                         AddKeyword ("public", Token.PUBLIC);
 563                         AddKeyword ("readonly", Token.READONLY);
 564                         AddKeyword ("ref", Token.REF);
 565                         AddKeyword ("remove", Token.REMOVE);
 566                         AddKeyword ("return", Token.RETURN);
 567                         AddKeyword ("sbyte", Token.SBYTE);
 568                         AddKeyword ("sealed", Token.SEALED);
 569                         AddKeyword ("set", Token.SET);
 570                         AddKeyword ("short", Token.SHORT);
 571                         AddKeyword ("sizeof", Token.SIZEOF);
 572                         AddKeyword ("stackalloc", Token.STACKALLOC);
 573                         AddKeyword ("static", Token.STATIC);
 574                         AddKeyword ("string", Token.STRING);
 575                         AddKeyword ("struct", Token.STRUCT);
 576                         AddKeyword ("switch", Token.SWITCH);
 577                         AddKeyword ("this", Token.THIS);
 578                         AddKeyword ("throw", Token.THROW);
 579                         AddKeyword ("true", Token.TRUE);
 580                         AddKeyword ("try", Token.TRY);
 581                         AddKeyword ("typeof", Token.TYPEOF);
 582                         AddKeyword ("uint", Token.UINT);
 583                         AddKeyword ("ulong", Token.ULONG);
 584                         AddKeyword ("unchecked", Token.UNCHECKED);
 585                         AddKeyword ("unsafe", Token.UNSAFE);
 586                         AddKeyword ("ushort", Token.USHORT);
 587                         AddKeyword ("using", Token.USING);
 588                         AddKeyword ("virtual", Token.VIRTUAL);
 589                         AddKeyword ("void", Token.VOID);
 590                         AddKeyword ("volatile", Token.VOLATILE);
 591                         AddKeyword ("while", Token.WHILE);
 592                         AddKeyword ("partial", Token.PARTIAL);
 593                         AddKeyword ("where", Token.WHERE);
 594
 595                         // LINQ keywords
 596                         AddKeyword ("from", Token.FROM);
 597                         AddKeyword ("join", Token.JOIN);
 598                         AddKeyword ("on", Token.ON);
 599                         AddKeyword ("equals", Token.EQUALS);
 600                         AddKeyword ("select", Token.SELECT);
 601                         AddKeyword ("group", Token.GROUP);
 602                         AddKeyword ("by", Token.BY);
 603                         AddKeyword ("let", Token.LET);
 604                         AddKeyword ("orderby", Token.ORDERBY);
 605                         AddKeyword ("ascending", Token.ASCENDING);
 606                         AddKeyword ("descending", Token.DESCENDING);
 607                         AddKeyword ("into", Token.INTO);
 608
 609                         // Contextual async keywords
 610                         AddKeyword ("async", Token.ASYNC);
 611                         AddKeyword ("await", Token.AWAIT);
 612
 613                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 614
 615                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 616                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 617                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 618                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 619                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 620                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 621                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 622                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 623                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 624                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 625                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 626                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 627
 628                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 629                         styles = NumberStyles.Float;
 630
 631                         string_builder = new System.Text.StringBuilder ();
 632                 }
 633
 634                 int GetKeyword (char[] id, int id_len)
 635                 {
 636                         //
 637                         // Keywords are stored in an array of arrays grouped by their
 638                         // length and then by the first character
 639                         //
 640                         if (id_len >= keywords.Length || keywords [id_len] == null)
 641                                 return -1;
 642
 643                         int first_index = id [0] - '_';
 644                         if (first_index > 'z' - '_')
 645                                 return -1;
 646
 647                         var kwe = keywords [id_len] [first_index];
 648                         if (kwe == null)
 649                                 return -1;
 650
 651                         int res;
 652                         do {
 653                                 res = kwe.Token;
 654                                 for (int i = 1; i < id_len; ++i) {
 655                                         if (id [i] != kwe.Value [i]) {
 656                                                 res = 0;
 657                                                 kwe = kwe.Next;
 658                                                 break;
 659                                         }
 660                                 }
 661                         } while (res == 0 && kwe != null);
 662
 663                         if (res == 0)
 664                                 return -1;
 665
 666                         int next_token;
 667                         switch (res) {
 668                         case Token.GET:
 669                         case Token.SET:
 670                                 if (!handle_get_set)
 671                                         res = -1;
 672                                 break;
 673                         case Token.REMOVE:
 674                         case Token.ADD:
 675                                 if (!handle_remove_add)
 676                                         res = -1;
 677                                 break;
 678                         case Token.EXTERN:
 679                                 if (parsing_declaration == 0)
 680                                         res = Token.EXTERN_ALIAS;
 681                                 break;
 682                         case Token.DEFAULT:
 683                                 if (peek_token () == Token.COLON) {
 684                                         token ();
 685                                         res = Token.DEFAULT_COLON;
 686                                 }
 687                                 break;
 688                         case Token.WHERE:
 689                                 if (!handle_where && !query_parsing)
 690                                         res = -1;
 691                                 break;
 692                         case Token.FROM:
 693                                 //
 694                                 // A query expression is any expression that starts with `from identifier'
 695                                 // followed by any token except ; , =
 696                                 //
 697                                 if (!query_parsing) {
 698                                         if (lambda_arguments_parsing) {
 699                                                 res = -1;
 700                                                 break;
 701                                         }
 702
 703                                         PushPosition ();
 704                                         // HACK: to disable generics micro-parser, because PushPosition does not
 705                                         // store identifiers array
 706                                         parsing_generic_less_than = 1;
 707                                         switch (xtoken ()) {
 708                                         case Token.IDENTIFIER:
 709                                         case Token.INT:
 710                                         case Token.BOOL:
 711                                         case Token.BYTE:
 712                                         case Token.CHAR:
 713                                         case Token.DECIMAL:
 714                                         case Token.FLOAT:
 715                                         case Token.LONG:
 716                                         case Token.OBJECT:
 717                                         case Token.STRING:
 718                                         case Token.UINT:
 719                                         case Token.ULONG:
 720                                                 next_token = xtoken ();
 721                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 722                                                         goto default;
 723
 724                                                 res = Token.FROM_FIRST;
 725                                                 query_parsing = true;
 726                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 727                                                         Report.FeatureIsNotAvailable (context, Location, "query expressions");
 728                                                 break;
 729                                         case Token.VOID:
 730                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 731                                                 break;
 732                                         default:
 733                                                 PopPosition ();
 734                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 735                                                 // which has been overwritten before we grabbed the identifier
 736                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 737                                                 return -1;
 738                                         }
 739                                         PopPosition ();
 740                                 }
 741                                 break;
 742                         case Token.JOIN:
 743                         case Token.ON:
 744                         case Token.EQUALS:
 745                         case Token.SELECT:
 746                         case Token.GROUP:
 747                         case Token.BY:
 748                         case Token.LET:
 749                         case Token.ORDERBY:
 750                         case Token.ASCENDING:
 751                         case Token.DESCENDING:
 752                         case Token.INTO:
 753                                 if (!query_parsing)
 754                                         res = -1;
 755                                 break;
 756
 757                         case Token.USING:
 758                         case Token.NAMESPACE:
 759                                 // TODO: some explanation needed
 760                                 check_incorrect_doc_comment ();
 761                                 break;
 762
 763                         case Token.PARTIAL:
 764                                 if (parsing_block > 0) {
 765                                         res = -1;
 766                                         break;
 767                                 }
 768
 769                                 // Save current position and parse next token.
 770                                 PushPosition ();
 771
 772                                 next_token = token ();
 773                                 bool ok = (next_token == Token.CLASS) ||
 774                                         (next_token == Token.STRUCT) ||
 775                                         (next_token == Token.INTERFACE) ||
 776                                         (next_token == Token.VOID);
 777
 778                                 PopPosition ();
 779
 780                                 if (ok) {
 781                                         if (next_token == Token.VOID) {
 782                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 783                                                         Report.FeatureIsNotAvailable (context, Location, "partial methods");
 784                                         } else if (context.Settings.Version == LanguageVersion.ISO_1)
 785                                                 Report.FeatureIsNotAvailable (context, Location, "partial types");
 786
 787                                         return res;
 788                                 }
 789
 790                                 if (next_token < Token.LAST_KEYWORD) {
 791                                         Report.Error (267, Location,
 792                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 793                                         return token ();
 794                                 }
 795
 796                                 res = -1;
 797                                 break;
 798
 799                         case Token.ASYNC:
 800                                 if (parsing_modifiers) {
 801                                         //
 802                                         // Skip attributes section or constructor called async
 803                                         //
 804                                         if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) {
 805                                                 res = -1;
 806                                         } else {
 807                                                 // async is keyword
 808                                         }
 809                                 } else if (parsing_block > 0) {
 810                                         switch (peek_token ()) {
 811                                         case Token.DELEGATE:
 812                                         case Token.OPEN_PARENS_LAMBDA:
 813                                                 // async is keyword
 814                                                 break;
 815                                         case Token.IDENTIFIER:
 816                                                 PushPosition ();
 817                                                 xtoken ();
 818                                                 if (xtoken () != Token.ARROW)
 819                                                         res = -1;
 820
 821                                                 PopPosition ();
 822                                                 break;
 823                                         default:
 824                                                 res = -1;
 825                                                 break;
 826                                         }
 827                                 } else {
 828                                         res = -1;
 829                                 }
 830
 831                                 if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) {
 832                                         Report.FeatureIsNotAvailable (context, Location, "asynchronous functions");
 833                                 }
 834
 835                                 break;
 836
 837                         case Token.AWAIT:
 838                                 if (parsing_block == 0)
 839                                         res = -1;
 840
 841                                 break;
 842                         }
 843
 844
 845                         return res;
 846                 }
 847
 848                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 849                 {
 850                         //
 851                         // Keywords are stored in an array of arrays grouped by their
 852                         // length and then by the first character
 853                         //
 854                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 855                                 return PreprocessorDirective.Invalid;
 856
 857                         int first_index = id[0] - '_';
 858                         if (first_index > 'z' - '_')
 859                                 return PreprocessorDirective.Invalid;
 860
 861                         var kwe = keywords_preprocessor[id_len][first_index];
 862                         if (kwe == null)
 863                                 return PreprocessorDirective.Invalid;
 864
 865                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 866                         do {
 867                                 res = kwe.Token;
 868                                 for (int i = 1; i < id_len; ++i) {
 869                                         if (id[i] != kwe.Value[i]) {
 870                                                 res = 0;
 871                                                 kwe = kwe.Next;
 872                                                 break;
 873                                         }
 874                                 }
 875                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 876
 877                         return res;
 878                 }
 879
 880                 public Location Location {
 881                         get {
 882                                 return new Location (ref_line, hidden ? -1 : col);
 883                         }
 884                 }
 885
 886                 static bool is_identifier_start_character (int c)
 887                 {
 888                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 889                 }
 890
 891                 static bool is_identifier_part_character (char c)
 892                 {
 893                         if (c >= 'a' && c <= 'z')
 894                                 return true;
 895
 896                         if (c >= 'A' && c <= 'Z')
 897                                 return true;
 898
 899                         if (c == '_' || (c >= '0' && c <= '9'))
 900                                 return true;
 901
 902                         if (c < 0x80)
 903                                 return false;
 904
 905                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 906                 }
 907
 908                 public static bool IsKeyword (string s)
 909                 {
 910                         return keyword_strings.ContainsKey (s);
 911                 }
 912
 913                 //
 914                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 915                 //
 916                 int TokenizeOpenParens ()
 917                 {
 918                         int ptoken;
 919                         current_token = -1;
 920
 921                         int bracket_level = 0;
 922                         bool is_type = false;
 923                         bool can_be_type = false;
 924
 925                         while (true) {
 926                                 ptoken = current_token;
 927                                 token ();
 928
 929                                 switch (current_token) {
 930                                 case Token.CLOSE_PARENS:
 931                                         token ();
 932
 933                                         //
 934                                         // Expression inside parens is lambda, (int i) =>
 935                                         //
 936                                         if (current_token == Token.ARROW)
 937                                                 return Token.OPEN_PARENS_LAMBDA;
 938
 939                                         //
 940                                         // Expression inside parens is single type, (int[])
 941                                         //
 942                                         if (is_type)
 943                                                 return Token.OPEN_PARENS_CAST;
 944
 945                                         //
 946                                         // Expression is possible cast, look at next token, (T)null
 947                                         //
 948                                         if (can_be_type) {
 949                                                 switch (current_token) {
 950                                                 case Token.OPEN_PARENS:
 951                                                 case Token.BANG:
 952                                                 case Token.TILDE:
 953                                                 case Token.IDENTIFIER:
 954                                                 case Token.LITERAL:
 955                                                 case Token.BASE:
 956                                                 case Token.CHECKED:
 957                                                 case Token.DELEGATE:
 958                                                 case Token.FALSE:
 959                                                 case Token.FIXED:
 960                                                 case Token.NEW:
 961                                                 case Token.NULL:
 962                                                 case Token.SIZEOF:
 963                                                 case Token.THIS:
 964                                                 case Token.THROW:
 965                                                 case Token.TRUE:
 966                                                 case Token.TYPEOF:
 967                                                 case Token.UNCHECKED:
 968                                                 case Token.UNSAFE:
 969                                                 case Token.DEFAULT:
 970
 971                                                 //
 972                                                 // These can be part of a member access
 973                                                 //
 974                                                 case Token.INT:
 975                                                 case Token.UINT:
 976                                                 case Token.SHORT:
 977                                                 case Token.USHORT:
 978                                                 case Token.LONG:
 979                                                 case Token.ULONG:
 980                                                 case Token.DOUBLE:
 981                                                 case Token.FLOAT:
 982                                                 case Token.CHAR:
 983                                                 case Token.BYTE:
 984                                                 case Token.DECIMAL:
 985                                                 case Token.BOOL:
 986                                                         return Token.OPEN_PARENS_CAST;
 987                                                 }
 988                                         }
 989                                         return Token.OPEN_PARENS;
 990
 991                                 case Token.DOT:
 992                                 case Token.DOUBLE_COLON:
 993                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
 994                                                 goto default;
 995
 996                                         continue;
 997
 998                                 case Token.IDENTIFIER:
 999                                         switch (ptoken) {
1000                                         case Token.DOT:
1001                                                 if (bracket_level == 0) {
1002                                                         is_type = false;
1003                                                         can_be_type = true;
1004                                                 }
1005
1006                                                 continue;
1007                                         case Token.OP_GENERICS_LT:
1008                                         case Token.COMMA:
1009                                         case Token.DOUBLE_COLON:
1010                                         case -1:
1011                                                 if (bracket_level == 0)
1012                                                         can_be_type = true;
1013                                                 continue;
1014                                         default:
1015                                                 can_be_type = is_type = false;
1016                                                 continue;
1017                                         }
1018
1019                                 case Token.OBJECT:
1020                                 case Token.STRING:
1021                                 case Token.BOOL:
1022                                 case Token.DECIMAL:
1023                                 case Token.FLOAT:
1024                                 case Token.DOUBLE:
1025                                 case Token.SBYTE:
1026                                 case Token.BYTE:
1027                                 case Token.SHORT:
1028                                 case Token.USHORT:
1029                                 case Token.INT:
1030                                 case Token.UINT:
1031                                 case Token.LONG:
1032                                 case Token.ULONG:
1033                                 case Token.CHAR:
1034                                 case Token.VOID:
1035                                         if (bracket_level == 0)
1036                                                 is_type = true;
1037                                         continue;
1038
1039                                 case Token.COMMA:
1040                                         if (bracket_level == 0) {
1041                                                 bracket_level = 100;
1042                                                 can_be_type = is_type = false;
1043                                         }
1044                                         continue;
1045
1046                                 case Token.OP_GENERICS_LT:
1047                                 case Token.OPEN_BRACKET:
1048                                         if (bracket_level++ == 0)
1049                                                 is_type = true;
1050                                         continue;
1051
1052                                 case Token.OP_GENERICS_GT:
1053                                 case Token.CLOSE_BRACKET:
1054                                         --bracket_level;
1055                                         continue;
1056
1057                                 case Token.INTERR_NULLABLE:
1058                                 case Token.STAR:
1059                                         if (bracket_level == 0)
1060                                                 is_type = true;
1061                                         continue;
1062
1063                                 case Token.REF:
1064                                 case Token.OUT:
1065                                         can_be_type = is_type = false;
1066                                         continue;
1067
1068                                 default:
1069                                         return Token.OPEN_PARENS;
1070                                 }
1071                         }
1072                 }
1073
1074                 public static bool IsValidIdentifier (string s)
1075                 {
1076                         if (s == null || s.Length == 0)
1077                                 return false;
1078
1079                         if (!is_identifier_start_character (s [0]))
1080                                 return false;
1081
1082                         for (int i = 1; i < s.Length; i ++)
1083                                 if (! is_identifier_part_character (s [i]))
1084                                         return false;
1085
1086                         return true;
1087                 }
1088
1089                 bool parse_less_than ()
1090                 {
1091                 start:
1092                         int the_token = token ();
1093                         if (the_token == Token.OPEN_BRACKET) {
1094                                 do {
1095                                         the_token = token ();
1096                                 } while (the_token != Token.CLOSE_BRACKET);
1097                                 the_token = token ();
1098                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1099                                 the_token = token ();
1100                         }
1101                         switch (the_token) {
1102                         case Token.IDENTIFIER:
1103                         case Token.OBJECT:
1104                         case Token.STRING:
1105                         case Token.BOOL:
1106                         case Token.DECIMAL:
1107                         case Token.FLOAT:
1108                         case Token.DOUBLE:
1109                         case Token.SBYTE:
1110                         case Token.BYTE:
1111                         case Token.SHORT:
1112                         case Token.USHORT:
1113                         case Token.INT:
1114                         case Token.UINT:
1115                         case Token.LONG:
1116                         case Token.ULONG:
1117                         case Token.CHAR:
1118                         case Token.VOID:
1119                                 break;
1120                         case Token.OP_GENERICS_GT:
1121                         case Token.IN:
1122                         case Token.OUT:
1123                                 return true;
1124
1125                         default:
1126                                 return false;
1127                         }
1128                 again:
1129                         the_token = token ();
1130
1131                         if (the_token == Token.OP_GENERICS_GT)
1132                                 return true;
1133                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1134                                 goto start;
1135                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1136                                 goto again;
1137                         else if (the_token == Token.OP_GENERICS_LT) {
1138                                 if (!parse_less_than ())
1139                                         return false;
1140                                 goto again;
1141                         } else if (the_token == Token.OPEN_BRACKET) {
1142                         rank_specifiers:
1143                                 the_token = token ();
1144                                 if (the_token == Token.CLOSE_BRACKET)
1145                                         goto again;
1146                                 else if (the_token == Token.COMMA)
1147                                         goto rank_specifiers;
1148                                 return false;
1149                         }
1150
1151                         return false;
1152                 }
1153
1154                 bool parse_generic_dimension (out int dimension)
1155                 {
1156                         dimension = 1;
1157
1158                 again:
1159                         int the_token = token ();
1160                         if (the_token == Token.OP_GENERICS_GT)
1161                                 return true;
1162                         else if (the_token == Token.COMMA) {
1163                                 dimension++;
1164                                 goto again;
1165                         }
1166
1167                         return false;
1168                 }
1169
1170                 public int peek_token ()
1171                 {
1172                         int the_token;
1173
1174                         PushPosition ();
1175                         the_token = token ();
1176                         PopPosition ();
1177
1178                         return the_token;
1179                 }
1180
1181                 //
1182                 // Tonizes `?' using custom disambiguous rules to return one
1183                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1184                 //
1185                 // Tricky expression look like:
1186                 //
1187                 // Foo ? a = x ? b : c;
1188                 //
1189                 int TokenizePossibleNullableType ()
1190                 {
1191                         if (parsing_block == 0 || parsing_type > 0)
1192                                 return Token.INTERR_NULLABLE;
1193
1194                         int d = peek_char ();
1195                         if (d == '?') {
1196                                 get_char ();
1197                                 return Token.OP_COALESCING;
1198                         }
1199
1200                         switch (current_token) {
1201                         case Token.CLOSE_PARENS:
1202                         case Token.TRUE:
1203                         case Token.FALSE:
1204                         case Token.NULL:
1205                         case Token.LITERAL:
1206                                 return Token.INTERR;
1207                         }
1208
1209                         if (d != ' ') {
1210                                 if (d == ',' || d == ';' || d == '>')
1211                                         return Token.INTERR_NULLABLE;
1212                                 if (d == '*' || (d >= '0' && d <= '9'))
1213                                         return Token.INTERR;
1214                         }
1215
1216                         PushPosition ();
1217                         current_token = Token.NONE;
1218                         int next_token;
1219                         switch (xtoken ()) {
1220                         case Token.LITERAL:
1221                         case Token.TRUE:
1222                         case Token.FALSE:
1223                         case Token.NULL:
1224                         case Token.THIS:
1225                         case Token.NEW:
1226                                 next_token = Token.INTERR;
1227                                 break;
1228
1229                         case Token.SEMICOLON:
1230                         case Token.COMMA:
1231                         case Token.CLOSE_PARENS:
1232                         case Token.OPEN_BRACKET:
1233                         case Token.OP_GENERICS_GT:
1234                         case Token.INTERR:
1235                                 next_token = Token.INTERR_NULLABLE;
1236                                 break;
1237
1238                         default:
1239                                 next_token = -1;
1240                                 break;
1241                         }
1242
1243                         if (next_token == -1) {
1244                                 switch (xtoken ()) {
1245                                 case Token.COMMA:
1246                                 case Token.SEMICOLON:
1247                                 case Token.OPEN_BRACE:
1248                                 case Token.CLOSE_PARENS:
1249                                 case Token.IN:
1250                                         next_token = Token.INTERR_NULLABLE;
1251                                         break;
1252
1253                                 case Token.COLON:
1254                                         next_token = Token.INTERR;
1255                                         break;
1256
1257                                 default:
1258                                         int ntoken;
1259                                         int interrs = 1;
1260                                         int colons = 0;
1261                                         //
1262                                         // All shorcuts failed, do it hard way
1263                                         //
1264                                         while ((ntoken = xtoken ()) != Token.EOF) {
1265                                                 if (ntoken == Token.SEMICOLON)
1266                                                         break;
1267
1268                                                 if (ntoken == Token.COLON) {
1269                                                         if (++colons == interrs)
1270                                                                 break;
1271                                                         continue;
1272                                                 }
1273
1274                                                 if (ntoken == Token.INTERR) {
1275                                                         ++interrs;
1276                                                         continue;
1277                                                 }
1278                                         }
1279
1280                                         next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1281                                         break;
1282                                 }
1283                         }
1284
1285                         PopPosition ();
1286                         return next_token;
1287                 }
1288
1289                 bool decimal_digits (int c)
1290                 {
1291                         int d;
1292                         bool seen_digits = false;
1293
1294                         if (c != -1){
1295                                 if (number_pos == max_number_size)
1296                                         Error_NumericConstantTooLong ();
1297                                 number_builder [number_pos++] = (char) c;
1298                         }
1299
1300                         //
1301                         // We use peek_char2, because decimal_digits needs to do a
1302                         // 2-character look-ahead (5.ToString for example).
1303                         //
1304                         while ((d = peek_char2 ()) != -1){
1305                                 if (d >= '0' && d <= '9'){
1306                                         if (number_pos == max_number_size)
1307                                                 Error_NumericConstantTooLong ();
1308                                         number_builder [number_pos++] = (char) d;
1309                                         get_char ();
1310                                         seen_digits = true;
1311                                 } else
1312                                         break;
1313                         }
1314
1315                         return seen_digits;
1316                 }
1317
1318                 static bool is_hex (int e)
1319                 {
1320                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1321                 }
1322
1323                 static TypeCode real_type_suffix (int c)
1324                 {
1325                         switch (c){
1326                         case 'F': case 'f':
1327                                 return TypeCode.Single;
1328                         case 'D': case 'd':
1329                                 return TypeCode.Double;
1330                         case 'M': case 'm':
1331                                 return TypeCode.Decimal;
1332                         default:
1333                                 return TypeCode.Empty;
1334                         }
1335                 }
1336
1337                 ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc)
1338                 {
1339                         bool is_unsigned = false;
1340                         bool is_long = false;
1341
1342                         if (c != -1){
1343                                 bool scanning = true;
1344                                 do {
1345                                         switch (c){
1346                                         case 'U': case 'u':
1347                                                 if (is_unsigned)
1348                                                         scanning = false;
1349                                                 is_unsigned = true;
1350                                                 get_char ();
1351                                                 break;
1352
1353                                         case 'l':
1354                                                 if (!is_unsigned){
1355                                                         //
1356                                                         // if we have not seen anything in between
1357                                                         // report this error
1358                                                         //
1359                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1360                                                 }
1361
1362                                                 goto case 'L';
1363
1364                                         case 'L':
1365                                                 if (is_long)
1366                                                         scanning = false;
1367                                                 is_long = true;
1368                                                 get_char ();
1369                                                 break;
1370
1371                                         default:
1372                                                 scanning = false;
1373                                                 break;
1374                                         }
1375                                         c = peek_char ();
1376                                 } while (scanning);
1377                         }
1378
1379                         if (is_long && is_unsigned){
1380                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1381                         }
1382
1383                         if (is_unsigned){
1384                                 // uint if possible, or ulong else.
1385
1386                                 if ((ul & 0xffffffff00000000) == 0)
1387                                         return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc);
1388                                 else
1389                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1390                         } else if (is_long){
1391                                 // long if possible, ulong otherwise
1392                                 if ((ul & 0x8000000000000000) != 0)
1393                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1394                                 else
1395                                         return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1396                         } else {
1397                                 // int, uint, long or ulong in that order
1398                                 if ((ul & 0xffffffff00000000) == 0){
1399                                         uint ui = (uint) ul;
1400
1401                                         if ((ui & 0x80000000) != 0)
1402                                                 return new UIntLiteral (context.BuiltinTypes, ui, loc);
1403                                         else
1404                                                 return new IntLiteral (context.BuiltinTypes, (int) ui, loc);
1405                                 } else {
1406                                         if ((ul & 0x8000000000000000) != 0)
1407                                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1408                                         else
1409                                                 return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1410                                 }
1411                         }
1412                 }
1413
1414                 //
1415                 // given `c' as the next char in the input decide whether
1416                 // we need to convert to a special type, and then choose
1417                 // the best representation for the integer
1418                 //
1419                 ILiteralConstant adjust_int (int c, Location loc)
1420                 {
1421                         try {
1422                                 if (number_pos > 9){
1423                                         ulong ul = (uint) (number_builder [0] - '0');
1424
1425                                         for (int i = 1; i < number_pos; i++){
1426                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1427                                         }
1428
1429                                         return integer_type_suffix (ul, c, loc);
1430                                 } else {
1431                                         uint ui = (uint) (number_builder [0] - '0');
1432
1433                                         for (int i = 1; i < number_pos; i++){
1434                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1435                                         }
1436
1437                                         return integer_type_suffix (ui, c, loc);
1438                                 }
1439                         } catch (OverflowException) {
1440                                 Error_NumericConstantTooLong ();
1441                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1442                         }
1443                         catch (FormatException) {
1444                                 Report.Error (1013, Location, "Invalid number");
1445                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1446                         }
1447                 }
1448
1449                 ILiteralConstant adjust_real (TypeCode t, Location loc)
1450                 {
1451                         string s = new string (number_builder, 0, number_pos);
1452                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1453
1454                         switch (t){
1455                         case TypeCode.Decimal:
1456                                 try {
1457                                         return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc);
1458                                 } catch (OverflowException) {
1459                                         Report.Error (594, Location, error_details, "decimal");
1460                                         return new DecimalLiteral (context.BuiltinTypes, 0, loc);
1461                                 }
1462                         case TypeCode.Single:
1463                                 try {
1464                                         return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc);
1465                                 } catch (OverflowException) {
1466                                         Report.Error (594, Location, error_details, "float");
1467                                         return new FloatLiteral (context.BuiltinTypes, 0, loc);
1468                                 }
1469                         default:
1470                                 try {
1471                                         return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc);
1472                                 } catch (OverflowException) {
1473                                         Report.Error (594, loc, error_details, "double");
1474                                         return new DoubleLiteral (context.BuiltinTypes, 0, loc);
1475                                 }
1476                         }
1477                 }
1478
1479                 ILiteralConstant handle_hex (Location loc)
1480                 {
1481                         int d;
1482                         ulong ul;
1483
1484                         get_char ();
1485                         while ((d = peek_char ()) != -1){
1486                                 if (is_hex (d)){
1487                                         number_builder [number_pos++] = (char) d;
1488                                         get_char ();
1489                                 } else
1490                                         break;
1491                         }
1492
1493                         string s = new String (number_builder, 0, number_pos);
1494
1495                         try {
1496                                 if (number_pos <= 8)
1497                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1498                                 else
1499                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1500
1501                                 return integer_type_suffix (ul, peek_char (), loc);
1502                         } catch (OverflowException){
1503                                 Error_NumericConstantTooLong ();
1504                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1505                         }
1506                         catch (FormatException) {
1507                                 Report.Error (1013, Location, "Invalid number");
1508                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1509                         }
1510                 }
1511
1512                 //
1513                 // Invoked if we know we have .digits or digits
1514                 //
1515                 int is_number (int c)
1516                 {
1517                         ILiteralConstant res;
1518
1519 #if FULL_AST
1520                         int read_start = reader.Position - 1;
1521                         if (c == '.') {
1522                                 //
1523                                 // Caller did peek_char
1524                                 //
1525                                 --read_start;
1526                         }
1527 #endif
1528                         number_pos = 0;
1529                         var loc = Location;
1530
1531                         if (c >= '0' && c <= '9'){
1532                                 if (c == '0'){
1533                                         int peek = peek_char ();
1534
1535                                         if (peek == 'x' || peek == 'X') {
1536                                                 val = res = handle_hex (loc);
1537 #if FULL_AST
1538                                                 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1539 #endif
1540
1541                                                 return Token.LITERAL;
1542                                         }
1543                                 }
1544                                 decimal_digits (c);
1545                                 c = get_char ();
1546                         }
1547
1548                         //
1549                         // We need to handle the case of
1550                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1551                         //
1552                         bool is_real = false;
1553                         if (c == '.'){
1554                                 if (decimal_digits ('.')){
1555                                         is_real = true;
1556                                         c = get_char ();
1557                                 } else {
1558                                         putback ('.');
1559                                         number_pos--;
1560                                         val = res = adjust_int (-1, loc);
1561
1562 #if FULL_AST
1563                                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1564 #endif
1565                                         return Token.LITERAL;
1566                                 }
1567                         }
1568
1569                         if (c == 'e' || c == 'E'){
1570                                 is_real = true;
1571                                 if (number_pos == max_number_size)
1572                                         Error_NumericConstantTooLong ();
1573                                 number_builder [number_pos++] = (char) c;
1574                                 c = get_char ();
1575
1576                                 if (c == '+'){
1577                                         if (number_pos == max_number_size)
1578                                                 Error_NumericConstantTooLong ();
1579                                         number_builder [number_pos++] = '+';
1580                                         c = -1;
1581                                 } else if (c == '-') {
1582                                         if (number_pos == max_number_size)
1583                                                 Error_NumericConstantTooLong ();
1584                                         number_builder [number_pos++] = '-';
1585                                         c = -1;
1586                                 } else {
1587                                         if (number_pos == max_number_size)
1588                                                 Error_NumericConstantTooLong ();
1589                                         number_builder [number_pos++] = '+';
1590                                 }
1591
1592                                 decimal_digits (c);
1593                                 c = get_char ();
1594                         }
1595
1596                         var type = real_type_suffix (c);
1597                         if (type == TypeCode.Empty && !is_real) {
1598                                 putback (c);
1599                                 res = adjust_int (c, loc);
1600                         } else {
1601                                 is_real = true;
1602
1603                                 if (type == TypeCode.Empty) {
1604                                         putback (c);
1605                                 }
1606
1607                                 res = adjust_real (type, loc);
1608                         }
1609
1610                         val = res;
1611
1612 #if FULL_AST
1613                         var chars = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty && c > 0 ? 1 : 0));
1614                         if (chars[chars.Length - 1] == '\r')
1615                                 Array.Resize (ref chars, chars.Length - 1);
1616                         res.ParsedValue = chars;
1617 #endif
1618
1619                         return Token.LITERAL;
1620                 }
1621
1622                 //
1623                 // Accepts exactly count (4 or 8) hex, no more no less
1624                 //
1625                 int getHex (int count, out int surrogate, out bool error)
1626                 {
1627                         int i;
1628                         int total = 0;
1629                         int c;
1630                         int top = count != -1 ? count : 4;
1631
1632                         get_char ();
1633                         error = false;
1634                         surrogate = 0;
1635                         for (i = 0; i < top; i++){
1636                                 c = get_char ();
1637
1638                                 if (c >= '0' && c <= '9')
1639                                         c = (int) c - (int) '0';
1640                                 else if (c >= 'A' && c <= 'F')
1641                                         c = (int) c - (int) 'A' + 10;
1642                                 else if (c >= 'a' && c <= 'f')
1643                                         c = (int) c - (int) 'a' + 10;
1644                                 else {
1645                                         error = true;
1646                                         return 0;
1647                                 }
1648
1649                                 total = (total * 16) + c;
1650                                 if (count == -1){
1651                                         int p = peek_char ();
1652                                         if (p == -1)
1653                                                 break;
1654                                         if (!is_hex ((char)p))
1655                                                 break;
1656                                 }
1657                         }
1658
1659                         if (top == 8) {
1660                                 if (total > 0x0010FFFF) {
1661                                         error = true;
1662                                         return 0;
1663                                 }
1664
1665                                 if (total >= 0x00010000) {
1666                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1667                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1668                                 }
1669                         }
1670
1671                         return total;
1672                 }
1673
1674                 int escape (int c, out int surrogate)
1675                 {
1676                         bool error;
1677                         int d;
1678                         int v;
1679
1680                         d = peek_char ();
1681                         if (c != '\\') {
1682                                 surrogate = 0;
1683                                 return c;
1684                         }
1685
1686                         switch (d){
1687                         case 'a':
1688                                 v = '\a'; break;
1689                         case 'b':
1690                                 v = '\b'; break;
1691                         case 'n':
1692                                 v = '\n'; break;
1693                         case 't':
1694                                 v = '\t'; break;
1695                         case 'v':
1696                                 v = '\v'; break;
1697                         case 'r':
1698                                 v = '\r'; break;
1699                         case '\\':
1700                                 v = '\\'; break;
1701                         case 'f':
1702                                 v = '\f'; break;
1703                         case '0':
1704                                 v = 0; break;
1705                         case '"':
1706                                 v = '"'; break;
1707                         case '\'':
1708                                 v = '\''; break;
1709                         case 'x':
1710                                 v = getHex (-1, out surrogate, out error);
1711                                 if (error)
1712                                         goto default;
1713                                 return v;
1714                         case 'u':
1715                         case 'U':
1716                                 return EscapeUnicode (d, out surrogate);
1717                         default:
1718                                 surrogate = 0;
1719                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1720                                 return d;
1721                         }
1722
1723                         get_char ();
1724                         surrogate = 0;
1725                         return v;
1726                 }
1727
1728                 int EscapeUnicode (int ch, out int surrogate)
1729                 {
1730                         bool error;
1731                         if (ch == 'U') {
1732                                 ch = getHex (8, out surrogate, out error);
1733                         } else {
1734                                 ch = getHex (4, out surrogate, out error);
1735                         }
1736
1737                         if (error)
1738                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1739
1740                         return ch;
1741                 }
1742
1743                 int get_char ()
1744                 {
1745                         int x;
1746                         if (putback_char != -1) {
1747                                 x = putback_char;
1748                                 putback_char = -1;
1749                         } else {
1750                                 x = reader.Read ();
1751                         }
1752
1753                         if (x == '\r') {
1754                                 if (peek_char () == '\n') {
1755                                         putback_char = -1;
1756                                 }
1757
1758                                 x = '\n';
1759                                 advance_line ();
1760                         } else if (x == '\n') {
1761                                 advance_line ();
1762                         } else {
1763                                 col++;
1764                         }
1765                         return x;
1766                 }
1767
1768                 void advance_line ()
1769                 {
1770                         line++;
1771                         ref_line++;
1772                         previous_col = col;
1773                         col = 0;
1774                 }
1775
1776                 int peek_char ()
1777                 {
1778                         if (putback_char == -1)
1779                                 putback_char = reader.Read ();
1780                         return putback_char;
1781                 }
1782
1783                 int peek_char2 ()
1784                 {
1785                         if (putback_char != -1)
1786                                 return putback_char;
1787                         return reader.Peek ();
1788                 }
1789
1790                 void putback (int c)
1791                 {
1792                         if (putback_char != -1){
1793                                 Console.WriteLine ("Col: " + col);
1794                                 Console.WriteLine ("Row: " + line);
1795                                 Console.WriteLine ("Name: " + ref_name.Name);
1796                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1797                                 throw new Exception ("This should not happen putback on putback");
1798                         }
1799                         if (c == '\n' || col == 0) {
1800                                 // It won't happen though.
1801                                 line--;
1802                                 ref_line--;
1803                                 col = previous_col;
1804                         }
1805                         else
1806                                 col--;
1807                         putback_char = c;
1808                 }
1809
1810                 public bool advance ()
1811                 {
1812                         return peek_char () != -1 || CompleteOnEOF;
1813                 }
1814
1815                 public Object Value {
1816                         get {
1817                                 return val;
1818                         }
1819                 }
1820
1821                 public Object value ()
1822                 {
1823                         return val;
1824                 }
1825
1826                 public int token ()
1827                 {
1828                         current_token = xtoken ();
1829                         return current_token;
1830                 }
1831
1832                 int TokenizePreprocessorIdentifier (out int c)
1833                 {
1834                         // skip over white space
1835                         do {
1836                                 c = get_char ();
1837                         } while (c == ' ' || c == '\t');
1838
1839
1840                         int pos = 0;
1841                         while (c != -1 && c >= 'a' && c <= 'z') {
1842                                 id_builder[pos++] = (char) c;
1843                                 c = get_char ();
1844                                 if (c == '\\') {
1845                                         int peek = peek_char ();
1846                                         if (peek == 'U' || peek == 'u') {
1847                                                 int surrogate;
1848                                                 c = EscapeUnicode (c, out surrogate);
1849                                                 if (surrogate != 0) {
1850                                                         if (is_identifier_part_character ((char) c)) {
1851                                                                 id_builder[pos++] = (char) c;
1852                                                         }
1853                                                         c = surrogate;
1854                                                 }
1855                                         }
1856                                 }
1857                         }
1858
1859                         return pos;
1860                 }
1861
1862                 PreprocessorDirective get_cmd_arg (out string arg)
1863                 {
1864                         int c;
1865
1866                         tokens_seen = false;
1867                         arg = "";
1868
1869                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorIdentifier (out c));
1870
1871                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
1872                                 return cmd;
1873
1874                         // skip over white space
1875                         while (c == ' ' || c == '\t')
1876                                 c = get_char ();
1877
1878                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
1879                         int pos = 0;
1880
1881                         while (c != -1 && c != '\n') {
1882                                 if (c == '\\' && has_identifier_argument >= 0) {
1883                                         if (has_identifier_argument != 0) {
1884                                                 has_identifier_argument = 1;
1885
1886                                                 int peek = peek_char ();
1887                                                 if (peek == 'U' || peek == 'u') {
1888                                                         int surrogate;
1889                                                         c = EscapeUnicode (c, out surrogate);
1890                                                         if (surrogate != 0) {
1891                                                                 if (is_identifier_part_character ((char) c)) {
1892                                                                         if (pos == value_builder.Length)
1893                                                                                 Array.Resize (ref value_builder, pos * 2);
1894
1895                                                                         value_builder[pos++] = (char) c;
1896                                                                 }
1897                                                                 c = surrogate;
1898                                                         }
1899                                                 }
1900                                         } else {
1901                                                 has_identifier_argument = -1;
1902                                         }
1903                                 } else if (c == '/' && peek_char () == '/') {
1904                                         //
1905                                         // Eat single-line comments
1906                                         //
1907                                         get_char ();
1908                                         do {
1909                                                 c = get_char ();
1910                                         } while (c != -1 && c != '\n');
1911
1912                                         break;
1913                                 }
1914
1915                                 if (pos == value_builder.Length)
1916                                         Array.Resize (ref value_builder, pos * 2);
1917
1918                                 value_builder[pos++] = (char) c;
1919                                 c = get_char ();
1920                         }
1921
1922                         if (pos != 0) {
1923                                 if (pos > max_id_size)
1924                                         arg = new string (value_builder, 0, pos);
1925                                 else
1926                                         arg = InternIdentifier (value_builder, pos);
1927
1928                                 // Eat any trailing whitespaces
1929                                 arg = arg.Trim (simple_whitespaces);
1930                         }
1931
1932                         return cmd;
1933                 }
1934
1935                 //
1936                 // Handles the #line directive
1937                 //
1938                 bool PreProcessLine (string arg)
1939                 {
1940                         if (arg.Length == 0)
1941                                 return false;
1942
1943                         if (arg == "default"){
1944                                 ref_line = line;
1945                                 ref_name = file_name;
1946                                 hidden = false;
1947                                 Location.Push (file_name, ref_name);
1948                                 return true;
1949                         } else if (arg == "hidden"){
1950                                 hidden = true;
1951                                 return true;
1952                         }
1953
1954                         try {
1955                                 int pos;
1956
1957                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1958                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1959                                         pos++;
1960
1961                                         char [] quotes = { '\"' };
1962
1963                                         string name = arg.Substring (pos). Trim (quotes);
1964                                         ref_name = context.LookupFile (file_name, name);
1965                                         file_name.AddIncludeFile (ref_name);
1966                                         hidden = false;
1967                                         Location.Push (file_name, ref_name);
1968                                 } else {
1969                                         ref_line = System.Int32.Parse (arg);
1970                                         hidden = false;
1971                                 }
1972                         } catch {
1973                                 return false;
1974                         }
1975
1976                         return true;
1977                 }
1978
1979                 //
1980                 // Handles #define and #undef
1981                 //
1982                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
1983                 {
1984                         if (ident.Length == 0 || ident == "true" || ident == "false"){
1985                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
1986                                 return;
1987                         }
1988
1989                         if (ident.IndexOfAny (simple_whitespaces) != -1){
1990                                 Error_EndLineExpected ();
1991                                 return;
1992                         }
1993
1994                         if (!is_identifier_start_character (ident [0]))
1995                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
1996
1997                         foreach (char c in ident.Substring (1)){
1998                                 if (!is_identifier_part_character (c)){
1999                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
2000                                         return;
2001                                 }
2002                         }
2003
2004                         if (!caller_is_taking)
2005                                 return;
2006
2007                         if (is_define) {
2008                                 //
2009                                 // #define ident
2010                                 //
2011                                 if (context.Settings.IsConditionalSymbolDefined (ident))
2012                                         return;
2013
2014                                 file_name.AddDefine (ident);
2015                         } else {
2016                                 //
2017                                 // #undef ident
2018                                 //
2019                                 file_name.AddUndefine (ident);
2020                         }
2021                 }
2022
2023                 byte read_hex (out bool error)
2024                 {
2025                         int total;
2026                         int c = get_char ();
2027
2028                         if ((c >= '0') && (c <= '9'))
2029                                 total = (int) c - (int) '0';
2030                         else if ((c >= 'A') && (c <= 'F'))
2031                                 total = (int) c - (int) 'A' + 10;
2032                         else if ((c >= 'a') && (c <= 'f'))
2033                                 total = (int) c - (int) 'a' + 10;
2034                         else {
2035                                 error = true;
2036                                 return 0;
2037                         }
2038
2039                         total *= 16;
2040                         c = get_char ();
2041
2042                         if ((c >= '0') && (c <= '9'))
2043                                 total += (int) c - (int) '0';
2044                         else if ((c >= 'A') && (c <= 'F'))
2045                                 total += (int) c - (int) 'A' + 10;
2046                         else if ((c >= 'a') && (c <= 'f'))
2047                                 total += (int) c - (int) 'a' + 10;
2048                         else {
2049                                 error = true;
2050                                 return 0;
2051                         }
2052
2053                         error = false;
2054                         return (byte) total;
2055                 }
2056
2057                 //
2058                 // Parses #pragma checksum
2059                 //
2060                 bool ParsePragmaChecksum ()
2061                 {
2062                         //
2063                         // The syntax is ` "foo.txt" "{guid}" "hash"'
2064                         //
2065                         int c = get_char ();
2066
2067                         if (c != '"')
2068                                 return false;
2069
2070                         string_builder.Length = 0;
2071                         while (c != -1 && c != '\n') {
2072                                 c = get_char ();
2073                                 if (c == '"') {
2074                                         c = get_char ();
2075                                         break;
2076                                 }
2077
2078                                 string_builder.Append ((char) c);
2079                         }
2080
2081                         if (string_builder.Length == 0) {
2082                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
2083                         }
2084
2085                         // TODO: Any white-spaces count
2086                         if (c != ' ')
2087                                 return false;
2088
2089                         SourceFile file = context.LookupFile (file_name, string_builder.ToString ());
2090
2091                         if (get_char () != '"' || get_char () != '{')
2092                                 return false;
2093
2094                         bool error;
2095                         byte[] guid_bytes = new byte [16];
2096                         int i = 0;
2097
2098                         for (; i < 4; i++) {
2099                                 guid_bytes [i] = read_hex (out error);
2100                                 if (error)
2101                                         return false;
2102                         }
2103
2104                         if (get_char () != '-')
2105                                 return false;
2106
2107                         for (; i < 10; i++) {
2108                                 guid_bytes [i] = read_hex (out error);
2109                                 if (error)
2110                                         return false;
2111
2112                                 guid_bytes [i++] = read_hex (out error);
2113                                 if (error)
2114                                         return false;
2115
2116                                 if (get_char () != '-')
2117                                         return false;
2118                         }
2119
2120                         for (; i < 16; i++) {
2121                                 guid_bytes [i] = read_hex (out error);
2122                                 if (error)
2123                                         return false;
2124                         }
2125
2126                         if (get_char () != '}' || get_char () != '"')
2127                                 return false;
2128
2129                         // TODO: Any white-spaces count
2130                         c = get_char ();
2131                         if (c != ' ')
2132                                 return false;
2133
2134                         if (get_char () != '"')
2135                                 return false;
2136
2137                         // Any length of checksum
2138                         List<byte> checksum_bytes = new List<byte> (16);
2139
2140                         c = peek_char ();
2141                         while (c != '"' && c != -1) {
2142                                 checksum_bytes.Add (read_hex (out error));
2143                                 if (error)
2144                                         return false;
2145
2146                                 c = peek_char ();
2147                         }
2148
2149                         if (c == '/') {
2150                                 ReadSingleLineComment ();
2151                         } else if (get_char () != '"') {
2152                                 return false;
2153                         }
2154
2155                         file.SetChecksum (guid_bytes, checksum_bytes.ToArray ());
2156                         ref_name.AutoGenerated = true;
2157                         return true;
2158                 }
2159
2160                 bool IsTokenIdentifierEqual (char[] identifier)
2161                 {
2162                         for (int i = 0; i < identifier.Length; ++i) {
2163                                 if (identifier[i] != id_builder[i])
2164                                         return false;
2165                         }
2166
2167                         return true;
2168                 }
2169
2170                 int TokenizePragmaNumber (ref int c)
2171                 {
2172                         number_pos = 0;
2173
2174                         int number;
2175
2176                         if (c >= '0' && c <= '9') {
2177                                 decimal_digits (c);
2178                                 uint ui = (uint) (number_builder[0] - '0');
2179
2180                                 try {
2181                                         for (int i = 1; i < number_pos; i++) {
2182                                                 ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2183                                         }
2184
2185                                         number = (int) ui;
2186                                 } catch (OverflowException) {
2187                                         Error_NumericConstantTooLong ();
2188                                         number = -1;
2189                                 }
2190
2191
2192                                 c = get_char ();
2193
2194                                 // skip over white space
2195                                 while (c == ' ' || c == '\t')
2196                                         c = get_char ();
2197
2198                                 if (c == ',') {
2199                                         c = get_char ();
2200                                 }
2201
2202                                 // skip over white space
2203                                 while (c == ' ' || c == '\t')
2204                                         c = get_char ();
2205                         } else {
2206                                 number = -1;
2207                                 if (c == '/') {
2208                                         ReadSingleLineComment ();
2209                                 } else {
2210                                         Report.Warning (1692, 1, Location, "Invalid number");
2211
2212                                         // Read everything till the end of the line or file
2213                                         do {
2214                                                 c = get_char ();
2215                                         } while (c != -1 && c != '\n');
2216                                 }
2217                         }
2218
2219                         return number;
2220                 }
2221
2222                 void ReadSingleLineComment ()
2223                 {
2224                         if (peek_char () != '/')
2225                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2226
2227                         // Read everything till the end of the line or file
2228                         int c;
2229                         do {
2230                                 c = get_char ();
2231                         } while (c != -1 && c != '\n');
2232                 }
2233
2234                 /// <summary>
2235                 /// Handles #pragma directive
2236                 /// </summary>
2237                 void ParsePragmaDirective (string arg)
2238                 {
2239                         int c;
2240                         int length = TokenizePreprocessorIdentifier (out c);
2241                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2242                                 length = TokenizePreprocessorIdentifier (out c);
2243
2244                                 //
2245                                 // #pragma warning disable
2246                                 // #pragma warning restore
2247                                 //
2248                                 if (length == pragma_warning_disable.Length) {
2249                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2250                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2251                                                 // skip over white space
2252                                                 while (c == ' ' || c == '\t')
2253                                                         c = get_char ();
2254
2255                                                 var loc = Location;
2256
2257                                                 if (c == '\n' || c == '/') {
2258                                                         if (c == '/')
2259                                                                 ReadSingleLineComment ();
2260
2261                                                         //
2262                                                         // Disable/Restore all warnings
2263                                                         //
2264                                                         if (disable) {
2265                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2266                                                         } else {
2267                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2268                                                         }
2269                                                 } else {
2270                                                         //
2271                                                         // Disable/Restore a warning or group of warnings
2272                                                         //
2273                                                         int code;
2274                                                         do {
2275                                                                 code = TokenizePragmaNumber (ref c);
2276                                                                 if (code > 0) {
2277                                                                         if (disable) {
2278                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, Report);
2279                                                                         } else {
2280                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, Report);
2281                                                                         }
2282                                                                 }
2283                                                         } while (code >= 0 && c != '\n' && c != -1);
2284                                                 }
2285
2286                                                 return;
2287                                         }
2288                                 }
2289
2290                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2291                                 return;
2292                         }
2293
2294                         //
2295                         // #pragma checksum
2296                         //
2297                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2298                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2299                                         Report.Warning (1695, 1, Location,
2300                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2301                                 }
2302
2303                                 return;
2304                         }
2305
2306                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2307                 }
2308
2309                 bool eval_val (string s)
2310                 {
2311                         if (s == "true")
2312                                 return true;
2313                         if (s == "false")
2314                                 return false;
2315
2316                         return file_name.IsConditionalDefined (context, s);
2317                 }
2318
2319                 bool pp_primary (ref string s)
2320                 {
2321                         s = s.Trim ();
2322                         int len = s.Length;
2323
2324                         if (len > 0){
2325                                 char c = s [0];
2326
2327                                 if (c == '('){
2328                                         s = s.Substring (1);
2329                                         bool val = pp_expr (ref s, false);
2330                                         if (s.Length > 0 && s [0] == ')'){
2331                                                 s = s.Substring (1);
2332                                                 return val;
2333                                         }
2334                                         Error_InvalidDirective ();
2335                                         return false;
2336                                 }
2337
2338                                 if (is_identifier_start_character (c)){
2339                                         int j = 1;
2340
2341                                         while (j < len){
2342                                                 c = s [j];
2343
2344                                                 if (is_identifier_part_character (c)){
2345                                                         j++;
2346                                                         continue;
2347                                                 }
2348                                                 bool v = eval_val (s.Substring (0, j));
2349                                                 s = s.Substring (j);
2350                                                 return v;
2351                                         }
2352                                         bool vv = eval_val (s);
2353                                         s = "";
2354                                         return vv;
2355                                 }
2356                         }
2357                         Error_InvalidDirective ();
2358                         return false;
2359                 }
2360
2361                 bool pp_unary (ref string s)
2362                 {
2363                         s = s.Trim ();
2364                         int len = s.Length;
2365
2366                         if (len > 0){
2367                                 if (s [0] == '!'){
2368                                         if (len > 1 && s [1] == '='){
2369                                                 Error_InvalidDirective ();
2370                                                 return false;
2371                                         }
2372                                         s = s.Substring (1);
2373                                         return ! pp_primary (ref s);
2374                                 } else
2375                                         return pp_primary (ref s);
2376                         } else {
2377                                 Error_InvalidDirective ();
2378                                 return false;
2379                         }
2380                 }
2381
2382                 bool pp_eq (ref string s)
2383                 {
2384                         bool va = pp_unary (ref s);
2385
2386                         s = s.Trim ();
2387                         int len = s.Length;
2388                         if (len > 0){
2389                                 if (s [0] == '='){
2390                                         if (len > 2 && s [1] == '='){
2391                                                 s = s.Substring (2);
2392                                                 return va == pp_unary (ref s);
2393                                         } else {
2394                                                 Error_InvalidDirective ();
2395                                                 return false;
2396                                         }
2397                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2398                                         s = s.Substring (2);
2399
2400                                         return va != pp_unary (ref s);
2401
2402                                 }
2403                         }
2404
2405                         return va;
2406
2407                 }
2408
2409                 bool pp_and (ref string s)
2410                 {
2411                         bool va = pp_eq (ref s);
2412
2413                         s = s.Trim ();
2414                         int len = s.Length;
2415                         if (len > 0){
2416                                 if (s [0] == '&'){
2417                                         if (len > 2 && s [1] == '&'){
2418                                                 s = s.Substring (2);
2419                                                 return (va & pp_and (ref s));
2420                                         } else {
2421                                                 Error_InvalidDirective ();
2422                                                 return false;
2423                                         }
2424                                 }
2425                         }
2426                         return va;
2427                 }
2428
2429                 //
2430                 // Evaluates an expression for `#if' or `#elif'
2431                 //
2432                 bool pp_expr (ref string s, bool isTerm)
2433                 {
2434                         bool va = pp_and (ref s);
2435                         s = s.Trim ();
2436                         int len = s.Length;
2437                         if (len > 0){
2438                                 char c = s [0];
2439
2440                                 if (c == '|'){
2441                                         if (len > 2 && s [1] == '|'){
2442                                                 s = s.Substring (2);
2443                                                 return va | pp_expr (ref s, isTerm);
2444                                         } else {
2445                                                 Error_InvalidDirective ();
2446                                                 return false;
2447                                         }
2448                                 }
2449                                 if (isTerm) {
2450                                         Error_EndLineExpected ();
2451                                         return false;
2452                                 }
2453                         }
2454
2455                         return va;
2456                 }
2457
2458                 bool eval (string s)
2459                 {
2460                         bool v = pp_expr (ref s, true);
2461                         s = s.Trim ();
2462                         if (s.Length != 0){
2463                                 return false;
2464                         }
2465
2466                         return v;
2467                 }
2468
2469                 void Error_NumericConstantTooLong ()
2470                 {
2471                         Report.Error (1021, Location, "Integral constant is too large");
2472                 }
2473
2474                 void Error_InvalidDirective ()
2475                 {
2476                         Report.Error (1517, Location, "Invalid preprocessor directive");
2477                 }
2478
2479                 void Error_UnexpectedDirective (string extra)
2480                 {
2481                         Report.Error (
2482                                 1028, Location,
2483                                 "Unexpected processor directive ({0})", extra);
2484                 }
2485
2486                 void Error_TokensSeen ()
2487                 {
2488                         Report.Error (1032, Location,
2489                                 "Cannot define or undefine preprocessor symbols after first token in file");
2490                 }
2491
2492                 void Eror_WrongPreprocessorLocation ()
2493                 {
2494                         Report.Error (1040, Location,
2495                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2496                 }
2497
2498                 void Error_EndLineExpected ()
2499                 {
2500                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2501                 }
2502
2503                 //
2504                 // Raises a warning when tokenizer found documentation comment
2505                 // on unexpected place
2506                 //
2507                 void WarningMisplacedComment (Location loc)
2508                 {
2509                         if (doc_state != XmlCommentState.Error) {
2510                                 doc_state = XmlCommentState.Error;
2511                                 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element");
2512                         }
2513                 }
2514
2515                 //
2516                 // if true, then the code continues processing the code
2517                 // if false, the code stays in a loop until another directive is
2518                 // reached.
2519                 // When caller_is_taking is false we ignore all directives except the ones
2520                 // which can help us to identify where the #if block ends
2521                 bool ParsePreprocessingDirective (bool caller_is_taking)
2522                 {
2523                         string arg;
2524                         bool region_directive = false;
2525
2526                         var directive = get_cmd_arg (out arg);
2527
2528                         //
2529                         // The first group of pre-processing instructions is always processed
2530                         //
2531                         switch (directive) {
2532                         case PreprocessorDirective.Region:
2533                                 region_directive = true;
2534                                 arg = "true";
2535                                 goto case PreprocessorDirective.If;
2536
2537                         case PreprocessorDirective.Endregion:
2538                                 if (ifstack == null || ifstack.Count == 0){
2539                                         Error_UnexpectedDirective ("no #region for this #endregion");
2540                                         return true;
2541                                 }
2542                                 int pop = ifstack.Pop ();
2543
2544                                 if ((pop & REGION) == 0)
2545                                         Report.Error (1027, Location, "Expected `#endif' directive");
2546
2547                                 return caller_is_taking;
2548
2549                         case PreprocessorDirective.If:
2550                                 if (ifstack == null)
2551                                         ifstack = new Stack<int> (2);
2552
2553                                 int flags = region_directive ? REGION : 0;
2554                                 if (ifstack.Count == 0){
2555                                         flags |= PARENT_TAKING;
2556                                 } else {
2557                                         int state = ifstack.Peek ();
2558                                         if ((state & TAKING) != 0) {
2559                                                 flags |= PARENT_TAKING;
2560                                         }
2561                                 }
2562
2563                                 if (eval (arg) && caller_is_taking) {
2564                                         ifstack.Push (flags | TAKING);
2565                                         return true;
2566                                 }
2567                                 ifstack.Push (flags);
2568                                 return false;
2569
2570                         case PreprocessorDirective.Endif:
2571                                 if (ifstack == null || ifstack.Count == 0){
2572                                         Error_UnexpectedDirective ("no #if for this #endif");
2573                                         return true;
2574                                 } else {
2575                                         pop = ifstack.Pop ();
2576
2577                                         if ((pop & REGION) != 0)
2578                                                 Report.Error (1038, Location, "#endregion directive expected");
2579
2580                                         if (arg.Length != 0) {
2581                                                 Error_EndLineExpected ();
2582                                         }
2583
2584                                         if (ifstack.Count == 0)
2585                                                 return true;
2586
2587                                         int state = ifstack.Peek ();
2588                                         return (state & TAKING) != 0;
2589                                 }
2590
2591                         case PreprocessorDirective.Elif:
2592                                 if (ifstack == null || ifstack.Count == 0){
2593                                         Error_UnexpectedDirective ("no #if for this #elif");
2594                                         return true;
2595                                 } else {
2596                                         int state = ifstack.Pop ();
2597
2598                                         if ((state & REGION) != 0) {
2599                                                 Report.Error (1038, Location, "#endregion directive expected");
2600                                                 return true;
2601                                         }
2602
2603                                         if ((state & ELSE_SEEN) != 0){
2604                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2605                                                 return true;
2606                                         }
2607
2608                                         if ((state & TAKING) != 0) {
2609                                                 ifstack.Push (0);
2610                                                 return false;
2611                                         }
2612
2613                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2614                                                 ifstack.Push (state | TAKING);
2615                                                 return true;
2616                                         }
2617
2618                                         ifstack.Push (state);
2619                                         return false;
2620                                 }
2621
2622                         case PreprocessorDirective.Else:
2623                                 if (ifstack == null || ifstack.Count == 0){
2624                                         Error_UnexpectedDirective ("no #if for this #else");
2625                                         return true;
2626                                 } else {
2627                                         int state = ifstack.Peek ();
2628
2629                                         if ((state & REGION) != 0) {
2630                                                 Report.Error (1038, Location, "#endregion directive expected");
2631                                                 return true;
2632                                         }
2633
2634                                         if ((state & ELSE_SEEN) != 0){
2635                                                 Error_UnexpectedDirective ("#else within #else");
2636                                                 return true;
2637                                         }
2638
2639                                         ifstack.Pop ();
2640
2641                                         if (arg.Length != 0) {
2642                                                 Error_EndLineExpected ();
2643                                                 return true;
2644                                         }
2645
2646                                         bool ret = false;
2647                                         if ((state & PARENT_TAKING) != 0) {
2648                                                 ret = (state & TAKING) == 0;
2649
2650                                                 if (ret)
2651                                                         state |= TAKING;
2652                                                 else
2653                                                         state &= ~TAKING;
2654                                         }
2655
2656                                         ifstack.Push (state | ELSE_SEEN);
2657
2658                                         return ret;
2659                                 }
2660                         case PreprocessorDirective.Define:
2661                                 if (any_token_seen){
2662                                         Error_TokensSeen ();
2663                                         return caller_is_taking;
2664                                 }
2665                                 PreProcessDefinition (true, arg, caller_is_taking);
2666                                 return caller_is_taking;
2667
2668                         case PreprocessorDirective.Undef:
2669                                 if (any_token_seen){
2670                                         Error_TokensSeen ();
2671                                         return caller_is_taking;
2672                                 }
2673                                 PreProcessDefinition (false, arg, caller_is_taking);
2674                                 return caller_is_taking;
2675
2676                         case PreprocessorDirective.Invalid:
2677                                 Report.Error (1024, Location, "Wrong preprocessor directive");
2678                                 return true;
2679                         }
2680
2681                         //
2682                         // These are only processed if we are in a `taking' block
2683                         //
2684                         if (!caller_is_taking)
2685                                 return false;
2686
2687                         switch (directive){
2688                         case PreprocessorDirective.Error:
2689                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2690                                 return true;
2691
2692                         case PreprocessorDirective.Warning:
2693                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2694                                 return true;
2695
2696                         case PreprocessorDirective.Pragma:
2697                                 if (context.Settings.Version == LanguageVersion.ISO_1) {
2698                                         Report.FeatureIsNotAvailable (context, Location, "#pragma");
2699                                 }
2700
2701                                 ParsePragmaDirective (arg);
2702                                 return true;
2703
2704                         case PreprocessorDirective.Line:
2705                                 if (!PreProcessLine (arg))
2706                                         Report.Error (
2707                                                 1576, Location,
2708                                                 "The line number specified for #line directive is missing or invalid");
2709                                 return caller_is_taking;
2710                         }
2711
2712                         throw new NotImplementedException (directive.ToString ());
2713                 }
2714
2715                 private int consume_string (bool quoted)
2716                 {
2717                         int c;
2718                         int pos = 0;
2719                         Location start_location = Location;
2720                         if (quoted)
2721                                 start_location = start_location - 1;
2722
2723 #if FULL_AST
2724                         int reader_pos = reader.Position;
2725 #endif
2726
2727                         while (true){
2728                                 c = get_char ();
2729                                 if (c == '"') {
2730                                         if (quoted && peek_char () == '"') {
2731                                                 if (pos == value_builder.Length)
2732                                                         Array.Resize (ref value_builder, pos * 2);
2733
2734                                                 value_builder[pos++] = (char) c;
2735                                                 get_char ();
2736                                                 continue;
2737                                         }
2738
2739                                         string s;
2740                                         if (pos == 0)
2741                                                 s = string.Empty;
2742                                         else if (pos <= 4)
2743                                                 s = InternIdentifier (value_builder, pos);
2744                                         else
2745                                                 s = new string (value_builder, 0, pos);
2746
2747                                         ILiteralConstant res = new StringLiteral (context.BuiltinTypes, s, start_location);
2748                                         val = res;
2749 #if FULL_AST
2750                                         res.ParsedValue = quoted ?
2751                                                 reader.ReadChars (reader_pos - 2, reader.Position - 1) :
2752                                                 reader.ReadChars (reader_pos - 1, reader.Position);
2753 #endif
2754
2755                                         return Token.LITERAL;
2756                                 }
2757
2758                                 if (c == '\n') {
2759                                         if (!quoted) {
2760                                                 Report.Error (1010, Location, "Newline in constant");
2761                                                 val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location);
2762                                                 return Token.LITERAL;
2763                                         }
2764                                 } else if (c == '\\' && !quoted) {
2765                                         int surrogate;
2766                                         c = escape (c, out surrogate);
2767                                         if (c == -1)
2768                                                 return Token.ERROR;
2769                                         if (surrogate != 0) {
2770                                                 if (pos == value_builder.Length)
2771                                                         Array.Resize (ref value_builder, pos * 2);
2772
2773                                                 value_builder[pos++] = (char) c;
2774                                                 c = surrogate;
2775                                         }
2776                                 } else if (c == -1) {
2777                                         Report.Error (1039, Location, "Unterminated string literal");
2778                                         return Token.EOF;
2779                                 }
2780
2781                                 if (pos == value_builder.Length)
2782                                         Array.Resize (ref value_builder, pos * 2);
2783
2784                                 value_builder[pos++] = (char) c;
2785                         }
2786                 }
2787
2788                 private int consume_identifier (int s)
2789                 {
2790                         int res = consume_identifier (s, false);
2791
2792                         if (doc_state == XmlCommentState.Allowed)
2793                                 doc_state = XmlCommentState.NotAllowed;
2794
2795                         return res;
2796                 }
2797
2798                 int consume_identifier (int c, bool quoted)
2799                 {
2800                         //
2801                         // This method is very performance sensitive. It accounts
2802                         // for approximately 25% of all parser time
2803                         //
2804
2805                         int pos = 0;
2806                         int column = col;
2807                         if (quoted)
2808                                 --column;
2809
2810                         if (c == '\\') {
2811                                 int surrogate;
2812                                 c = escape (c, out surrogate);
2813                                 if (surrogate != 0) {
2814                                         id_builder [pos++] = (char) c;
2815                                         c = surrogate;
2816                                 }
2817                         }
2818
2819                         id_builder [pos++] = (char) c;
2820
2821                         try {
2822                                 while (true) {
2823                                         c = reader.Read ();
2824
2825                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
2826                                                 id_builder [pos++] = (char) c;
2827                                                 continue;
2828                                         }
2829
2830                                         if (c < 0x80) {
2831                                                 if (c == '\\') {
2832                                                         int surrogate;
2833                                                         c = escape (c, out surrogate);
2834                                                         if (is_identifier_part_character ((char) c))
2835                                                                 id_builder[pos++] = (char) c;
2836
2837                                                         if (surrogate != 0) {
2838                                                                 c = surrogate;
2839                                                         }
2840
2841                                                         continue;
2842                                                 }
2843                                         } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
2844                                                 id_builder [pos++] = (char) c;
2845                                                 continue;
2846                                         }
2847
2848                                         putback_char = c;
2849                                         break;
2850                                 }
2851                         } catch (IndexOutOfRangeException) {
2852                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2853                                 --pos;
2854                                 col += pos;
2855                         }
2856
2857                         col += pos - 1;
2858
2859                         //
2860                         // Optimization: avoids doing the keyword lookup
2861                         // on uppercase letters
2862                         //
2863                         if (id_builder [0] >= '_' && !quoted) {
2864                                 int keyword = GetKeyword (id_builder, pos);
2865                                 if (keyword != -1) {
2866                                         val = LocatedToken.Create (keyword == Token.AWAIT ? "await" : null, ref_line, column);
2867                                         return keyword;
2868                                 }
2869                         }
2870
2871                         string s = InternIdentifier (id_builder, pos);
2872                         val = LocatedToken.Create (s, ref_line, column);
2873                         if (quoted && parsing_attribute_section)
2874                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2875
2876                         return Token.IDENTIFIER;
2877                 }
2878
2879                 static string InternIdentifier (char[] charBuffer, int length)
2880                 {
2881                         //
2882                         // Keep identifiers in an array of hashtables to avoid needless
2883                         // allocations
2884                         //
2885                         var identifiers_group = identifiers[length];
2886                         string s;
2887                         if (identifiers_group != null) {
2888                                 if (identifiers_group.TryGetValue (charBuffer, out s)) {
2889                                         return s;
2890                                 }
2891                         } else {
2892                                 // TODO: this should be number of files dependant
2893                                 // corlib compilation peaks at 1000 and System.Core at 150
2894                                 int capacity = length > 20 ? 10 : 100;
2895                                 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length));
2896                                 identifiers[length] = identifiers_group;
2897                         }
2898
2899                         char[] chars = new char[length];
2900                         Array.Copy (charBuffer, chars, length);
2901
2902                         s = new string (charBuffer, 0, length);
2903                         identifiers_group.Add (chars, s);
2904                         return s;
2905                 }
2906
2907                 public int xtoken ()
2908                 {
2909                         int d, c;
2910
2911                         // Whether we have seen comments on the current line
2912                         bool comments_seen = false;
2913                         while ((c = get_char ()) != -1) {
2914                                 switch (c) {
2915                                 case '\t':
2916                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
2917                                         continue;
2918
2919                                 case ' ':
2920                                 case '\f':
2921                                 case '\v':
2922                                 case 0xa0:
2923                                 case 0:
2924                                 case 0xFEFF:    // Ignore BOM anywhere in the file
2925                                         continue;
2926
2927 /*                              This is required for compatibility with .NET
2928                                 case 0xEF:
2929                                         if (peek_char () == 0xBB) {
2930                                                 PushPosition ();
2931                                                 get_char ();
2932                                                 if (get_char () == 0xBF)
2933                                                         continue;
2934                                                 PopPosition ();
2935                                         }
2936                                         break;
2937 */
2938                                 case '\\':
2939                                         tokens_seen = true;
2940                                         return consume_identifier (c);
2941
2942                                 case '{':
2943                                         val = LocatedToken.Create (ref_line, col);
2944                                         return Token.OPEN_BRACE;
2945                                 case '}':
2946                                         val = LocatedToken.Create (ref_line, col);
2947                                         return Token.CLOSE_BRACE;
2948                                 case '[':
2949                                         // To block doccomment inside attribute declaration.
2950                                         if (doc_state == XmlCommentState.Allowed)
2951                                                 doc_state = XmlCommentState.NotAllowed;
2952
2953                                         val = LocatedToken.Create (ref_line, col);
2954
2955                                         if (parsing_block == 0 || lambda_arguments_parsing)
2956                                                 return Token.OPEN_BRACKET;
2957
2958                                         int next = peek_char ();
2959                                         switch (next) {
2960                                         case ']':
2961                                         case ',':
2962                                                 return Token.OPEN_BRACKET;
2963
2964                                         case ' ':
2965                                         case '\f':
2966                                         case '\v':
2967                                         case '\r':
2968                                         case '\n':
2969                                         case '/':
2970                                                 next = peek_token ();
2971                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
2972                                                         return Token.OPEN_BRACKET;
2973
2974                                                 return Token.OPEN_BRACKET_EXPR;
2975                                         default:
2976                                                 return Token.OPEN_BRACKET_EXPR;
2977                                         }
2978                                 case ']':
2979                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2980                                         return Token.CLOSE_BRACKET;
2981                                 case '(':
2982                                         val = LocatedToken.Create (ref_line, col);
2983                                         //
2984                                         // An expression versions of parens can appear in block context only
2985                                         //
2986                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
2987
2988                                                 //
2989                                                 // Optmize most common case where we know that parens
2990                                                 // is not special
2991                                                 //
2992                                                 switch (current_token) {
2993                                                 case Token.IDENTIFIER:
2994                                                 case Token.IF:
2995                                                 case Token.FOR:
2996                                                 case Token.FOREACH:
2997                                                 case Token.TYPEOF:
2998                                                 case Token.WHILE:
2999                                                 case Token.USING:
3000                                                 case Token.DEFAULT:
3001                                                 case Token.DELEGATE:
3002                                                 case Token.OP_GENERICS_GT:
3003                                                         return Token.OPEN_PARENS;
3004                                                 }
3005
3006                                                 // Optimize using peek
3007                                                 int xx = peek_char ();
3008                                                 switch (xx) {
3009                                                 case '(':
3010                                                 case '\'':
3011                                                 case '"':
3012                                                 case '0':
3013                                                 case '1':
3014                                                         return Token.OPEN_PARENS;
3015                                                 }
3016
3017                                                 lambda_arguments_parsing = true;
3018                                                 PushPosition ();
3019                                                 d = TokenizeOpenParens ();
3020                                                 PopPosition ();
3021                                                 lambda_arguments_parsing = false;
3022                                                 return d;
3023                                         }
3024
3025                                         return Token.OPEN_PARENS;
3026                                 case ')':
3027                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3028                                         return Token.CLOSE_PARENS;
3029                                 case ',':
3030                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3031                                         return Token.COMMA;
3032                                 case ';':
3033                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3034                                         return Token.SEMICOLON;
3035                                 case '~':
3036                                         val = LocatedToken.Create (ref_line, col);
3037                                         return Token.TILDE;
3038                                 case '?':
3039                                         val = LocatedToken.Create (ref_line, col);
3040                                         return TokenizePossibleNullableType ();
3041                                 case '<':
3042                                         val = LocatedToken.Create (ref_line, col);
3043                                         if (parsing_generic_less_than++ > 0)
3044                                                 return Token.OP_GENERICS_LT;
3045
3046                                         return TokenizeLessThan ();
3047
3048                                 case '>':
3049                                         val = LocatedToken.Create (ref_line, col);
3050                                         d = peek_char ();
3051
3052                                         if (d == '='){
3053                                                 get_char ();
3054                                                 return Token.OP_GE;
3055                                         }
3056
3057                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
3058                                                 parsing_generic_less_than--;
3059                                                 return Token.OP_GENERICS_GT;
3060                                         }
3061
3062                                         if (d == '>') {
3063                                                 get_char ();
3064                                                 d = peek_char ();
3065
3066                                                 if (d == '=') {
3067                                                         get_char ();
3068                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
3069                                                 }
3070                                                 return Token.OP_SHIFT_RIGHT;
3071                                         }
3072
3073                                         return Token.OP_GT;
3074
3075                                 case '+':
3076                                         val = LocatedToken.Create (ref_line, col);
3077                                         d = peek_char ();
3078                                         if (d == '+') {
3079                                                 d = Token.OP_INC;
3080                                         } else if (d == '=') {
3081                                                 d = Token.OP_ADD_ASSIGN;
3082                                         } else {
3083                                                 return Token.PLUS;
3084                                         }
3085                                         get_char ();
3086                                         return d;
3087
3088                                 case '-':
3089                                         val = LocatedToken.Create (ref_line, col);
3090                                         d = peek_char ();
3091                                         if (d == '-') {
3092                                                 d = Token.OP_DEC;
3093                                         } else if (d == '=')
3094                                                 d = Token.OP_SUB_ASSIGN;
3095                                         else if (d == '>')
3096                                                 d = Token.OP_PTR;
3097                                         else {
3098                                                 return Token.MINUS;
3099                                         }
3100                                         get_char ();
3101                                         return d;
3102
3103                                 case '!':
3104                                         val = LocatedToken.Create (ref_line, col);
3105                                         if (peek_char () == '='){
3106                                                 get_char ();
3107                                                 return Token.OP_NE;
3108                                         }
3109                                         return Token.BANG;
3110
3111                                 case '=':
3112                                         val = LocatedToken.Create (ref_line, col);
3113                                         d = peek_char ();
3114                                         if (d == '='){
3115                                                 get_char ();
3116                                                 return Token.OP_EQ;
3117                                         }
3118                                         if (d == '>'){
3119                                                 get_char ();
3120                                                 return Token.ARROW;
3121                                         }
3122
3123                                         return Token.ASSIGN;
3124
3125                                 case '&':
3126                                         val = LocatedToken.Create (ref_line, col);
3127                                         d = peek_char ();
3128                                         if (d == '&'){
3129                                                 get_char ();
3130                                                 return Token.OP_AND;
3131                                         }
3132                                         if (d == '='){
3133                                                 get_char ();
3134                                                 return Token.OP_AND_ASSIGN;
3135                                         }
3136                                         return Token.BITWISE_AND;
3137
3138                                 case '|':
3139                                         val = LocatedToken.Create (ref_line, col);
3140                                         d = peek_char ();
3141                                         if (d == '|'){
3142                                                 get_char ();
3143                                                 return Token.OP_OR;
3144                                         }
3145                                         if (d == '='){
3146                                                 get_char ();
3147                                                 return Token.OP_OR_ASSIGN;
3148                                         }
3149                                         return Token.BITWISE_OR;
3150
3151                                 case '*':
3152                                         val = LocatedToken.Create (ref_line, col);
3153                                         if (peek_char () == '='){
3154                                                 get_char ();
3155                                                 return Token.OP_MULT_ASSIGN;
3156                                         }
3157                                         return Token.STAR;
3158
3159                                 case '/':
3160                                         d = peek_char ();
3161                                         if (d == '='){
3162                                                 val = LocatedToken.Create (ref_line, col);
3163                                                 get_char ();
3164                                                 return Token.OP_DIV_ASSIGN;
3165                                         }
3166
3167                                         // Handle double-slash comments.
3168                                         if (d == '/'){
3169                                                 get_char ();
3170                                                 if (doc_processing) {
3171                                                         if (peek_char () == '/') {
3172                                                                 get_char ();
3173                                                                 // Don't allow ////.
3174                                                                 if ((d = peek_char ()) != '/') {
3175                                                                         if (doc_state == XmlCommentState.Allowed)
3176                                                                                 handle_one_line_xml_comment ();
3177                                                                         else if (doc_state == XmlCommentState.NotAllowed)
3178                                                                                 WarningMisplacedComment (Location - 3);
3179                                                                 }
3180                                                         } else {
3181                                                                 if (xml_comment_buffer.Length > 0)
3182                                                                         doc_state = XmlCommentState.NotAllowed;
3183                                                         }
3184                                                 }
3185
3186                                                 while ((d = get_char ()) != -1 && d != '\n');
3187
3188                                                 any_token_seen |= tokens_seen;
3189                                                 tokens_seen = false;
3190                                                 comments_seen = false;
3191                                                 continue;
3192                                         } else if (d == '*'){
3193                                                 get_char ();
3194                                                 bool docAppend = false;
3195                                                 if (doc_processing && peek_char () == '*') {
3196                                                         get_char ();
3197                                                         // But when it is /**/, just do nothing.
3198                                                         if (peek_char () == '/') {
3199                                                                 get_char ();
3200                                                                 continue;
3201                                                         }
3202                                                         if (doc_state == XmlCommentState.Allowed)
3203                                                                 docAppend = true;
3204                                                         else if (doc_state == XmlCommentState.NotAllowed) {
3205                                                                 WarningMisplacedComment (Location - 2);
3206                                                         }
3207                                                 }
3208
3209                                                 int current_comment_start = 0;
3210                                                 if (docAppend) {
3211                                                         current_comment_start = xml_comment_buffer.Length;
3212                                                         xml_comment_buffer.Append (Environment.NewLine);
3213                                                 }
3214
3215                                                 while ((d = get_char ()) != -1){
3216                                                         if (d == '*' && peek_char () == '/'){
3217                                                                 get_char ();
3218                                                                 comments_seen = true;
3219                                                                 break;
3220                                                         }
3221                                                         if (docAppend)
3222                                                                 xml_comment_buffer.Append ((char) d);
3223
3224                                                         if (d == '\n'){
3225                                                                 any_token_seen |= tokens_seen;
3226                                                                 tokens_seen = false;
3227                                                                 //
3228                                                                 // Reset 'comments_seen' just to be consistent.
3229                                                                 // It doesn't matter either way, here.
3230                                                                 //
3231                                                                 comments_seen = false;
3232                                                         }
3233                                                 }
3234                                                 if (!comments_seen)
3235                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3236
3237                                                 if (docAppend)
3238                                                         update_formatted_doc_comment (current_comment_start);
3239                                                 continue;
3240                                         }
3241                                         val = LocatedToken.Create (ref_line, col);
3242                                         return Token.DIV;
3243
3244                                 case '%':
3245                                         val = LocatedToken.Create (ref_line, col);
3246                                         if (peek_char () == '='){
3247                                                 get_char ();
3248                                                 return Token.OP_MOD_ASSIGN;
3249                                         }
3250                                         return Token.PERCENT;
3251
3252                                 case '^':
3253                                         val = LocatedToken.Create (ref_line, col);
3254                                         if (peek_char () == '='){
3255                                                 get_char ();
3256                                                 return Token.OP_XOR_ASSIGN;
3257                                         }
3258                                         return Token.CARRET;
3259
3260                                 case ':':
3261                                         val = LocatedToken.Create (ref_line, col);
3262                                         if (peek_char () == ':') {
3263                                                 get_char ();
3264                                                 return Token.DOUBLE_COLON;
3265                                         }
3266                                         return Token.COLON;
3267
3268                                 case '0': case '1': case '2': case '3': case '4':
3269                                 case '5': case '6': case '7': case '8': case '9':
3270                                         tokens_seen = true;
3271                                         return is_number (c);
3272
3273                                 case '\n': // white space
3274                                         any_token_seen |= tokens_seen;
3275                                         tokens_seen = false;
3276                                         comments_seen = false;
3277                                         continue;
3278
3279                                 case '.':
3280                                         tokens_seen = true;
3281                                         d = peek_char ();
3282                                         if (d >= '0' && d <= '9')
3283                                                 return is_number (c);
3284
3285                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3286                                         return Token.DOT;
3287
3288                                 case '#':
3289                                         if (tokens_seen || comments_seen) {
3290                                                 Eror_WrongPreprocessorLocation ();
3291                                                 return Token.ERROR;
3292                                         }
3293
3294                                         if (ParsePreprocessingDirective (true))
3295                                                 continue;
3296
3297                                         bool directive_expected = false;
3298                                         while ((c = get_char ()) != -1) {
3299                                                 if (col == 1) {
3300                                                         directive_expected = true;
3301                                                 } else if (!directive_expected) {
3302                                                         // TODO: Implement comment support for disabled code and uncomment this code
3303 //                                                      if (c == '#') {
3304 //                                                              Eror_WrongPreprocessorLocation ();
3305 //                                                              return Token.ERROR;
3306 //                                                      }
3307                                                         continue;
3308                                                 }
3309
3310                                                 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' )
3311                                                         continue;
3312
3313                                                 if (c == '#') {
3314                                                         if (ParsePreprocessingDirective (false))
3315                                                                 break;
3316                                                 }
3317                                                 directive_expected = false;
3318                                         }
3319
3320                                         if (c != -1) {
3321                                                 tokens_seen = false;
3322                                                 continue;
3323                                         }
3324
3325                                         return Token.EOF;
3326
3327                                 case '"':
3328                                         return consume_string (false);
3329
3330                                 case '\'':
3331                                         return TokenizeBackslash ();
3332
3333                                 case '@':
3334                                         c = get_char ();
3335                                         if (c == '"') {
3336                                                 tokens_seen = true;
3337                                                 return consume_string (true);
3338                                         }
3339
3340                                         if (is_identifier_start_character (c)){
3341                                                 return consume_identifier (c, true);
3342                                         }
3343
3344                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3345                                         return Token.ERROR;
3346
3347                                 case EvalStatementParserCharacter:
3348                                         return Token.EVAL_STATEMENT_PARSER;
3349                                 case EvalCompilationUnitParserCharacter:
3350                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3351                                 case EvalUsingDeclarationsParserCharacter:
3352                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3353                                 case DocumentationXref:
3354                                         return Token.DOC_SEE;
3355                                 }
3356
3357                                 if (is_identifier_start_character (c)) {
3358                                         tokens_seen = true;
3359                                         return consume_identifier (c);
3360                                 }
3361
3362                                 if (char.IsWhiteSpace ((char) c))
3363                                         continue;
3364
3365                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3366                         }
3367
3368                         if (CompleteOnEOF){
3369                                 if (generated)
3370                                         return Token.COMPLETE_COMPLETION;
3371
3372                                 generated = true;
3373                                 return Token.GENERATE_COMPLETION;
3374                         }
3375
3376
3377                         return Token.EOF;
3378                 }
3379
3380                 int TokenizeBackslash ()
3381                 {
3382 #if FULL_AST
3383                         int read_start = reader.Position;
3384 #endif
3385                         Location start_location = Location;
3386                         int c = get_char ();
3387                         tokens_seen = true;
3388                         if (c == '\'') {
3389                                 val = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3390                                 Report.Error (1011, start_location, "Empty character literal");
3391                                 return Token.LITERAL;
3392                         }
3393
3394                         if (c == '\n') {
3395                                 Report.Error (1010, start_location, "Newline in constant");
3396                                 return Token.ERROR;
3397                         }
3398
3399                         int d;
3400                         c = escape (c, out d);
3401                         if (c == -1)
3402                                 return Token.ERROR;
3403                         if (d != 0)
3404                                 throw new NotImplementedException ();
3405
3406                         ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3407                         val = res;
3408                         c = get_char ();
3409
3410                         if (c != '\'') {
3411                                 Report.Error (1012, start_location, "Too many characters in character literal");
3412
3413                                 // Try to recover, read until newline or next "'"
3414                                 while ((c = get_char ()) != -1) {
3415                                         if (c == '\n' || c == '\'')
3416                                                 break;
3417                                 }
3418                         }
3419
3420 #if FULL_AST
3421                         res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position);
3422 #endif
3423
3424                         return Token.LITERAL;
3425                 }
3426
3427                 int TokenizeLessThan ()
3428                 {
3429                         int d;
3430                         if (handle_typeof) {
3431                                 PushPosition ();
3432                                 if (parse_generic_dimension (out d)) {
3433                                         val = d;
3434                                         DiscardPosition ();
3435                                         return Token.GENERIC_DIMENSION;
3436                                 }
3437                                 PopPosition ();
3438                         }
3439
3440                         // Save current position and parse next token.
3441                         PushPosition ();
3442                         if (parse_less_than ()) {
3443                                 if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) {
3444                                         d = Token.OP_GENERICS_LT_DECL;
3445                                 } else {
3446                                         d = Token.OP_GENERICS_LT;
3447                                 }
3448                                 PopPosition ();
3449                                 return d;
3450                         }
3451
3452                         PopPosition ();
3453                         parsing_generic_less_than = 0;
3454
3455                         d = peek_char ();
3456                         if (d == '<') {
3457                                 get_char ();
3458                                 d = peek_char ();
3459
3460                                 if (d == '=') {
3461                                         get_char ();
3462                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3463                                 }
3464                                 return Token.OP_SHIFT_LEFT;
3465                         }
3466
3467                         if (d == '=') {
3468                                 get_char ();
3469                                 return Token.OP_LE;
3470                         }
3471                         return Token.OP_LT;
3472                 }
3473
3474                 //
3475                 // Handles one line xml comment
3476                 //
3477                 private void handle_one_line_xml_comment ()
3478                 {
3479                         int c;
3480                         while ((c = peek_char ()) == ' ')
3481                                 get_char (); // skip heading whitespaces.
3482                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3483                                 xml_comment_buffer.Append ((char) get_char ());
3484                         }
3485                         if (c == '\r' || c == '\n')
3486                                 xml_comment_buffer.Append (Environment.NewLine);
3487                 }
3488
3489                 //
3490                 // Remove heading "*" in Javadoc-like xml documentation.
3491                 //
3492                 private void update_formatted_doc_comment (int current_comment_start)
3493                 {
3494                         int length = xml_comment_buffer.Length - current_comment_start;
3495                         string [] lines = xml_comment_buffer.ToString (
3496                                 current_comment_start,
3497                                 length).Replace ("\r", "").Split ('\n');
3498
3499                         // The first line starts with /**, thus it is not target
3500                         // for the format check.
3501                         for (int i = 1; i < lines.Length; i++) {
3502                                 string s = lines [i];
3503                                 int idx = s.IndexOf ('*');
3504                                 string head = null;
3505                                 if (idx < 0) {
3506                                         if (i < lines.Length - 1)
3507                                                 return;
3508                                         head = s;
3509                                 } else
3510                                         head = s.Substring (0, idx);
3511                                 foreach (char c in head)
3512                                         if (c != ' ')
3513                                                 return;
3514                                 lines [i] = s.Substring (idx + 1);
3515                         }
3516                         xml_comment_buffer.Remove (current_comment_start, length);
3517                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3518                 }
3519
3520                 //
3521                 // Checks if there was incorrect doc comments and raise
3522                 // warnings.
3523                 //
3524                 public void check_incorrect_doc_comment ()
3525                 {
3526                         if (xml_comment_buffer.Length > 0)
3527                                 WarningMisplacedComment (Location);
3528                 }
3529
3530                 //
3531                 // Consumes the saved xml comment lines (if any)
3532                 // as for current target member or type.
3533                 //
3534                 public string consume_doc_comment ()
3535                 {
3536                         if (xml_comment_buffer.Length > 0) {
3537                                 string ret = xml_comment_buffer.ToString ();
3538                                 reset_doc_comment ();
3539                                 return ret;
3540                         }
3541                         return null;
3542                 }
3543
3544                 Report Report {
3545                         get { return context.Report; }
3546                 }
3547
3548                 void reset_doc_comment ()
3549                 {
3550                         xml_comment_buffer.Length = 0;
3551                 }
3552
3553                 public void cleanup ()
3554                 {
3555                         if (ifstack != null && ifstack.Count >= 1) {
3556                                 int state = ifstack.Pop ();
3557                                 if ((state & REGION) != 0)
3558                                         Report.Error (1038, Location, "#endregion directive expected");
3559                                 else
3560                                         Report.Error (1027, Location, "Expected `#endif' directive");
3561                         }
3562                 }
3563         }
3564
3565         //
3566         // Indicates whether it accepts XML documentation or not.
3567         //
3568         public enum XmlCommentState {
3569                 // comment is allowed in this state.
3570                 Allowed,
3571                 // comment is not allowed in this state.
3572                 NotAllowed,
3573                 // once comments appeared when it is NotAllowed, then the
3574                 // state is changed to it, until the state is changed to
3575                 // .Allowed.
3576                 Error
3577         }
3578 }
3579