mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@gmail.com)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 // Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20
  21 namespace Mono.CSharp
  22 {
  23         /// <summary>
  24         ///    Tokenizer for C# source code.
  25         /// </summary>
  26
  27         public class Tokenizer : yyParser.yyInput
  28         {
  29                 class KeywordEntry<T>
  30                 {
  31                         public readonly T Token;
  32                         public KeywordEntry<T> Next;
  33                         public readonly char[] Value;
  34
  35                         public KeywordEntry (string value, T token)
  36                         {
  37                                 this.Value = value.ToCharArray ();
  38                                 this.Token = token;
  39                         }
  40                 }
  41
  42                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  43                 {
  44                         readonly int length;
  45
  46                         public IdentifiersComparer (int length)
  47                         {
  48                                 this.length = length;
  49                         }
  50
  51                         public bool Equals (char[] x, char[] y)
  52                         {
  53                                 for (int i = 0; i < length; ++i)
  54                                         if (x [i] != y [i])
  55                                                 return false;
  56
  57                                 return true;
  58                         }
  59
  60                         public int GetHashCode (char[] obj)
  61                         {
  62                                 int h = 0;
  63                                 for (int i = 0; i < length; ++i)
  64                                         h = (h << 5) - h + obj [i];
  65
  66                                 return h;
  67                         }
  68                 }
  69
  70                 //
  71                 // This class has to be used in the parser only, it reuses token
  72                 // details after each parse
  73                 //
  74                 public class LocatedToken
  75                 {
  76                         int row, column;
  77                         string value;
  78
  79                         static LocatedToken[] buffer = new LocatedToken[0];
  80                         static int pos;
  81
  82                         private LocatedToken ()
  83                         {
  84                         }
  85
  86                         public static LocatedToken Create (int row, int column)
  87                         {
  88                                 return Create (null, row, column);
  89                         }
  90
  91                         public static LocatedToken Create (string value, Location loc)
  92                         {
  93                                 return Create (value, loc.Row, loc.Column);
  94                         }
  95
  96                         public static LocatedToken Create (string value, int row, int column)
  97                         {
  98                                 //
  99                                 // TODO: I am not very happy about the logic but it's the best
 100                                 // what I could come up with for now.
 101                                 // Ideally we should be using just tiny buffer (256 elements) which
 102                                 // is enough to hold all details for currect stack and recycle elements
 103                                 // poped from the stack but there is a trick needed to recycle
 104                                 // them properly.
 105                                 //
 106                                 LocatedToken entry;
 107                                 if (pos >= buffer.Length) {
 108                                         entry = new LocatedToken ();
 109                                 } else {
 110                                         entry = buffer [pos];
 111                                         if (entry == null) {
 112                                                 entry = new LocatedToken ();
 113                                                 buffer [pos] = entry;
 114                                         }
 115
 116                                         ++pos;
 117                                 }
 118                                 entry.value = value;
 119                                 entry.row = row;
 120                                 entry.column = column;
 121                                 return entry;
 122                         }
 123
 124                         //
 125                         // Used for token not required by expression evaluator
 126                         //
 127                         [Conditional ("FULL_AST")]
 128                         public static void CreateOptional (int row, int col, ref object token)
 129                         {
 130                                 token = Create (row, col);
 131                         }
 132
 133                         public static void Initialize ()
 134                         {
 135 #if !FULL_AST
 136                                 if (buffer.Length == 0)
 137                                         buffer = new LocatedToken [15000];
 138 #endif
 139                                 pos = 0;
 140                         }
 141
 142                         public Location Location {
 143                                 get { return new Location (row, column); }
 144                         }
 145
 146                         public string Value {
 147                                 get { return value; }
 148                         }
 149                 }
 150
 151                 public enum PreprocessorDirective
 152                 {
 153                         Invalid = 0,
 154
 155                         Region = 1,
 156                         Endregion = 2,
 157                         If = 3 | RequiresArgument,
 158                         Endif = 4,
 159                         Elif = 5 | RequiresArgument,
 160                         Else = 6,
 161                         Define = 7 | RequiresArgument,
 162                         Undef = 8 | RequiresArgument,
 163                         Error = 9,
 164                         Warning = 10,
 165                         Pragma = 11 | CustomArgumentsParsing,
 166                         Line = 12,
 167
 168                         CustomArgumentsParsing = 1 << 10,
 169                         RequiresArgument = 1 << 11
 170                 }
 171
 172                 readonly SeekableStreamReader reader;
 173                 readonly CompilationSourceFile source_file;
 174                 readonly CompilerContext context;
 175
 176                 SourceFile current_source;
 177                 bool hidden = false;
 178                 int ref_line = 1;
 179                 int line = 1;
 180                 int col = 0;
 181                 int previous_col;
 182                 int current_token;
 183                 readonly int tab_size;
 184                 bool handle_get_set = false;
 185                 bool handle_remove_add = false;
 186                 bool handle_where = false;
 187                 bool handle_typeof = false;
 188                 bool lambda_arguments_parsing;
 189                 List<Location> escaped_identifiers;
 190                 int parsing_generic_less_than;
 191                 readonly bool doc_processing;
 192
 193                 //
 194                 // Used mainly for parser optimizations. Some expressions for instance
 195                 // can appear only in block (including initializer, base initializer)
 196                 // scope only
 197                 //
 198                 public int parsing_block;
 199                 internal bool query_parsing;
 200
 201                 //
 202                 // When parsing type only, useful for ambiguous nullable types
 203                 //
 204                 public int parsing_type;
 205
 206                 //
 207                 // Set when parsing generic declaration (type or method header)
 208                 //
 209                 public bool parsing_generic_declaration;
 210                 public bool parsing_generic_declaration_doc;
 211
 212                 //
 213                 // The value indicates that we have not reach any declaration or
 214                 // namespace yet
 215                 //
 216                 public int parsing_declaration;
 217
 218                 public bool parsing_attribute_section;
 219
 220                 public bool parsing_modifiers;
 221
 222                 //
 223                 // The special characters to inject on streams to run the unit parser
 224                 // in the special expression mode. Using private characters from
 225                 // Plane Sixteen (U+100000 to U+10FFFD)
 226                 //
 227                 // This character is only tested just before the tokenizer is about to report
 228                 // an error;   So on the regular operation mode, this addition will have no
 229                 // impact on the tokenizer's performance.
 230                 //
 231
 232                 public const int EvalStatementParserCharacter = 0x100000;
 233                 public const int EvalCompilationUnitParserCharacter = 0x100001;
 234                 public const int EvalUsingDeclarationsParserCharacter = 0x100002;
 235                 public const int DocumentationXref = 0x100003;
 236
 237                 //
 238                 // XML documentation buffer. The save point is used to divide
 239                 // comments on types and comments on members.
 240                 //
 241                 StringBuilder xml_comment_buffer;
 242
 243                 //
 244                 // See comment on XmlCommentState enumeration.
 245                 //
 246                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 247
 248                 //
 249                 // Whether tokens have been seen on this line
 250                 //
 251                 bool tokens_seen = false;
 252
 253                 //
 254                 // Set to true once the GENERATE_COMPLETION token has bee
 255                 // returned.   This helps produce one GENERATE_COMPLETION,
 256                 // as many COMPLETE_COMPLETION as necessary to complete the
 257                 // AST tree and one final EOF.
 258                 //
 259                 bool generated;
 260
 261                 //
 262                 // Whether a token has been seen on the file
 263                 // This is needed because `define' is not allowed to be used
 264                 // after a token has been seen.
 265                 //
 266                 bool any_token_seen;
 267
 268                 //
 269                 // Class variables
 270                 //
 271                 static readonly KeywordEntry<int>[][] keywords;
 272                 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 273                 static readonly HashSet<string> keyword_strings;
 274                 static readonly NumberStyles styles;
 275                 static readonly NumberFormatInfo csharp_format_info;
 276
 277                 // Pragma arguments
 278                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 279                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 280                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 281                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 282
 283                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 284
 285                 public bool PropertyParsing {
 286                         get { return handle_get_set; }
 287                         set { handle_get_set = value; }
 288                 }
 289
 290                 public bool EventParsing {
 291                         get { return handle_remove_add; }
 292                         set { handle_remove_add = value; }
 293                 }
 294
 295                 public bool ConstraintsParsing {
 296                         get { return handle_where; }
 297                         set { handle_where = value; }
 298                 }
 299
 300                 public bool TypeOfParsing {
 301                         get { return handle_typeof; }
 302                         set { handle_typeof = value; }
 303                 }
 304
 305                 public XmlCommentState doc_state {
 306                         get { return xml_doc_state; }
 307                         set {
 308                                 if (value == XmlCommentState.Allowed) {
 309                                         check_incorrect_doc_comment ();
 310                                         reset_doc_comment ();
 311                                 }
 312                                 xml_doc_state = value;
 313                         }
 314                 }
 315
 316                 //
 317                 // This is used to trigger completion generation on the parser
 318                 public bool CompleteOnEOF;
 319
 320                 void AddEscapedIdentifier (Location loc)
 321                 {
 322                         if (escaped_identifiers == null)
 323                                 escaped_identifiers = new List<Location> ();
 324
 325                         escaped_identifiers.Add (loc);
 326                 }
 327
 328                 public bool IsEscapedIdentifier (ATypeNameExpression name)
 329                 {
 330                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 331                 }
 332
 333                 //
 334                 // Values for the associated token returned
 335                 //
 336                 internal int putback_char;      // Used by repl only
 337                 object val;
 338
 339                 //
 340                 // Pre-processor
 341                 //
 342                 const int TAKING        = 1;
 343                 const int ELSE_SEEN     = 4;
 344                 const int PARENT_TAKING = 8;
 345                 const int REGION        = 16;
 346
 347                 //
 348                 // pre-processor if stack state:
 349                 //
 350                 Stack<int> ifstack;
 351
 352                 const int max_id_size = 512;
 353                 const int max_number_size = 512;
 354
 355 #if FULL_AST
 356                 readonly char [] id_builder = new char [max_id_size];
 357
 358                 Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 359
 360                 char [] number_builder = new char [max_number_size];
 361                 int number_pos;
 362
 363                 char[] value_builder = new char[256];
 364 #else
 365                 static readonly char [] id_builder = new char [max_id_size];
 366
 367                 static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 368
 369                 static char [] number_builder = new char [max_number_size];
 370                 static int number_pos;
 371
 372                 static char[] value_builder = new char[256];
 373 #endif
 374
 375                 public int Line {
 376                         get {
 377                                 return ref_line;
 378                         }
 379                 }
 380
 381                 //
 382                 // This is used when the tokenizer needs to save
 383                 // the current position as it needs to do some parsing
 384                 // on its own to deamiguate a token in behalf of the
 385                 // parser.
 386                 //
 387                 Stack<Position> position_stack = new Stack<Position> (2);
 388
 389                 class Position {
 390                         public int position;
 391                         public int line;
 392                         public int ref_line;
 393                         public int col;
 394                         public bool hidden;
 395                         public int putback_char;
 396                         public int previous_col;
 397                         public Stack<int> ifstack;
 398                         public int parsing_generic_less_than;
 399                         public int current_token;
 400                         public object val;
 401
 402                         public Position (Tokenizer t)
 403                         {
 404                                 position = t.reader.Position;
 405                                 line = t.line;
 406                                 ref_line = t.ref_line;
 407                                 col = t.col;
 408                                 hidden = t.hidden;
 409                                 putback_char = t.putback_char;
 410                                 previous_col = t.previous_col;
 411                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 412                                         // There is no simple way to clone Stack<T> all
 413                                         // methods reverse the order
 414                                         var clone = t.ifstack.ToArray ();
 415                                         Array.Reverse (clone);
 416                                         ifstack = new Stack<int> (clone);
 417                                 }
 418                                 parsing_generic_less_than = t.parsing_generic_less_than;
 419                                 current_token = t.current_token;
 420                                 val = t.val;
 421                         }
 422                 }
 423
 424                 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file)
 425                 {
 426                         this.source_file = file;
 427                         this.context = file.Compiler;
 428                         this.current_source = file.SourceFile;
 429
 430                         reader = input;
 431
 432                         putback_char = -1;
 433
 434                         xml_comment_buffer = new StringBuilder ();
 435                         doc_processing = context.Settings.DocumentationFile != null;
 436
 437                         tab_size = context.Settings.TabSize;
 438
 439                         Mono.CSharp.Location.Push (current_source);
 440                 }
 441
 442                 public void PushPosition ()
 443                 {
 444                         position_stack.Push (new Position (this));
 445                 }
 446
 447                 public void PopPosition ()
 448                 {
 449                         Position p = position_stack.Pop ();
 450
 451                         reader.Position = p.position;
 452                         ref_line = p.ref_line;
 453                         line = p.line;
 454                         col = p.col;
 455                         hidden = p.hidden;
 456                         putback_char = p.putback_char;
 457                         previous_col = p.previous_col;
 458                         ifstack = p.ifstack;
 459                         parsing_generic_less_than = p.parsing_generic_less_than;
 460                         current_token = p.current_token;
 461                         val = p.val;
 462                 }
 463
 464                 // Do not reset the position, ignore it.
 465                 public void DiscardPosition ()
 466                 {
 467                         position_stack.Pop ();
 468                 }
 469
 470                 static void AddKeyword (string kw, int token)
 471                 {
 472                         keyword_strings.Add (kw);
 473
 474                         AddKeyword (keywords, kw, token);
 475                 }
 476
 477                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 478                 {
 479                         AddKeyword (keywords_preprocessor, kw, directive);
 480                 }
 481
 482                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 483                 {
 484                         int length = kw.Length;
 485                         if (keywords[length] == null) {
 486                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 487                         }
 488
 489                         int char_index = kw[0] - '_';
 490                         var kwe = keywords[length][char_index];
 491                         if (kwe == null) {
 492                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 493                                 return;
 494                         }
 495
 496                         while (kwe.Next != null) {
 497                                 kwe = kwe.Next;
 498                         }
 499
 500                         kwe.Next = new KeywordEntry<T> (kw, token);
 501                 }
 502
 503                 //
 504                 // Class initializer
 505                 //
 506                 static Tokenizer ()
 507                 {
 508                         keyword_strings = new HashSet<string> ();
 509
 510                         // 11 is the length of the longest keyword for now
 511                         keywords = new KeywordEntry<int>[11][];
 512
 513                         AddKeyword ("__arglist", Token.ARGLIST);
 514                         AddKeyword ("__makeref", Token.MAKEREF);
 515                         AddKeyword ("__reftype", Token.REFTYPE);
 516                         AddKeyword ("__refvalue", Token.REFVALUE);
 517                         AddKeyword ("abstract", Token.ABSTRACT);
 518                         AddKeyword ("as", Token.AS);
 519                         AddKeyword ("add", Token.ADD);
 520                         AddKeyword ("base", Token.BASE);
 521                         AddKeyword ("bool", Token.BOOL);
 522                         AddKeyword ("break", Token.BREAK);
 523                         AddKeyword ("byte", Token.BYTE);
 524                         AddKeyword ("case", Token.CASE);
 525                         AddKeyword ("catch", Token.CATCH);
 526                         AddKeyword ("char", Token.CHAR);
 527                         AddKeyword ("checked", Token.CHECKED);
 528                         AddKeyword ("class", Token.CLASS);
 529                         AddKeyword ("const", Token.CONST);
 530                         AddKeyword ("continue", Token.CONTINUE);
 531                         AddKeyword ("decimal", Token.DECIMAL);
 532                         AddKeyword ("default", Token.DEFAULT);
 533                         AddKeyword ("delegate", Token.DELEGATE);
 534                         AddKeyword ("do", Token.DO);
 535                         AddKeyword ("double", Token.DOUBLE);
 536                         AddKeyword ("else", Token.ELSE);
 537                         AddKeyword ("enum", Token.ENUM);
 538                         AddKeyword ("event", Token.EVENT);
 539                         AddKeyword ("explicit", Token.EXPLICIT);
 540                         AddKeyword ("extern", Token.EXTERN);
 541                         AddKeyword ("false", Token.FALSE);
 542                         AddKeyword ("finally", Token.FINALLY);
 543                         AddKeyword ("fixed", Token.FIXED);
 544                         AddKeyword ("float", Token.FLOAT);
 545                         AddKeyword ("for", Token.FOR);
 546                         AddKeyword ("foreach", Token.FOREACH);
 547                         AddKeyword ("goto", Token.GOTO);
 548                         AddKeyword ("get", Token.GET);
 549                         AddKeyword ("if", Token.IF);
 550                         AddKeyword ("implicit", Token.IMPLICIT);
 551                         AddKeyword ("in", Token.IN);
 552                         AddKeyword ("int", Token.INT);
 553                         AddKeyword ("interface", Token.INTERFACE);
 554                         AddKeyword ("internal", Token.INTERNAL);
 555                         AddKeyword ("is", Token.IS);
 556                         AddKeyword ("lock", Token.LOCK);
 557                         AddKeyword ("long", Token.LONG);
 558                         AddKeyword ("namespace", Token.NAMESPACE);
 559                         AddKeyword ("new", Token.NEW);
 560                         AddKeyword ("null", Token.NULL);
 561                         AddKeyword ("object", Token.OBJECT);
 562                         AddKeyword ("operator", Token.OPERATOR);
 563                         AddKeyword ("out", Token.OUT);
 564                         AddKeyword ("override", Token.OVERRIDE);
 565                         AddKeyword ("params", Token.PARAMS);
 566                         AddKeyword ("private", Token.PRIVATE);
 567                         AddKeyword ("protected", Token.PROTECTED);
 568                         AddKeyword ("public", Token.PUBLIC);
 569                         AddKeyword ("readonly", Token.READONLY);
 570                         AddKeyword ("ref", Token.REF);
 571                         AddKeyword ("remove", Token.REMOVE);
 572                         AddKeyword ("return", Token.RETURN);
 573                         AddKeyword ("sbyte", Token.SBYTE);
 574                         AddKeyword ("sealed", Token.SEALED);
 575                         AddKeyword ("set", Token.SET);
 576                         AddKeyword ("short", Token.SHORT);
 577                         AddKeyword ("sizeof", Token.SIZEOF);
 578                         AddKeyword ("stackalloc", Token.STACKALLOC);
 579                         AddKeyword ("static", Token.STATIC);
 580                         AddKeyword ("string", Token.STRING);
 581                         AddKeyword ("struct", Token.STRUCT);
 582                         AddKeyword ("switch", Token.SWITCH);
 583                         AddKeyword ("this", Token.THIS);
 584                         AddKeyword ("throw", Token.THROW);
 585                         AddKeyword ("true", Token.TRUE);
 586                         AddKeyword ("try", Token.TRY);
 587                         AddKeyword ("typeof", Token.TYPEOF);
 588                         AddKeyword ("uint", Token.UINT);
 589                         AddKeyword ("ulong", Token.ULONG);
 590                         AddKeyword ("unchecked", Token.UNCHECKED);
 591                         AddKeyword ("unsafe", Token.UNSAFE);
 592                         AddKeyword ("ushort", Token.USHORT);
 593                         AddKeyword ("using", Token.USING);
 594                         AddKeyword ("virtual", Token.VIRTUAL);
 595                         AddKeyword ("void", Token.VOID);
 596                         AddKeyword ("volatile", Token.VOLATILE);
 597                         AddKeyword ("while", Token.WHILE);
 598                         AddKeyword ("partial", Token.PARTIAL);
 599                         AddKeyword ("where", Token.WHERE);
 600
 601                         // LINQ keywords
 602                         AddKeyword ("from", Token.FROM);
 603                         AddKeyword ("join", Token.JOIN);
 604                         AddKeyword ("on", Token.ON);
 605                         AddKeyword ("equals", Token.EQUALS);
 606                         AddKeyword ("select", Token.SELECT);
 607                         AddKeyword ("group", Token.GROUP);
 608                         AddKeyword ("by", Token.BY);
 609                         AddKeyword ("let", Token.LET);
 610                         AddKeyword ("orderby", Token.ORDERBY);
 611                         AddKeyword ("ascending", Token.ASCENDING);
 612                         AddKeyword ("descending", Token.DESCENDING);
 613                         AddKeyword ("into", Token.INTO);
 614
 615                         // Contextual async keywords
 616                         AddKeyword ("async", Token.ASYNC);
 617                         AddKeyword ("await", Token.AWAIT);
 618
 619                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 620
 621                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 622                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 623                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 624                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 625                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 626                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 627                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 628                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 629                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 630                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 631                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 632                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 633
 634                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 635                         styles = NumberStyles.Float;
 636                 }
 637
 638                 int GetKeyword (char[] id, int id_len)
 639                 {
 640                         //
 641                         // Keywords are stored in an array of arrays grouped by their
 642                         // length and then by the first character
 643                         //
 644                         if (id_len >= keywords.Length || keywords [id_len] == null)
 645                                 return -1;
 646
 647                         int first_index = id [0] - '_';
 648                         if (first_index > 'z' - '_')
 649                                 return -1;
 650
 651                         var kwe = keywords [id_len] [first_index];
 652                         if (kwe == null)
 653                                 return -1;
 654
 655                         int res;
 656                         do {
 657                                 res = kwe.Token;
 658                                 for (int i = 1; i < id_len; ++i) {
 659                                         if (id [i] != kwe.Value [i]) {
 660                                                 res = 0;
 661                                                 kwe = kwe.Next;
 662                                                 break;
 663                                         }
 664                                 }
 665                         } while (res == 0 && kwe != null);
 666
 667                         if (res == 0)
 668                                 return -1;
 669
 670                         int next_token;
 671                         switch (res) {
 672                         case Token.GET:
 673                         case Token.SET:
 674                                 if (!handle_get_set)
 675                                         res = -1;
 676                                 break;
 677                         case Token.REMOVE:
 678                         case Token.ADD:
 679                                 if (!handle_remove_add)
 680                                         res = -1;
 681                                 break;
 682                         case Token.EXTERN:
 683                                 if (parsing_declaration == 0)
 684                                         res = Token.EXTERN_ALIAS;
 685                                 break;
 686                         case Token.DEFAULT:
 687                                 if (peek_token () == Token.COLON) {
 688                                         token ();
 689                                         res = Token.DEFAULT_COLON;
 690                                 }
 691                                 break;
 692                         case Token.WHERE:
 693                                 if (!handle_where && !query_parsing)
 694                                         res = -1;
 695                                 break;
 696                         case Token.FROM:
 697                                 //
 698                                 // A query expression is any expression that starts with `from identifier'
 699                                 // followed by any token except ; , =
 700                                 //
 701                                 if (!query_parsing) {
 702                                         if (lambda_arguments_parsing) {
 703                                                 res = -1;
 704                                                 break;
 705                                         }
 706
 707                                         PushPosition ();
 708                                         // HACK: to disable generics micro-parser, because PushPosition does not
 709                                         // store identifiers array
 710                                         parsing_generic_less_than = 1;
 711                                         switch (xtoken ()) {
 712                                         case Token.IDENTIFIER:
 713                                         case Token.INT:
 714                                         case Token.BOOL:
 715                                         case Token.BYTE:
 716                                         case Token.CHAR:
 717                                         case Token.DECIMAL:
 718                                         case Token.FLOAT:
 719                                         case Token.LONG:
 720                                         case Token.OBJECT:
 721                                         case Token.STRING:
 722                                         case Token.UINT:
 723                                         case Token.ULONG:
 724                                                 next_token = xtoken ();
 725                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 726                                                         goto default;
 727
 728                                                 res = Token.FROM_FIRST;
 729                                                 query_parsing = true;
 730                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 731                                                         Report.FeatureIsNotAvailable (context, Location, "query expressions");
 732                                                 break;
 733                                         case Token.VOID:
 734                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 735                                                 break;
 736                                         default:
 737                                                 PopPosition ();
 738                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 739                                                 // which has been overwritten before we grabbed the identifier
 740                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 741                                                 return -1;
 742                                         }
 743                                         PopPosition ();
 744                                 }
 745                                 break;
 746                         case Token.JOIN:
 747                         case Token.ON:
 748                         case Token.EQUALS:
 749                         case Token.SELECT:
 750                         case Token.GROUP:
 751                         case Token.BY:
 752                         case Token.LET:
 753                         case Token.ORDERBY:
 754                         case Token.ASCENDING:
 755                         case Token.DESCENDING:
 756                         case Token.INTO:
 757                                 if (!query_parsing)
 758                                         res = -1;
 759                                 break;
 760
 761                         case Token.USING:
 762                         case Token.NAMESPACE:
 763                                 // TODO: some explanation needed
 764                                 check_incorrect_doc_comment ();
 765                                 break;
 766
 767                         case Token.PARTIAL:
 768                                 if (parsing_block > 0) {
 769                                         res = -1;
 770                                         break;
 771                                 }
 772
 773                                 // Save current position and parse next token.
 774                                 PushPosition ();
 775
 776                                 next_token = token ();
 777                                 bool ok = (next_token == Token.CLASS) ||
 778                                         (next_token == Token.STRUCT) ||
 779                                         (next_token == Token.INTERFACE) ||
 780                                         (next_token == Token.VOID);
 781
 782                                 PopPosition ();
 783
 784                                 if (ok) {
 785                                         if (next_token == Token.VOID) {
 786                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 787                                                         Report.FeatureIsNotAvailable (context, Location, "partial methods");
 788                                         } else if (context.Settings.Version == LanguageVersion.ISO_1)
 789                                                 Report.FeatureIsNotAvailable (context, Location, "partial types");
 790
 791                                         return res;
 792                                 }
 793
 794                                 if (next_token < Token.LAST_KEYWORD) {
 795                                         Report.Error (267, Location,
 796                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 797                                         return token ();
 798                                 }
 799
 800                                 res = -1;
 801                                 break;
 802
 803                         case Token.ASYNC:
 804                                 if (parsing_modifiers) {
 805                                         //
 806                                         // Skip attributes section or constructor called async
 807                                         //
 808                                         if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) {
 809                                                 res = -1;
 810                                         } else {
 811                                                 // async is keyword
 812                                         }
 813                                 } else if (parsing_block > 0) {
 814                                         switch (peek_token ()) {
 815                                         case Token.DELEGATE:
 816                                         case Token.OPEN_PARENS_LAMBDA:
 817                                                 // async is keyword
 818                                                 break;
 819                                         case Token.IDENTIFIER:
 820                                                 PushPosition ();
 821                                                 xtoken ();
 822                                                 if (xtoken () != Token.ARROW)
 823                                                         res = -1;
 824
 825                                                 PopPosition ();
 826                                                 break;
 827                                         default:
 828                                                 res = -1;
 829                                                 break;
 830                                         }
 831                                 } else {
 832                                         res = -1;
 833                                 }
 834
 835                                 if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) {
 836                                         Report.FeatureIsNotAvailable (context, Location, "asynchronous functions");
 837                                 }
 838
 839                                 break;
 840
 841                         case Token.AWAIT:
 842                                 if (parsing_block == 0)
 843                                         res = -1;
 844
 845                                 break;
 846                         }
 847
 848
 849                         return res;
 850                 }
 851
 852                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 853                 {
 854                         //
 855                         // Keywords are stored in an array of arrays grouped by their
 856                         // length and then by the first character
 857                         //
 858                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 859                                 return PreprocessorDirective.Invalid;
 860
 861                         int first_index = id[0] - '_';
 862                         if (first_index > 'z' - '_')
 863                                 return PreprocessorDirective.Invalid;
 864
 865                         var kwe = keywords_preprocessor[id_len][first_index];
 866                         if (kwe == null)
 867                                 return PreprocessorDirective.Invalid;
 868
 869                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 870                         do {
 871                                 res = kwe.Token;
 872                                 for (int i = 1; i < id_len; ++i) {
 873                                         if (id[i] != kwe.Value[i]) {
 874                                                 res = 0;
 875                                                 kwe = kwe.Next;
 876                                                 break;
 877                                         }
 878                                 }
 879                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 880
 881                         return res;
 882                 }
 883
 884                 public Location Location {
 885                         get {
 886                                 return new Location (ref_line, hidden ? -1 : col);
 887                         }
 888                 }
 889
 890                 static bool is_identifier_start_character (int c)
 891                 {
 892                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 893                 }
 894
 895                 static bool is_identifier_part_character (char c)
 896                 {
 897                         if (c >= 'a' && c <= 'z')
 898                                 return true;
 899
 900                         if (c >= 'A' && c <= 'Z')
 901                                 return true;
 902
 903                         if (c == '_' || (c >= '0' && c <= '9'))
 904                                 return true;
 905
 906                         if (c < 0x80)
 907                                 return false;
 908
 909                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 910                 }
 911
 912                 public static bool IsKeyword (string s)
 913                 {
 914                         return keyword_strings.Contains (s);
 915                 }
 916
 917                 //
 918                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 919                 //
 920                 int TokenizeOpenParens ()
 921                 {
 922                         int ptoken;
 923                         current_token = -1;
 924
 925                         int bracket_level = 0;
 926                         bool is_type = false;
 927                         bool can_be_type = false;
 928
 929                         while (true) {
 930                                 ptoken = current_token;
 931                                 token ();
 932
 933                                 switch (current_token) {
 934                                 case Token.CLOSE_PARENS:
 935                                         token ();
 936
 937                                         //
 938                                         // Expression inside parens is lambda, (int i) =>
 939                                         //
 940                                         if (current_token == Token.ARROW)
 941                                                 return Token.OPEN_PARENS_LAMBDA;
 942
 943                                         //
 944                                         // Expression inside parens is single type, (int[])
 945                                         //
 946                                         if (is_type)
 947                                                 return Token.OPEN_PARENS_CAST;
 948
 949                                         //
 950                                         // Expression is possible cast, look at next token, (T)null
 951                                         //
 952                                         if (can_be_type) {
 953                                                 switch (current_token) {
 954                                                 case Token.OPEN_PARENS:
 955                                                 case Token.BANG:
 956                                                 case Token.TILDE:
 957                                                 case Token.IDENTIFIER:
 958                                                 case Token.LITERAL:
 959                                                 case Token.BASE:
 960                                                 case Token.CHECKED:
 961                                                 case Token.DELEGATE:
 962                                                 case Token.FALSE:
 963                                                 case Token.FIXED:
 964                                                 case Token.NEW:
 965                                                 case Token.NULL:
 966                                                 case Token.SIZEOF:
 967                                                 case Token.THIS:
 968                                                 case Token.THROW:
 969                                                 case Token.TRUE:
 970                                                 case Token.TYPEOF:
 971                                                 case Token.UNCHECKED:
 972                                                 case Token.UNSAFE:
 973                                                 case Token.DEFAULT:
 974
 975                                                 //
 976                                                 // These can be part of a member access
 977                                                 //
 978                                                 case Token.INT:
 979                                                 case Token.UINT:
 980                                                 case Token.SHORT:
 981                                                 case Token.USHORT:
 982                                                 case Token.LONG:
 983                                                 case Token.ULONG:
 984                                                 case Token.DOUBLE:
 985                                                 case Token.FLOAT:
 986                                                 case Token.CHAR:
 987                                                 case Token.BYTE:
 988                                                 case Token.DECIMAL:
 989                                                 case Token.BOOL:
 990                                                         return Token.OPEN_PARENS_CAST;
 991                                                 }
 992                                         }
 993                                         return Token.OPEN_PARENS;
 994
 995                                 case Token.DOT:
 996                                 case Token.DOUBLE_COLON:
 997                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
 998                                                 goto default;
 999
1000                                         continue;
1001
1002                                 case Token.IDENTIFIER:
1003                                         switch (ptoken) {
1004                                         case Token.DOT:
1005                                                 if (bracket_level == 0) {
1006                                                         is_type = false;
1007                                                         can_be_type = true;
1008                                                 }
1009
1010                                                 continue;
1011                                         case Token.OP_GENERICS_LT:
1012                                         case Token.COMMA:
1013                                         case Token.DOUBLE_COLON:
1014                                         case -1:
1015                                                 if (bracket_level == 0)
1016                                                         can_be_type = true;
1017                                                 continue;
1018                                         default:
1019                                                 can_be_type = is_type = false;
1020                                                 continue;
1021                                         }
1022
1023                                 case Token.OBJECT:
1024                                 case Token.STRING:
1025                                 case Token.BOOL:
1026                                 case Token.DECIMAL:
1027                                 case Token.FLOAT:
1028                                 case Token.DOUBLE:
1029                                 case Token.SBYTE:
1030                                 case Token.BYTE:
1031                                 case Token.SHORT:
1032                                 case Token.USHORT:
1033                                 case Token.INT:
1034                                 case Token.UINT:
1035                                 case Token.LONG:
1036                                 case Token.ULONG:
1037                                 case Token.CHAR:
1038                                 case Token.VOID:
1039                                         if (bracket_level == 0)
1040                                                 is_type = true;
1041                                         continue;
1042
1043                                 case Token.COMMA:
1044                                         if (bracket_level == 0) {
1045                                                 bracket_level = 100;
1046                                                 can_be_type = is_type = false;
1047                                         }
1048                                         continue;
1049
1050                                 case Token.OP_GENERICS_LT:
1051                                 case Token.OPEN_BRACKET:
1052                                         if (bracket_level++ == 0)
1053                                                 is_type = true;
1054                                         continue;
1055
1056                                 case Token.OP_GENERICS_GT:
1057                                 case Token.CLOSE_BRACKET:
1058                                         --bracket_level;
1059                                         continue;
1060
1061                                 case Token.INTERR_NULLABLE:
1062                                 case Token.STAR:
1063                                         if (bracket_level == 0)
1064                                                 is_type = true;
1065                                         continue;
1066
1067                                 case Token.REF:
1068                                 case Token.OUT:
1069                                         can_be_type = is_type = false;
1070                                         continue;
1071
1072                                 default:
1073                                         return Token.OPEN_PARENS;
1074                                 }
1075                         }
1076                 }
1077
1078                 public static bool IsValidIdentifier (string s)
1079                 {
1080                         if (s == null || s.Length == 0)
1081                                 return false;
1082
1083                         if (!is_identifier_start_character (s [0]))
1084                                 return false;
1085
1086                         for (int i = 1; i < s.Length; i ++)
1087                                 if (! is_identifier_part_character (s [i]))
1088                                         return false;
1089
1090                         return true;
1091                 }
1092
1093                 bool parse_less_than ()
1094                 {
1095                 start:
1096                         int the_token = token ();
1097                         if (the_token == Token.OPEN_BRACKET) {
1098                                 do {
1099                                         the_token = token ();
1100                                 } while (the_token != Token.CLOSE_BRACKET);
1101                                 the_token = token ();
1102                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1103                                 the_token = token ();
1104                         }
1105                         switch (the_token) {
1106                         case Token.IDENTIFIER:
1107                         case Token.OBJECT:
1108                         case Token.STRING:
1109                         case Token.BOOL:
1110                         case Token.DECIMAL:
1111                         case Token.FLOAT:
1112                         case Token.DOUBLE:
1113                         case Token.SBYTE:
1114                         case Token.BYTE:
1115                         case Token.SHORT:
1116                         case Token.USHORT:
1117                         case Token.INT:
1118                         case Token.UINT:
1119                         case Token.LONG:
1120                         case Token.ULONG:
1121                         case Token.CHAR:
1122                         case Token.VOID:
1123                                 break;
1124                         case Token.OP_GENERICS_GT:
1125                         case Token.IN:
1126                         case Token.OUT:
1127                                 return true;
1128
1129                         default:
1130                                 return false;
1131                         }
1132                 again:
1133                         the_token = token ();
1134
1135                         if (the_token == Token.OP_GENERICS_GT)
1136                                 return true;
1137                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1138                                 goto start;
1139                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1140                                 goto again;
1141                         else if (the_token == Token.OP_GENERICS_LT) {
1142                                 if (!parse_less_than ())
1143                                         return false;
1144                                 goto again;
1145                         } else if (the_token == Token.OPEN_BRACKET) {
1146                         rank_specifiers:
1147                                 the_token = token ();
1148                                 if (the_token == Token.CLOSE_BRACKET)
1149                                         goto again;
1150                                 else if (the_token == Token.COMMA)
1151                                         goto rank_specifiers;
1152                                 return false;
1153                         }
1154
1155                         return false;
1156                 }
1157
1158                 bool parse_generic_dimension (out int dimension)
1159                 {
1160                         dimension = 1;
1161
1162                 again:
1163                         int the_token = token ();
1164                         if (the_token == Token.OP_GENERICS_GT)
1165                                 return true;
1166                         else if (the_token == Token.COMMA) {
1167                                 dimension++;
1168                                 goto again;
1169                         }
1170
1171                         return false;
1172                 }
1173
1174                 public int peek_token ()
1175                 {
1176                         int the_token;
1177
1178                         PushPosition ();
1179                         the_token = token ();
1180                         PopPosition ();
1181
1182                         return the_token;
1183                 }
1184
1185                 //
1186                 // Tonizes `?' using custom disambiguous rules to return one
1187                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1188                 //
1189                 // Tricky expression look like:
1190                 //
1191                 // Foo ? a = x ? b : c;
1192                 //
1193                 int TokenizePossibleNullableType ()
1194                 {
1195                         if (parsing_block == 0 || parsing_type > 0)
1196                                 return Token.INTERR_NULLABLE;
1197
1198                         int d = peek_char ();
1199                         if (d == '?') {
1200                                 get_char ();
1201                                 return Token.OP_COALESCING;
1202                         }
1203
1204                         switch (current_token) {
1205                         case Token.CLOSE_PARENS:
1206                         case Token.TRUE:
1207                         case Token.FALSE:
1208                         case Token.NULL:
1209                         case Token.LITERAL:
1210                                 return Token.INTERR;
1211                         }
1212
1213                         if (d != ' ') {
1214                                 if (d == ',' || d == ';' || d == '>')
1215                                         return Token.INTERR_NULLABLE;
1216                                 if (d == '*' || (d >= '0' && d <= '9'))
1217                                         return Token.INTERR;
1218                         }
1219
1220                         PushPosition ();
1221                         current_token = Token.NONE;
1222                         int next_token;
1223                         switch (xtoken ()) {
1224                         case Token.LITERAL:
1225                         case Token.TRUE:
1226                         case Token.FALSE:
1227                         case Token.NULL:
1228                         case Token.THIS:
1229                         case Token.NEW:
1230                                 next_token = Token.INTERR;
1231                                 break;
1232
1233                         case Token.SEMICOLON:
1234                         case Token.COMMA:
1235                         case Token.CLOSE_PARENS:
1236                         case Token.OPEN_BRACKET:
1237                         case Token.OP_GENERICS_GT:
1238                         case Token.INTERR:
1239                                 next_token = Token.INTERR_NULLABLE;
1240                                 break;
1241
1242                         default:
1243                                 next_token = -1;
1244                                 break;
1245                         }
1246
1247                         if (next_token == -1) {
1248                                 switch (xtoken ()) {
1249                                 case Token.COMMA:
1250                                 case Token.SEMICOLON:
1251                                 case Token.OPEN_BRACE:
1252                                 case Token.CLOSE_PARENS:
1253                                 case Token.IN:
1254                                         next_token = Token.INTERR_NULLABLE;
1255                                         break;
1256
1257                                 case Token.COLON:
1258                                         next_token = Token.INTERR;
1259                                         break;
1260
1261                                 default:
1262                                         int ntoken;
1263                                         int interrs = 1;
1264                                         int colons = 0;
1265                                         //
1266                                         // All shorcuts failed, do it hard way
1267                                         //
1268                                         while ((ntoken = xtoken ()) != Token.EOF) {
1269                                                 if (ntoken == Token.SEMICOLON)
1270                                                         break;
1271
1272                                                 if (ntoken == Token.COLON) {
1273                                                         if (++colons == interrs)
1274                                                                 break;
1275                                                         continue;
1276                                                 }
1277
1278                                                 if (ntoken == Token.INTERR) {
1279                                                         ++interrs;
1280                                                         continue;
1281                                                 }
1282                                         }
1283
1284                                         next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1285                                         break;
1286                                 }
1287                         }
1288
1289                         PopPosition ();
1290                         return next_token;
1291                 }
1292
1293                 bool decimal_digits (int c)
1294                 {
1295                         int d;
1296                         bool seen_digits = false;
1297
1298                         if (c != -1){
1299                                 if (number_pos == max_number_size)
1300                                         Error_NumericConstantTooLong ();
1301                                 number_builder [number_pos++] = (char) c;
1302                         }
1303
1304                         //
1305                         // We use peek_char2, because decimal_digits needs to do a
1306                         // 2-character look-ahead (5.ToString for example).
1307                         //
1308                         while ((d = peek_char2 ()) != -1){
1309                                 if (d >= '0' && d <= '9'){
1310                                         if (number_pos == max_number_size)
1311                                                 Error_NumericConstantTooLong ();
1312                                         number_builder [number_pos++] = (char) d;
1313                                         get_char ();
1314                                         seen_digits = true;
1315                                 } else
1316                                         break;
1317                         }
1318
1319                         return seen_digits;
1320                 }
1321
1322                 static bool is_hex (int e)
1323                 {
1324                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1325                 }
1326
1327                 static TypeCode real_type_suffix (int c)
1328                 {
1329                         switch (c){
1330                         case 'F': case 'f':
1331                                 return TypeCode.Single;
1332                         case 'D': case 'd':
1333                                 return TypeCode.Double;
1334                         case 'M': case 'm':
1335                                 return TypeCode.Decimal;
1336                         default:
1337                                 return TypeCode.Empty;
1338                         }
1339                 }
1340
1341                 ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc)
1342                 {
1343                         bool is_unsigned = false;
1344                         bool is_long = false;
1345
1346                         if (c != -1){
1347                                 bool scanning = true;
1348                                 do {
1349                                         switch (c){
1350                                         case 'U': case 'u':
1351                                                 if (is_unsigned)
1352                                                         scanning = false;
1353                                                 is_unsigned = true;
1354                                                 get_char ();
1355                                                 break;
1356
1357                                         case 'l':
1358                                                 if (!is_unsigned){
1359                                                         //
1360                                                         // if we have not seen anything in between
1361                                                         // report this error
1362                                                         //
1363                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1364                                                 }
1365
1366                                                 goto case 'L';
1367
1368                                         case 'L':
1369                                                 if (is_long)
1370                                                         scanning = false;
1371                                                 is_long = true;
1372                                                 get_char ();
1373                                                 break;
1374
1375                                         default:
1376                                                 scanning = false;
1377                                                 break;
1378                                         }
1379                                         c = peek_char ();
1380                                 } while (scanning);
1381                         }
1382
1383                         if (is_long && is_unsigned){
1384                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1385                         }
1386
1387                         if (is_unsigned){
1388                                 // uint if possible, or ulong else.
1389
1390                                 if ((ul & 0xffffffff00000000) == 0)
1391                                         return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc);
1392                                 else
1393                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1394                         } else if (is_long){
1395                                 // long if possible, ulong otherwise
1396                                 if ((ul & 0x8000000000000000) != 0)
1397                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1398                                 else
1399                                         return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1400                         } else {
1401                                 // int, uint, long or ulong in that order
1402                                 if ((ul & 0xffffffff00000000) == 0){
1403                                         uint ui = (uint) ul;
1404
1405                                         if ((ui & 0x80000000) != 0)
1406                                                 return new UIntLiteral (context.BuiltinTypes, ui, loc);
1407                                         else
1408                                                 return new IntLiteral (context.BuiltinTypes, (int) ui, loc);
1409                                 } else {
1410                                         if ((ul & 0x8000000000000000) != 0)
1411                                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1412                                         else
1413                                                 return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1414                                 }
1415                         }
1416                 }
1417
1418                 //
1419                 // given `c' as the next char in the input decide whether
1420                 // we need to convert to a special type, and then choose
1421                 // the best representation for the integer
1422                 //
1423                 ILiteralConstant adjust_int (int c, Location loc)
1424                 {
1425                         try {
1426                                 if (number_pos > 9){
1427                                         ulong ul = (uint) (number_builder [0] - '0');
1428
1429                                         for (int i = 1; i < number_pos; i++){
1430                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1431                                         }
1432
1433                                         return integer_type_suffix (ul, c, loc);
1434                                 } else {
1435                                         uint ui = (uint) (number_builder [0] - '0');
1436
1437                                         for (int i = 1; i < number_pos; i++){
1438                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1439                                         }
1440
1441                                         return integer_type_suffix (ui, c, loc);
1442                                 }
1443                         } catch (OverflowException) {
1444                                 Error_NumericConstantTooLong ();
1445                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1446                         }
1447                         catch (FormatException) {
1448                                 Report.Error (1013, Location, "Invalid number");
1449                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1450                         }
1451                 }
1452
1453                 ILiteralConstant adjust_real (TypeCode t, Location loc)
1454                 {
1455                         string s = new string (number_builder, 0, number_pos);
1456                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1457
1458                         switch (t){
1459                         case TypeCode.Decimal:
1460                                 try {
1461                                         return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc);
1462                                 } catch (OverflowException) {
1463                                         Report.Error (594, Location, error_details, "decimal");
1464                                         return new DecimalLiteral (context.BuiltinTypes, 0, loc);
1465                                 }
1466                         case TypeCode.Single:
1467                                 try {
1468                                         return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc);
1469                                 } catch (OverflowException) {
1470                                         Report.Error (594, Location, error_details, "float");
1471                                         return new FloatLiteral (context.BuiltinTypes, 0, loc);
1472                                 }
1473                         default:
1474                                 try {
1475                                         return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc);
1476                                 } catch (OverflowException) {
1477                                         Report.Error (594, loc, error_details, "double");
1478                                         return new DoubleLiteral (context.BuiltinTypes, 0, loc);
1479                                 }
1480                         }
1481                 }
1482
1483                 ILiteralConstant handle_hex (Location loc)
1484                 {
1485                         int d;
1486                         ulong ul;
1487
1488                         get_char ();
1489                         while ((d = peek_char ()) != -1){
1490                                 if (is_hex (d)){
1491                                         number_builder [number_pos++] = (char) d;
1492                                         get_char ();
1493                                 } else
1494                                         break;
1495                         }
1496
1497                         string s = new String (number_builder, 0, number_pos);
1498
1499                         try {
1500                                 if (number_pos <= 8)
1501                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1502                                 else
1503                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1504
1505                                 return integer_type_suffix (ul, peek_char (), loc);
1506                         } catch (OverflowException){
1507                                 Error_NumericConstantTooLong ();
1508                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1509                         }
1510                         catch (FormatException) {
1511                                 Report.Error (1013, Location, "Invalid number");
1512                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1513                         }
1514                 }
1515
1516                 //
1517                 // Invoked if we know we have .digits or digits
1518                 //
1519                 int is_number (int c)
1520                 {
1521                         ILiteralConstant res;
1522
1523 #if FULL_AST
1524                         int read_start = reader.Position - 1;
1525                         if (c == '.') {
1526                                 //
1527                                 // Caller did peek_char
1528                                 //
1529                                 --read_start;
1530                         }
1531 #endif
1532                         number_pos = 0;
1533                         var loc = Location;
1534
1535                         if (c >= '0' && c <= '9'){
1536                                 if (c == '0'){
1537                                         int peek = peek_char ();
1538
1539                                         if (peek == 'x' || peek == 'X') {
1540                                                 val = res = handle_hex (loc);
1541 #if FULL_AST
1542                                                 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1543 #endif
1544
1545                                                 return Token.LITERAL;
1546                                         }
1547                                 }
1548                                 decimal_digits (c);
1549                                 c = get_char ();
1550                         }
1551
1552                         //
1553                         // We need to handle the case of
1554                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1555                         //
1556                         bool is_real = false;
1557                         if (c == '.'){
1558                                 if (decimal_digits ('.')){
1559                                         is_real = true;
1560                                         c = get_char ();
1561                                 } else {
1562                                         putback ('.');
1563                                         number_pos--;
1564                                         val = res = adjust_int (-1, loc);
1565
1566 #if FULL_AST
1567                                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1568 #endif
1569                                         return Token.LITERAL;
1570                                 }
1571                         }
1572
1573                         if (c == 'e' || c == 'E'){
1574                                 is_real = true;
1575                                 if (number_pos == max_number_size)
1576                                         Error_NumericConstantTooLong ();
1577                                 number_builder [number_pos++] = (char) c;
1578                                 c = get_char ();
1579
1580                                 if (c == '+'){
1581                                         if (number_pos == max_number_size)
1582                                                 Error_NumericConstantTooLong ();
1583                                         number_builder [number_pos++] = '+';
1584                                         c = -1;
1585                                 } else if (c == '-') {
1586                                         if (number_pos == max_number_size)
1587                                                 Error_NumericConstantTooLong ();
1588                                         number_builder [number_pos++] = '-';
1589                                         c = -1;
1590                                 } else {
1591                                         if (number_pos == max_number_size)
1592                                                 Error_NumericConstantTooLong ();
1593                                         number_builder [number_pos++] = '+';
1594                                 }
1595
1596                                 decimal_digits (c);
1597                                 c = get_char ();
1598                         }
1599
1600                         var type = real_type_suffix (c);
1601                         if (type == TypeCode.Empty && !is_real) {
1602                                 putback (c);
1603                                 res = adjust_int (c, loc);
1604                         } else {
1605                                 is_real = true;
1606
1607                                 if (type == TypeCode.Empty) {
1608                                         putback (c);
1609                                 }
1610
1611                                 res = adjust_real (type, loc);
1612                         }
1613
1614                         val = res;
1615
1616 #if FULL_AST
1617                         var chars = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty && c > 0 ? 1 : 0));
1618                         if (chars[chars.Length - 1] == '\r')
1619                                 Array.Resize (ref chars, chars.Length - 1);
1620                         res.ParsedValue = chars;
1621 #endif
1622
1623                         return Token.LITERAL;
1624                 }
1625
1626                 //
1627                 // Accepts exactly count (4 or 8) hex, no more no less
1628                 //
1629                 int getHex (int count, out int surrogate, out bool error)
1630                 {
1631                         int i;
1632                         int total = 0;
1633                         int c;
1634                         int top = count != -1 ? count : 4;
1635
1636                         get_char ();
1637                         error = false;
1638                         surrogate = 0;
1639                         for (i = 0; i < top; i++){
1640                                 c = get_char ();
1641
1642                                 if (c >= '0' && c <= '9')
1643                                         c = (int) c - (int) '0';
1644                                 else if (c >= 'A' && c <= 'F')
1645                                         c = (int) c - (int) 'A' + 10;
1646                                 else if (c >= 'a' && c <= 'f')
1647                                         c = (int) c - (int) 'a' + 10;
1648                                 else {
1649                                         error = true;
1650                                         return 0;
1651                                 }
1652
1653                                 total = (total * 16) + c;
1654                                 if (count == -1){
1655                                         int p = peek_char ();
1656                                         if (p == -1)
1657                                                 break;
1658                                         if (!is_hex ((char)p))
1659                                                 break;
1660                                 }
1661                         }
1662
1663                         if (top == 8) {
1664                                 if (total > 0x0010FFFF) {
1665                                         error = true;
1666                                         return 0;
1667                                 }
1668
1669                                 if (total >= 0x00010000) {
1670                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1671                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1672                                 }
1673                         }
1674
1675                         return total;
1676                 }
1677
1678                 int escape (int c, out int surrogate)
1679                 {
1680                         bool error;
1681                         int d;
1682                         int v;
1683
1684                         d = peek_char ();
1685                         if (c != '\\') {
1686                                 surrogate = 0;
1687                                 return c;
1688                         }
1689
1690                         switch (d){
1691                         case 'a':
1692                                 v = '\a'; break;
1693                         case 'b':
1694                                 v = '\b'; break;
1695                         case 'n':
1696                                 v = '\n'; break;
1697                         case 't':
1698                                 v = '\t'; break;
1699                         case 'v':
1700                                 v = '\v'; break;
1701                         case 'r':
1702                                 v = '\r'; break;
1703                         case '\\':
1704                                 v = '\\'; break;
1705                         case 'f':
1706                                 v = '\f'; break;
1707                         case '0':
1708                                 v = 0; break;
1709                         case '"':
1710                                 v = '"'; break;
1711                         case '\'':
1712                                 v = '\''; break;
1713                         case 'x':
1714                                 v = getHex (-1, out surrogate, out error);
1715                                 if (error)
1716                                         goto default;
1717                                 return v;
1718                         case 'u':
1719                         case 'U':
1720                                 return EscapeUnicode (d, out surrogate);
1721                         default:
1722                                 surrogate = 0;
1723                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1724                                 return d;
1725                         }
1726
1727                         get_char ();
1728                         surrogate = 0;
1729                         return v;
1730                 }
1731
1732                 int EscapeUnicode (int ch, out int surrogate)
1733                 {
1734                         bool error;
1735                         if (ch == 'U') {
1736                                 ch = getHex (8, out surrogate, out error);
1737                         } else {
1738                                 ch = getHex (4, out surrogate, out error);
1739                         }
1740
1741                         if (error)
1742                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1743
1744                         return ch;
1745                 }
1746
1747                 int get_char ()
1748                 {
1749                         int x;
1750                         if (putback_char != -1) {
1751                                 x = putback_char;
1752                                 putback_char = -1;
1753                         } else {
1754                                 x = reader.Read ();
1755                         }
1756
1757                         if (x == '\r') {
1758                                 if (peek_char () == '\n') {
1759                                         putback_char = -1;
1760                                 }
1761
1762                                 x = '\n';
1763                                 advance_line ();
1764                         } else if (x == '\n') {
1765                                 advance_line ();
1766                         } else {
1767                                 col++;
1768                         }
1769                         return x;
1770                 }
1771
1772                 void advance_line ()
1773                 {
1774                         line++;
1775                         ref_line++;
1776                         previous_col = col;
1777                         col = 0;
1778                 }
1779
1780                 int peek_char ()
1781                 {
1782                         if (putback_char == -1)
1783                                 putback_char = reader.Read ();
1784                         return putback_char;
1785                 }
1786
1787                 int peek_char2 ()
1788                 {
1789                         if (putback_char != -1)
1790                                 return putback_char;
1791                         return reader.Peek ();
1792                 }
1793
1794                 public void putback (int c)
1795                 {
1796                         if (putback_char != -1){
1797                                 Console.WriteLine ("Col: " + col);
1798                                 Console.WriteLine ("Row: " + line);
1799                                 Console.WriteLine ("Name: " + current_source.Name);
1800                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1801                                 throw new Exception ("This should not happen putback on putback");
1802                         }
1803                         if (c == '\n' || col == 0) {
1804                                 // It won't happen though.
1805                                 line--;
1806                                 ref_line--;
1807                                 col = previous_col;
1808                         }
1809                         else
1810                                 col--;
1811                         putback_char = c;
1812                 }
1813
1814                 public bool advance ()
1815                 {
1816                         return peek_char () != -1 || CompleteOnEOF;
1817                 }
1818
1819                 public Object Value {
1820                         get {
1821                                 return val;
1822                         }
1823                 }
1824
1825                 public Object value ()
1826                 {
1827                         return val;
1828                 }
1829
1830                 public int token ()
1831                 {
1832                         current_token = xtoken ();
1833                         return current_token;
1834                 }
1835
1836                 int TokenizePreprocessorIdentifier (out int c)
1837                 {
1838                         // skip over white space
1839                         do {
1840                                 c = get_char ();
1841                         } while (c == ' ' || c == '\t');
1842
1843
1844                         int pos = 0;
1845                         while (c != -1 && c >= 'a' && c <= 'z') {
1846                                 id_builder[pos++] = (char) c;
1847                                 c = get_char ();
1848                                 if (c == '\\') {
1849                                         int peek = peek_char ();
1850                                         if (peek == 'U' || peek == 'u') {
1851                                                 int surrogate;
1852                                                 c = EscapeUnicode (c, out surrogate);
1853                                                 if (surrogate != 0) {
1854                                                         if (is_identifier_part_character ((char) c)) {
1855                                                                 id_builder[pos++] = (char) c;
1856                                                         }
1857                                                         c = surrogate;
1858                                                 }
1859                                         }
1860                                 }
1861                         }
1862
1863                         return pos;
1864                 }
1865
1866                 PreprocessorDirective get_cmd_arg (out string arg)
1867                 {
1868                         int c;
1869
1870                         tokens_seen = false;
1871                         arg = "";
1872
1873                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorIdentifier (out c));
1874
1875                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
1876                                 return cmd;
1877
1878                         // skip over white space
1879                         while (c == ' ' || c == '\t')
1880                                 c = get_char ();
1881
1882                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
1883                         int pos = 0;
1884
1885                         while (c != -1 && c != '\n') {
1886                                 if (c == '\\' && has_identifier_argument >= 0) {
1887                                         if (has_identifier_argument != 0) {
1888                                                 has_identifier_argument = 1;
1889
1890                                                 int peek = peek_char ();
1891                                                 if (peek == 'U' || peek == 'u') {
1892                                                         int surrogate;
1893                                                         c = EscapeUnicode (c, out surrogate);
1894                                                         if (surrogate != 0) {
1895                                                                 if (is_identifier_part_character ((char) c)) {
1896                                                                         if (pos == value_builder.Length)
1897                                                                                 Array.Resize (ref value_builder, pos * 2);
1898
1899                                                                         value_builder[pos++] = (char) c;
1900                                                                 }
1901                                                                 c = surrogate;
1902                                                         }
1903                                                 }
1904                                         } else {
1905                                                 has_identifier_argument = -1;
1906                                         }
1907                                 } else if (c == '/' && peek_char () == '/') {
1908                                         //
1909                                         // Eat single-line comments
1910                                         //
1911                                         get_char ();
1912                                         do {
1913                                                 c = get_char ();
1914                                         } while (c != -1 && c != '\n');
1915
1916                                         break;
1917                                 }
1918
1919                                 if (pos == value_builder.Length)
1920                                         Array.Resize (ref value_builder, pos * 2);
1921
1922                                 value_builder[pos++] = (char) c;
1923                                 c = get_char ();
1924                         }
1925
1926                         if (pos != 0) {
1927                                 if (pos > max_id_size)
1928                                         arg = new string (value_builder, 0, pos);
1929                                 else
1930                                         arg = InternIdentifier (value_builder, pos);
1931
1932                                 // Eat any trailing whitespaces
1933                                 arg = arg.Trim (simple_whitespaces);
1934                         }
1935
1936                         return cmd;
1937                 }
1938
1939                 //
1940                 // Handles the #line directive
1941                 //
1942                 bool PreProcessLine (string arg)
1943                 {
1944                         if (arg.Length == 0)
1945                                 return false;
1946
1947                         if (arg == "default"){
1948                                 ref_line = line;
1949                                 current_source = source_file.SourceFile;
1950                                 hidden = false;
1951                                 Location.Push (current_source);
1952                                 return true;
1953                         } else if (arg == "hidden"){
1954                                 hidden = true;
1955                                 return true;
1956                         }
1957
1958                         try {
1959                                 int pos;
1960
1961                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1962                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1963                                         pos++;
1964
1965                                         char [] quotes = { '\"' };
1966
1967                                         string name = arg.Substring (pos).Trim (quotes);
1968                                         current_source = context.LookupFile (source_file, name);
1969                                         source_file.AddIncludeFile (current_source);
1970                                         hidden = false;
1971                                         Location.Push (current_source);
1972                                 } else {
1973                                         ref_line = System.Int32.Parse (arg);
1974                                         hidden = false;
1975                                 }
1976                         } catch {
1977                                 return false;
1978                         }
1979
1980                         return true;
1981                 }
1982
1983                 //
1984                 // Handles #define and #undef
1985                 //
1986                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
1987                 {
1988                         if (ident.Length == 0 || ident == "true" || ident == "false"){
1989                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
1990                                 return;
1991                         }
1992
1993                         if (ident.IndexOfAny (simple_whitespaces) != -1){
1994                                 Error_EndLineExpected ();
1995                                 return;
1996                         }
1997
1998                         if (!is_identifier_start_character (ident [0]))
1999                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
2000
2001                         foreach (char c in ident.Substring (1)){
2002                                 if (!is_identifier_part_character (c)){
2003                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
2004                                         return;
2005                                 }
2006                         }
2007
2008                         if (!caller_is_taking)
2009                                 return;
2010
2011                         if (is_define) {
2012                                 //
2013                                 // #define ident
2014                                 //
2015                                 if (context.Settings.IsConditionalSymbolDefined (ident))
2016                                         return;
2017
2018                                 source_file.AddDefine (ident);
2019                         } else {
2020                                 //
2021                                 // #undef ident
2022                                 //
2023                                 source_file.AddUndefine (ident);
2024                         }
2025                 }
2026
2027                 byte read_hex (out bool error)
2028                 {
2029                         int total;
2030                         int c = get_char ();
2031
2032                         if ((c >= '0') && (c <= '9'))
2033                                 total = (int) c - (int) '0';
2034                         else if ((c >= 'A') && (c <= 'F'))
2035                                 total = (int) c - (int) 'A' + 10;
2036                         else if ((c >= 'a') && (c <= 'f'))
2037                                 total = (int) c - (int) 'a' + 10;
2038                         else {
2039                                 error = true;
2040                                 return 0;
2041                         }
2042
2043                         total *= 16;
2044                         c = get_char ();
2045
2046                         if ((c >= '0') && (c <= '9'))
2047                                 total += (int) c - (int) '0';
2048                         else if ((c >= 'A') && (c <= 'F'))
2049                                 total += (int) c - (int) 'A' + 10;
2050                         else if ((c >= 'a') && (c <= 'f'))
2051                                 total += (int) c - (int) 'a' + 10;
2052                         else {
2053                                 error = true;
2054                                 return 0;
2055                         }
2056
2057                         error = false;
2058                         return (byte) total;
2059                 }
2060
2061                 //
2062                 // Parses #pragma checksum
2063                 //
2064                 bool ParsePragmaChecksum ()
2065                 {
2066                         //
2067                         // The syntax is ` "foo.txt" "{guid}" "hash"'
2068                         //
2069                         int c = get_char ();
2070
2071                         if (c != '"')
2072                                 return false;
2073
2074                         var string_builder = new StringBuilder ();
2075                         while (c != -1 && c != '\n') {
2076                                 c = get_char ();
2077                                 if (c == '"') {
2078                                         c = get_char ();
2079                                         break;
2080                                 }
2081
2082                                 string_builder.Append ((char) c);
2083                         }
2084
2085                         if (string_builder.Length == 0) {
2086                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
2087                         }
2088
2089                         // TODO: Any white-spaces count
2090                         if (c != ' ')
2091                                 return false;
2092
2093                         SourceFile file = context.LookupFile (source_file, string_builder.ToString ());
2094
2095                         if (get_char () != '"' || get_char () != '{')
2096                                 return false;
2097
2098                         bool error;
2099                         byte[] guid_bytes = new byte [16];
2100                         int i = 0;
2101
2102                         for (; i < 4; i++) {
2103                                 guid_bytes [i] = read_hex (out error);
2104                                 if (error)
2105                                         return false;
2106                         }
2107
2108                         if (get_char () != '-')
2109                                 return false;
2110
2111                         for (; i < 10; i++) {
2112                                 guid_bytes [i] = read_hex (out error);
2113                                 if (error)
2114                                         return false;
2115
2116                                 guid_bytes [i++] = read_hex (out error);
2117                                 if (error)
2118                                         return false;
2119
2120                                 if (get_char () != '-')
2121                                         return false;
2122                         }
2123
2124                         for (; i < 16; i++) {
2125                                 guid_bytes [i] = read_hex (out error);
2126                                 if (error)
2127                                         return false;
2128                         }
2129
2130                         if (get_char () != '}' || get_char () != '"')
2131                                 return false;
2132
2133                         // TODO: Any white-spaces count
2134                         c = get_char ();
2135                         if (c != ' ')
2136                                 return false;
2137
2138                         if (get_char () != '"')
2139                                 return false;
2140
2141                         // Any length of checksum
2142                         List<byte> checksum_bytes = new List<byte> (16);
2143
2144                         c = peek_char ();
2145                         while (c != '"' && c != -1) {
2146                                 checksum_bytes.Add (read_hex (out error));
2147                                 if (error)
2148                                         return false;
2149
2150                                 c = peek_char ();
2151                         }
2152
2153                         if (c == '/') {
2154                                 ReadSingleLineComment ();
2155                         } else if (get_char () != '"') {
2156                                 return false;
2157                         }
2158
2159                         file.SetChecksum (guid_bytes, checksum_bytes.ToArray ());
2160                         current_source.AutoGenerated = true;
2161                         return true;
2162                 }
2163
2164                 bool IsTokenIdentifierEqual (char[] identifier)
2165                 {
2166                         for (int i = 0; i < identifier.Length; ++i) {
2167                                 if (identifier[i] != id_builder[i])
2168                                         return false;
2169                         }
2170
2171                         return true;
2172                 }
2173
2174                 int TokenizePragmaNumber (ref int c)
2175                 {
2176                         number_pos = 0;
2177
2178                         int number;
2179
2180                         if (c >= '0' && c <= '9') {
2181                                 decimal_digits (c);
2182                                 uint ui = (uint) (number_builder[0] - '0');
2183
2184                                 try {
2185                                         for (int i = 1; i < number_pos; i++) {
2186                                                 ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2187                                         }
2188
2189                                         number = (int) ui;
2190                                 } catch (OverflowException) {
2191                                         Error_NumericConstantTooLong ();
2192                                         number = -1;
2193                                 }
2194
2195
2196                                 c = get_char ();
2197
2198                                 // skip over white space
2199                                 while (c == ' ' || c == '\t')
2200                                         c = get_char ();
2201
2202                                 if (c == ',') {
2203                                         c = get_char ();
2204                                 }
2205
2206                                 // skip over white space
2207                                 while (c == ' ' || c == '\t')
2208                                         c = get_char ();
2209                         } else {
2210                                 number = -1;
2211                                 if (c == '/') {
2212                                         ReadSingleLineComment ();
2213                                 } else {
2214                                         Report.Warning (1692, 1, Location, "Invalid number");
2215
2216                                         // Read everything till the end of the line or file
2217                                         do {
2218                                                 c = get_char ();
2219                                         } while (c != -1 && c != '\n');
2220                                 }
2221                         }
2222
2223                         return number;
2224                 }
2225
2226                 void ReadSingleLineComment ()
2227                 {
2228                         if (peek_char () != '/')
2229                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2230
2231                         // Read everything till the end of the line or file
2232                         int c;
2233                         do {
2234                                 c = get_char ();
2235                         } while (c != -1 && c != '\n');
2236                 }
2237
2238                 /// <summary>
2239                 /// Handles #pragma directive
2240                 /// </summary>
2241                 void ParsePragmaDirective (string arg)
2242                 {
2243                         int c;
2244                         int length = TokenizePreprocessorIdentifier (out c);
2245                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2246                                 length = TokenizePreprocessorIdentifier (out c);
2247
2248                                 //
2249                                 // #pragma warning disable
2250                                 // #pragma warning restore
2251                                 //
2252                                 if (length == pragma_warning_disable.Length) {
2253                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2254                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2255                                                 // skip over white space
2256                                                 while (c == ' ' || c == '\t')
2257                                                         c = get_char ();
2258
2259                                                 var loc = Location;
2260
2261                                                 if (c == '\n' || c == '/') {
2262                                                         if (c == '/')
2263                                                                 ReadSingleLineComment ();
2264
2265                                                         //
2266                                                         // Disable/Restore all warnings
2267                                                         //
2268                                                         if (disable) {
2269                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2270                                                         } else {
2271                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2272                                                         }
2273                                                 } else {
2274                                                         //
2275                                                         // Disable/Restore a warning or group of warnings
2276                                                         //
2277                                                         int code;
2278                                                         do {
2279                                                                 code = TokenizePragmaNumber (ref c);
2280                                                                 if (code > 0) {
2281                                                                         if (disable) {
2282                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, context.Report);
2283                                                                         } else {
2284                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, context);
2285                                                                         }
2286                                                                 }
2287                                                         } while (code >= 0 && c != '\n' && c != -1);
2288                                                 }
2289
2290                                                 return;
2291                                         }
2292                                 }
2293
2294                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2295                                 return;
2296                         }
2297
2298                         //
2299                         // #pragma checksum
2300                         //
2301                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2302                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2303                                         Report.Warning (1695, 1, Location,
2304                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2305                                 }
2306
2307                                 return;
2308                         }
2309
2310                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2311                 }
2312
2313                 bool eval_val (string s)
2314                 {
2315                         if (s == "true")
2316                                 return true;
2317                         if (s == "false")
2318                                 return false;
2319
2320                         return source_file.IsConditionalDefined (s);
2321                 }
2322
2323                 bool pp_primary (ref string s)
2324                 {
2325                         s = s.Trim ();
2326                         int len = s.Length;
2327
2328                         if (len > 0){
2329                                 char c = s [0];
2330
2331                                 if (c == '('){
2332                                         s = s.Substring (1);
2333                                         bool val = pp_expr (ref s, false);
2334                                         if (s.Length > 0 && s [0] == ')'){
2335                                                 s = s.Substring (1);
2336                                                 return val;
2337                                         }
2338                                         Error_InvalidDirective ();
2339                                         return false;
2340                                 }
2341
2342                                 if (is_identifier_start_character (c)){
2343                                         int j = 1;
2344
2345                                         while (j < len){
2346                                                 c = s [j];
2347
2348                                                 if (is_identifier_part_character (c)){
2349                                                         j++;
2350                                                         continue;
2351                                                 }
2352                                                 bool v = eval_val (s.Substring (0, j));
2353                                                 s = s.Substring (j);
2354                                                 return v;
2355                                         }
2356                                         bool vv = eval_val (s);
2357                                         s = "";
2358                                         return vv;
2359                                 }
2360                         }
2361                         Error_InvalidDirective ();
2362                         return false;
2363                 }
2364
2365                 bool pp_unary (ref string s)
2366                 {
2367                         s = s.Trim ();
2368                         int len = s.Length;
2369
2370                         if (len > 0){
2371                                 if (s [0] == '!'){
2372                                         if (len > 1 && s [1] == '='){
2373                                                 Error_InvalidDirective ();
2374                                                 return false;
2375                                         }
2376                                         s = s.Substring (1);
2377                                         return ! pp_primary (ref s);
2378                                 } else
2379                                         return pp_primary (ref s);
2380                         } else {
2381                                 Error_InvalidDirective ();
2382                                 return false;
2383                         }
2384                 }
2385
2386                 bool pp_eq (ref string s)
2387                 {
2388                         bool va = pp_unary (ref s);
2389
2390                         s = s.Trim ();
2391                         int len = s.Length;
2392                         if (len > 0){
2393                                 if (s [0] == '='){
2394                                         if (len > 2 && s [1] == '='){
2395                                                 s = s.Substring (2);
2396                                                 return va == pp_unary (ref s);
2397                                         } else {
2398                                                 Error_InvalidDirective ();
2399                                                 return false;
2400                                         }
2401                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2402                                         s = s.Substring (2);
2403
2404                                         return va != pp_unary (ref s);
2405
2406                                 }
2407                         }
2408
2409                         return va;
2410
2411                 }
2412
2413                 bool pp_and (ref string s)
2414                 {
2415                         bool va = pp_eq (ref s);
2416
2417                         s = s.Trim ();
2418                         int len = s.Length;
2419                         if (len > 0){
2420                                 if (s [0] == '&'){
2421                                         if (len > 2 && s [1] == '&'){
2422                                                 s = s.Substring (2);
2423                                                 return (va & pp_and (ref s));
2424                                         } else {
2425                                                 Error_InvalidDirective ();
2426                                                 return false;
2427                                         }
2428                                 }
2429                         }
2430                         return va;
2431                 }
2432
2433                 //
2434                 // Evaluates an expression for `#if' or `#elif'
2435                 //
2436                 bool pp_expr (ref string s, bool isTerm)
2437                 {
2438                         bool va = pp_and (ref s);
2439                         s = s.Trim ();
2440                         int len = s.Length;
2441                         if (len > 0){
2442                                 char c = s [0];
2443
2444                                 if (c == '|'){
2445                                         if (len > 2 && s [1] == '|'){
2446                                                 s = s.Substring (2);
2447                                                 return va | pp_expr (ref s, isTerm);
2448                                         } else {
2449                                                 Error_InvalidDirective ();
2450                                                 return false;
2451                                         }
2452                                 }
2453                                 if (isTerm) {
2454                                         Error_EndLineExpected ();
2455                                         return false;
2456                                 }
2457                         }
2458
2459                         return va;
2460                 }
2461
2462                 bool eval (string s)
2463                 {
2464                         bool v = pp_expr (ref s, true);
2465                         s = s.Trim ();
2466                         if (s.Length != 0){
2467                                 return false;
2468                         }
2469
2470                         return v;
2471                 }
2472
2473                 void Error_NumericConstantTooLong ()
2474                 {
2475                         Report.Error (1021, Location, "Integral constant is too large");
2476                 }
2477
2478                 void Error_InvalidDirective ()
2479                 {
2480                         Report.Error (1517, Location, "Invalid preprocessor directive");
2481                 }
2482
2483                 void Error_UnexpectedDirective (string extra)
2484                 {
2485                         Report.Error (
2486                                 1028, Location,
2487                                 "Unexpected processor directive ({0})", extra);
2488                 }
2489
2490                 void Error_TokensSeen ()
2491                 {
2492                         Report.Error (1032, Location,
2493                                 "Cannot define or undefine preprocessor symbols after first token in file");
2494                 }
2495
2496                 void Eror_WrongPreprocessorLocation ()
2497                 {
2498                         Report.Error (1040, Location,
2499                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2500                 }
2501
2502                 void Error_EndLineExpected ()
2503                 {
2504                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2505                 }
2506
2507                 //
2508                 // Raises a warning when tokenizer found documentation comment
2509                 // on unexpected place
2510                 //
2511                 void WarningMisplacedComment (Location loc)
2512                 {
2513                         if (doc_state != XmlCommentState.Error) {
2514                                 doc_state = XmlCommentState.Error;
2515                                 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element");
2516                         }
2517                 }
2518
2519                 //
2520                 // if true, then the code continues processing the code
2521                 // if false, the code stays in a loop until another directive is
2522                 // reached.
2523                 // When caller_is_taking is false we ignore all directives except the ones
2524                 // which can help us to identify where the #if block ends
2525                 bool ParsePreprocessingDirective (bool caller_is_taking)
2526                 {
2527                         string arg;
2528                         bool region_directive = false;
2529
2530                         var directive = get_cmd_arg (out arg);
2531
2532                         //
2533                         // The first group of pre-processing instructions is always processed
2534                         //
2535                         switch (directive) {
2536                         case PreprocessorDirective.Region:
2537                                 region_directive = true;
2538                                 arg = "true";
2539                                 goto case PreprocessorDirective.If;
2540
2541                         case PreprocessorDirective.Endregion:
2542                                 if (ifstack == null || ifstack.Count == 0){
2543                                         Error_UnexpectedDirective ("no #region for this #endregion");
2544                                         return true;
2545                                 }
2546                                 int pop = ifstack.Pop ();
2547
2548                                 if ((pop & REGION) == 0)
2549                                         Report.Error (1027, Location, "Expected `#endif' directive");
2550
2551                                 return caller_is_taking;
2552
2553                         case PreprocessorDirective.If:
2554                                 if (ifstack == null)
2555                                         ifstack = new Stack<int> (2);
2556
2557                                 int flags = region_directive ? REGION : 0;
2558                                 if (ifstack.Count == 0){
2559                                         flags |= PARENT_TAKING;
2560                                 } else {
2561                                         int state = ifstack.Peek ();
2562                                         if ((state & TAKING) != 0) {
2563                                                 flags |= PARENT_TAKING;
2564                                         }
2565                                 }
2566
2567                                 if (eval (arg) && caller_is_taking) {
2568                                         ifstack.Push (flags | TAKING);
2569                                         return true;
2570                                 }
2571                                 ifstack.Push (flags);
2572                                 return false;
2573
2574                         case PreprocessorDirective.Endif:
2575                                 if (ifstack == null || ifstack.Count == 0){
2576                                         Error_UnexpectedDirective ("no #if for this #endif");
2577                                         return true;
2578                                 } else {
2579                                         pop = ifstack.Pop ();
2580
2581                                         if ((pop & REGION) != 0)
2582                                                 Report.Error (1038, Location, "#endregion directive expected");
2583
2584                                         if (arg.Length != 0) {
2585                                                 Error_EndLineExpected ();
2586                                         }
2587
2588                                         if (ifstack.Count == 0)
2589                                                 return true;
2590
2591                                         int state = ifstack.Peek ();
2592                                         return (state & TAKING) != 0;
2593                                 }
2594
2595                         case PreprocessorDirective.Elif:
2596                                 if (ifstack == null || ifstack.Count == 0){
2597                                         Error_UnexpectedDirective ("no #if for this #elif");
2598                                         return true;
2599                                 } else {
2600                                         int state = ifstack.Pop ();
2601
2602                                         if ((state & REGION) != 0) {
2603                                                 Report.Error (1038, Location, "#endregion directive expected");
2604                                                 return true;
2605                                         }
2606
2607                                         if ((state & ELSE_SEEN) != 0){
2608                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2609                                                 return true;
2610                                         }
2611
2612                                         if ((state & TAKING) != 0) {
2613                                                 ifstack.Push (0);
2614                                                 return false;
2615                                         }
2616
2617                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2618                                                 ifstack.Push (state | TAKING);
2619                                                 return true;
2620                                         }
2621
2622                                         ifstack.Push (state);
2623                                         return false;
2624                                 }
2625
2626                         case PreprocessorDirective.Else:
2627                                 if (ifstack == null || ifstack.Count == 0){
2628                                         Error_UnexpectedDirective ("no #if for this #else");
2629                                         return true;
2630                                 } else {
2631                                         int state = ifstack.Peek ();
2632
2633                                         if ((state & REGION) != 0) {
2634                                                 Report.Error (1038, Location, "#endregion directive expected");
2635                                                 return true;
2636                                         }
2637
2638                                         if ((state & ELSE_SEEN) != 0){
2639                                                 Error_UnexpectedDirective ("#else within #else");
2640                                                 return true;
2641                                         }
2642
2643                                         ifstack.Pop ();
2644
2645                                         if (arg.Length != 0) {
2646                                                 Error_EndLineExpected ();
2647                                                 return true;
2648                                         }
2649
2650                                         bool ret = false;
2651                                         if ((state & PARENT_TAKING) != 0) {
2652                                                 ret = (state & TAKING) == 0;
2653
2654                                                 if (ret)
2655                                                         state |= TAKING;
2656                                                 else
2657                                                         state &= ~TAKING;
2658                                         }
2659
2660                                         ifstack.Push (state | ELSE_SEEN);
2661
2662                                         return ret;
2663                                 }
2664                         case PreprocessorDirective.Define:
2665                                 if (any_token_seen){
2666                                         Error_TokensSeen ();
2667                                         return caller_is_taking;
2668                                 }
2669                                 PreProcessDefinition (true, arg, caller_is_taking);
2670                                 return caller_is_taking;
2671
2672                         case PreprocessorDirective.Undef:
2673                                 if (any_token_seen){
2674                                         Error_TokensSeen ();
2675                                         return caller_is_taking;
2676                                 }
2677                                 PreProcessDefinition (false, arg, caller_is_taking);
2678                                 return caller_is_taking;
2679
2680                         case PreprocessorDirective.Invalid:
2681                                 Report.Error (1024, Location, "Wrong preprocessor directive");
2682                                 return true;
2683                         }
2684
2685                         //
2686                         // These are only processed if we are in a `taking' block
2687                         //
2688                         if (!caller_is_taking)
2689                                 return false;
2690
2691                         switch (directive){
2692                         case PreprocessorDirective.Error:
2693                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2694                                 return true;
2695
2696                         case PreprocessorDirective.Warning:
2697                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2698                                 return true;
2699
2700                         case PreprocessorDirective.Pragma:
2701                                 if (context.Settings.Version == LanguageVersion.ISO_1) {
2702                                         Report.FeatureIsNotAvailable (context, Location, "#pragma");
2703                                 }
2704
2705                                 ParsePragmaDirective (arg);
2706                                 return true;
2707
2708                         case PreprocessorDirective.Line:
2709                                 if (!PreProcessLine (arg))
2710                                         Report.Error (
2711                                                 1576, Location,
2712                                                 "The line number specified for #line directive is missing or invalid");
2713                                 return caller_is_taking;
2714                         }
2715
2716                         throw new NotImplementedException (directive.ToString ());
2717                 }
2718
2719                 private int consume_string (bool quoted)
2720                 {
2721                         int c;
2722                         int pos = 0;
2723                         Location start_location = Location;
2724                         if (quoted)
2725                                 start_location = start_location - 1;
2726
2727 #if FULL_AST
2728                         int reader_pos = reader.Position;
2729 #endif
2730
2731                         while (true){
2732                                 c = get_char ();
2733                                 if (c == '"') {
2734                                         if (quoted && peek_char () == '"') {
2735                                                 if (pos == value_builder.Length)
2736                                                         Array.Resize (ref value_builder, pos * 2);
2737
2738                                                 value_builder[pos++] = (char) c;
2739                                                 get_char ();
2740                                                 continue;
2741                                         }
2742
2743                                         string s;
2744                                         if (pos == 0)
2745                                                 s = string.Empty;
2746                                         else if (pos <= 4)
2747                                                 s = InternIdentifier (value_builder, pos);
2748                                         else
2749                                                 s = new string (value_builder, 0, pos);
2750
2751                                         ILiteralConstant res = new StringLiteral (context.BuiltinTypes, s, start_location);
2752                                         val = res;
2753 #if FULL_AST
2754                                         res.ParsedValue = quoted ?
2755                                                 reader.ReadChars (reader_pos - 2, reader.Position - 1) :
2756                                                 reader.ReadChars (reader_pos - 1, reader.Position);
2757 #endif
2758
2759                                         return Token.LITERAL;
2760                                 }
2761
2762                                 if (c == '\n') {
2763                                         if (!quoted) {
2764                                                 Report.Error (1010, Location, "Newline in constant");
2765                                                 val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location);
2766                                                 return Token.LITERAL;
2767                                         }
2768                                 } else if (c == '\\' && !quoted) {
2769                                         int surrogate;
2770                                         c = escape (c, out surrogate);
2771                                         if (c == -1)
2772                                                 return Token.ERROR;
2773                                         if (surrogate != 0) {
2774                                                 if (pos == value_builder.Length)
2775                                                         Array.Resize (ref value_builder, pos * 2);
2776
2777                                                 value_builder[pos++] = (char) c;
2778                                                 c = surrogate;
2779                                         }
2780                                 } else if (c == -1) {
2781                                         Report.Error (1039, Location, "Unterminated string literal");
2782                                         return Token.EOF;
2783                                 }
2784
2785                                 if (pos == value_builder.Length)
2786                                         Array.Resize (ref value_builder, pos * 2);
2787
2788                                 value_builder[pos++] = (char) c;
2789                         }
2790                 }
2791
2792                 private int consume_identifier (int s)
2793                 {
2794                         int res = consume_identifier (s, false);
2795
2796                         if (doc_state == XmlCommentState.Allowed)
2797                                 doc_state = XmlCommentState.NotAllowed;
2798
2799                         return res;
2800                 }
2801
2802                 int consume_identifier (int c, bool quoted)
2803                 {
2804                         //
2805                         // This method is very performance sensitive. It accounts
2806                         // for approximately 25% of all parser time
2807                         //
2808
2809                         int pos = 0;
2810                         int column = col;
2811                         if (quoted)
2812                                 --column;
2813
2814                         if (c == '\\') {
2815                                 int surrogate;
2816                                 c = escape (c, out surrogate);
2817                                 if (surrogate != 0) {
2818                                         id_builder [pos++] = (char) c;
2819                                         c = surrogate;
2820                                 }
2821                         }
2822
2823                         id_builder [pos++] = (char) c;
2824
2825                         try {
2826                                 while (true) {
2827                                         c = reader.Read ();
2828
2829                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
2830                                                 id_builder [pos++] = (char) c;
2831                                                 continue;
2832                                         }
2833
2834                                         if (c < 0x80) {
2835                                                 if (c == '\\') {
2836                                                         int surrogate;
2837                                                         c = escape (c, out surrogate);
2838                                                         if (is_identifier_part_character ((char) c))
2839                                                                 id_builder[pos++] = (char) c;
2840
2841                                                         if (surrogate != 0) {
2842                                                                 c = surrogate;
2843                                                         }
2844
2845                                                         continue;
2846                                                 }
2847                                         } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
2848                                                 id_builder [pos++] = (char) c;
2849                                                 continue;
2850                                         }
2851
2852                                         putback_char = c;
2853                                         break;
2854                                 }
2855                         } catch (IndexOutOfRangeException) {
2856                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2857                                 --pos;
2858                                 col += pos;
2859                         }
2860
2861                         col += pos - 1;
2862
2863                         //
2864                         // Optimization: avoids doing the keyword lookup
2865                         // on uppercase letters
2866                         //
2867                         if (id_builder [0] >= '_' && !quoted) {
2868                                 int keyword = GetKeyword (id_builder, pos);
2869                                 if (keyword != -1) {
2870                                         val = LocatedToken.Create (keyword == Token.AWAIT ? "await" : null, ref_line, column);
2871                                         return keyword;
2872                                 }
2873                         }
2874
2875                         string s = InternIdentifier (id_builder, pos);
2876                         val = LocatedToken.Create (s, ref_line, column);
2877                         if (quoted && parsing_attribute_section)
2878                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2879
2880                         return Token.IDENTIFIER;
2881                 }
2882
2883                 string InternIdentifier (char[] charBuffer, int length)
2884                 {
2885                         //
2886                         // Keep identifiers in an array of hashtables to avoid needless
2887                         // allocations
2888                         //
2889                         var identifiers_group = identifiers[length];
2890                         string s;
2891                         if (identifiers_group != null) {
2892                                 if (identifiers_group.TryGetValue (charBuffer, out s)) {
2893                                         return s;
2894                                 }
2895                         } else {
2896                                 // TODO: this should be number of files dependant
2897                                 // corlib compilation peaks at 1000 and System.Core at 150
2898                                 int capacity = length > 20 ? 10 : 100;
2899                                 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length));
2900                                 identifiers[length] = identifiers_group;
2901                         }
2902
2903                         char[] chars = new char[length];
2904                         Array.Copy (charBuffer, chars, length);
2905
2906                         s = new string (charBuffer, 0, length);
2907                         identifiers_group.Add (chars, s);
2908                         return s;
2909                 }
2910
2911                 public int xtoken ()
2912                 {
2913                         int d, c;
2914
2915                         // Whether we have seen comments on the current line
2916                         bool comments_seen = false;
2917                         while ((c = get_char ()) != -1) {
2918                                 switch (c) {
2919                                 case '\t':
2920                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
2921                                         continue;
2922
2923                                 case ' ':
2924                                 case '\f':
2925                                 case '\v':
2926                                 case 0xa0:
2927                                 case 0:
2928                                 case 0xFEFF:    // Ignore BOM anywhere in the file
2929                                         continue;
2930
2931 /*                              This is required for compatibility with .NET
2932                                 case 0xEF:
2933                                         if (peek_char () == 0xBB) {
2934                                                 PushPosition ();
2935                                                 get_char ();
2936                                                 if (get_char () == 0xBF)
2937                                                         continue;
2938                                                 PopPosition ();
2939                                         }
2940                                         break;
2941 */
2942                                 case '\\':
2943                                         tokens_seen = true;
2944                                         return consume_identifier (c);
2945
2946                                 case '{':
2947                                         val = LocatedToken.Create (ref_line, col);
2948                                         return Token.OPEN_BRACE;
2949                                 case '}':
2950                                         val = LocatedToken.Create (ref_line, col);
2951                                         return Token.CLOSE_BRACE;
2952                                 case '[':
2953                                         // To block doccomment inside attribute declaration.
2954                                         if (doc_state == XmlCommentState.Allowed)
2955                                                 doc_state = XmlCommentState.NotAllowed;
2956
2957                                         val = LocatedToken.Create (ref_line, col);
2958
2959                                         if (parsing_block == 0 || lambda_arguments_parsing)
2960                                                 return Token.OPEN_BRACKET;
2961
2962                                         int next = peek_char ();
2963                                         switch (next) {
2964                                         case ']':
2965                                         case ',':
2966                                                 return Token.OPEN_BRACKET;
2967
2968                                         case ' ':
2969                                         case '\f':
2970                                         case '\v':
2971                                         case '\r':
2972                                         case '\n':
2973                                         case '/':
2974                                                 next = peek_token ();
2975                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
2976                                                         return Token.OPEN_BRACKET;
2977
2978                                                 return Token.OPEN_BRACKET_EXPR;
2979                                         default:
2980                                                 return Token.OPEN_BRACKET_EXPR;
2981                                         }
2982                                 case ']':
2983                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2984                                         return Token.CLOSE_BRACKET;
2985                                 case '(':
2986                                         val = LocatedToken.Create (ref_line, col);
2987                                         //
2988                                         // An expression versions of parens can appear in block context only
2989                                         //
2990                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
2991
2992                                                 //
2993                                                 // Optmize most common case where we know that parens
2994                                                 // is not special
2995                                                 //
2996                                                 switch (current_token) {
2997                                                 case Token.IDENTIFIER:
2998                                                 case Token.IF:
2999                                                 case Token.FOR:
3000                                                 case Token.FOREACH:
3001                                                 case Token.TYPEOF:
3002                                                 case Token.WHILE:
3003                                                 case Token.USING:
3004                                                 case Token.DEFAULT:
3005                                                 case Token.DELEGATE:
3006                                                 case Token.OP_GENERICS_GT:
3007                                                         return Token.OPEN_PARENS;
3008                                                 }
3009
3010                                                 // Optimize using peek
3011                                                 int xx = peek_char ();
3012                                                 switch (xx) {
3013                                                 case '(':
3014                                                 case '\'':
3015                                                 case '"':
3016                                                 case '0':
3017                                                 case '1':
3018                                                         return Token.OPEN_PARENS;
3019                                                 }
3020
3021                                                 lambda_arguments_parsing = true;
3022                                                 PushPosition ();
3023                                                 d = TokenizeOpenParens ();
3024                                                 PopPosition ();
3025                                                 lambda_arguments_parsing = false;
3026                                                 return d;
3027                                         }
3028
3029                                         return Token.OPEN_PARENS;
3030                                 case ')':
3031                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3032                                         return Token.CLOSE_PARENS;
3033                                 case ',':
3034                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3035                                         return Token.COMMA;
3036                                 case ';':
3037                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3038                                         return Token.SEMICOLON;
3039                                 case '~':
3040                                         val = LocatedToken.Create (ref_line, col);
3041                                         return Token.TILDE;
3042                                 case '?':
3043                                         val = LocatedToken.Create (ref_line, col);
3044                                         return TokenizePossibleNullableType ();
3045                                 case '<':
3046                                         val = LocatedToken.Create (ref_line, col);
3047                                         if (parsing_generic_less_than++ > 0)
3048                                                 return Token.OP_GENERICS_LT;
3049
3050                                         return TokenizeLessThan ();
3051
3052                                 case '>':
3053                                         val = LocatedToken.Create (ref_line, col);
3054                                         d = peek_char ();
3055
3056                                         if (d == '='){
3057                                                 get_char ();
3058                                                 return Token.OP_GE;
3059                                         }
3060
3061                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
3062                                                 parsing_generic_less_than--;
3063                                                 return Token.OP_GENERICS_GT;
3064                                         }
3065
3066                                         if (d == '>') {
3067                                                 get_char ();
3068                                                 d = peek_char ();
3069
3070                                                 if (d == '=') {
3071                                                         get_char ();
3072                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
3073                                                 }
3074                                                 return Token.OP_SHIFT_RIGHT;
3075                                         }
3076
3077                                         return Token.OP_GT;
3078
3079                                 case '+':
3080                                         val = LocatedToken.Create (ref_line, col);
3081                                         d = peek_char ();
3082                                         if (d == '+') {
3083                                                 d = Token.OP_INC;
3084                                         } else if (d == '=') {
3085                                                 d = Token.OP_ADD_ASSIGN;
3086                                         } else {
3087                                                 return Token.PLUS;
3088                                         }
3089                                         get_char ();
3090                                         return d;
3091
3092                                 case '-':
3093                                         val = LocatedToken.Create (ref_line, col);
3094                                         d = peek_char ();
3095                                         if (d == '-') {
3096                                                 d = Token.OP_DEC;
3097                                         } else if (d == '=')
3098                                                 d = Token.OP_SUB_ASSIGN;
3099                                         else if (d == '>')
3100                                                 d = Token.OP_PTR;
3101                                         else {
3102                                                 return Token.MINUS;
3103                                         }
3104                                         get_char ();
3105                                         return d;
3106
3107                                 case '!':
3108                                         val = LocatedToken.Create (ref_line, col);
3109                                         if (peek_char () == '='){
3110                                                 get_char ();
3111                                                 return Token.OP_NE;
3112                                         }
3113                                         return Token.BANG;
3114
3115                                 case '=':
3116                                         val = LocatedToken.Create (ref_line, col);
3117                                         d = peek_char ();
3118                                         if (d == '='){
3119                                                 get_char ();
3120                                                 return Token.OP_EQ;
3121                                         }
3122                                         if (d == '>'){
3123                                                 get_char ();
3124                                                 return Token.ARROW;
3125                                         }
3126
3127                                         return Token.ASSIGN;
3128
3129                                 case '&':
3130                                         val = LocatedToken.Create (ref_line, col);
3131                                         d = peek_char ();
3132                                         if (d == '&'){
3133                                                 get_char ();
3134                                                 return Token.OP_AND;
3135                                         }
3136                                         if (d == '='){
3137                                                 get_char ();
3138                                                 return Token.OP_AND_ASSIGN;
3139                                         }
3140                                         return Token.BITWISE_AND;
3141
3142                                 case '|':
3143                                         val = LocatedToken.Create (ref_line, col);
3144                                         d = peek_char ();
3145                                         if (d == '|'){
3146                                                 get_char ();
3147                                                 return Token.OP_OR;
3148                                         }
3149                                         if (d == '='){
3150                                                 get_char ();
3151                                                 return Token.OP_OR_ASSIGN;
3152                                         }
3153                                         return Token.BITWISE_OR;
3154
3155                                 case '*':
3156                                         val = LocatedToken.Create (ref_line, col);
3157                                         if (peek_char () == '='){
3158                                                 get_char ();
3159                                                 return Token.OP_MULT_ASSIGN;
3160                                         }
3161                                         return Token.STAR;
3162
3163                                 case '/':
3164                                         d = peek_char ();
3165                                         if (d == '='){
3166                                                 val = LocatedToken.Create (ref_line, col);
3167                                                 get_char ();
3168                                                 return Token.OP_DIV_ASSIGN;
3169                                         }
3170
3171                                         // Handle double-slash comments.
3172                                         if (d == '/'){
3173                                                 get_char ();
3174                                                 if (doc_processing) {
3175                                                         if (peek_char () == '/') {
3176                                                                 get_char ();
3177                                                                 // Don't allow ////.
3178                                                                 if ((d = peek_char ()) != '/') {
3179                                                                         if (doc_state == XmlCommentState.Allowed)
3180                                                                                 handle_one_line_xml_comment ();
3181                                                                         else if (doc_state == XmlCommentState.NotAllowed)
3182                                                                                 WarningMisplacedComment (Location - 3);
3183                                                                 }
3184                                                         } else {
3185                                                                 if (xml_comment_buffer.Length > 0)
3186                                                                         doc_state = XmlCommentState.NotAllowed;
3187                                                         }
3188                                                 }
3189
3190                                                 while ((d = get_char ()) != -1 && d != '\n');
3191
3192                                                 any_token_seen |= tokens_seen;
3193                                                 tokens_seen = false;
3194                                                 comments_seen = false;
3195                                                 continue;
3196                                         } else if (d == '*'){
3197                                                 get_char ();
3198                                                 bool docAppend = false;
3199                                                 if (doc_processing && peek_char () == '*') {
3200                                                         get_char ();
3201                                                         // But when it is /**/, just do nothing.
3202                                                         if (peek_char () == '/') {
3203                                                                 get_char ();
3204                                                                 continue;
3205                                                         }
3206                                                         if (doc_state == XmlCommentState.Allowed)
3207                                                                 docAppend = true;
3208                                                         else if (doc_state == XmlCommentState.NotAllowed) {
3209                                                                 WarningMisplacedComment (Location - 2);
3210                                                         }
3211                                                 }
3212
3213                                                 int current_comment_start = 0;
3214                                                 if (docAppend) {
3215                                                         current_comment_start = xml_comment_buffer.Length;
3216                                                         xml_comment_buffer.Append (Environment.NewLine);
3217                                                 }
3218
3219                                                 while ((d = get_char ()) != -1){
3220                                                         if (d == '*' && peek_char () == '/'){
3221                                                                 get_char ();
3222                                                                 comments_seen = true;
3223                                                                 break;
3224                                                         }
3225                                                         if (docAppend)
3226                                                                 xml_comment_buffer.Append ((char) d);
3227
3228                                                         if (d == '\n'){
3229                                                                 any_token_seen |= tokens_seen;
3230                                                                 tokens_seen = false;
3231                                                                 //
3232                                                                 // Reset 'comments_seen' just to be consistent.
3233                                                                 // It doesn't matter either way, here.
3234                                                                 //
3235                                                                 comments_seen = false;
3236                                                         }
3237                                                 }
3238                                                 if (!comments_seen)
3239                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3240
3241                                                 if (docAppend)
3242                                                         update_formatted_doc_comment (current_comment_start);
3243                                                 continue;
3244                                         }
3245                                         val = LocatedToken.Create (ref_line, col);
3246                                         return Token.DIV;
3247
3248                                 case '%':
3249                                         val = LocatedToken.Create (ref_line, col);
3250                                         if (peek_char () == '='){
3251                                                 get_char ();
3252                                                 return Token.OP_MOD_ASSIGN;
3253                                         }
3254                                         return Token.PERCENT;
3255
3256                                 case '^':
3257                                         val = LocatedToken.Create (ref_line, col);
3258                                         if (peek_char () == '='){
3259                                                 get_char ();
3260                                                 return Token.OP_XOR_ASSIGN;
3261                                         }
3262                                         return Token.CARRET;
3263
3264                                 case ':':
3265                                         val = LocatedToken.Create (ref_line, col);
3266                                         if (peek_char () == ':') {
3267                                                 get_char ();
3268                                                 return Token.DOUBLE_COLON;
3269                                         }
3270                                         return Token.COLON;
3271
3272                                 case '0': case '1': case '2': case '3': case '4':
3273                                 case '5': case '6': case '7': case '8': case '9':
3274                                         tokens_seen = true;
3275                                         return is_number (c);
3276
3277                                 case '\n': // white space
3278                                         any_token_seen |= tokens_seen;
3279                                         tokens_seen = false;
3280                                         comments_seen = false;
3281                                         continue;
3282
3283                                 case '.':
3284                                         tokens_seen = true;
3285                                         d = peek_char ();
3286                                         if (d >= '0' && d <= '9')
3287                                                 return is_number (c);
3288
3289                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3290                                         return Token.DOT;
3291
3292                                 case '#':
3293                                         if (tokens_seen || comments_seen) {
3294                                                 Eror_WrongPreprocessorLocation ();
3295                                                 return Token.ERROR;
3296                                         }
3297
3298                                         if (ParsePreprocessingDirective (true))
3299                                                 continue;
3300
3301                                         bool directive_expected = false;
3302                                         while ((c = get_char ()) != -1) {
3303                                                 if (col == 1) {
3304                                                         directive_expected = true;
3305                                                 } else if (!directive_expected) {
3306                                                         // TODO: Implement comment support for disabled code and uncomment this code
3307 //                                                      if (c == '#') {
3308 //                                                              Eror_WrongPreprocessorLocation ();
3309 //                                                              return Token.ERROR;
3310 //                                                      }
3311                                                         continue;
3312                                                 }
3313
3314                                                 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' )
3315                                                         continue;
3316
3317                                                 if (c == '#') {
3318                                                         if (ParsePreprocessingDirective (false))
3319                                                                 break;
3320                                                 }
3321                                                 directive_expected = false;
3322                                         }
3323
3324                                         if (c != -1) {
3325                                                 tokens_seen = false;
3326                                                 continue;
3327                                         }
3328
3329                                         return Token.EOF;
3330
3331                                 case '"':
3332                                         return consume_string (false);
3333
3334                                 case '\'':
3335                                         return TokenizeBackslash ();
3336
3337                                 case '@':
3338                                         c = get_char ();
3339                                         if (c == '"') {
3340                                                 tokens_seen = true;
3341                                                 return consume_string (true);
3342                                         }
3343
3344                                         if (is_identifier_start_character (c)){
3345                                                 return consume_identifier (c, true);
3346                                         }
3347
3348                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3349                                         return Token.ERROR;
3350
3351                                 case EvalStatementParserCharacter:
3352                                         return Token.EVAL_STATEMENT_PARSER;
3353                                 case EvalCompilationUnitParserCharacter:
3354                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3355                                 case EvalUsingDeclarationsParserCharacter:
3356                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3357                                 case DocumentationXref:
3358                                         return Token.DOC_SEE;
3359                                 }
3360
3361                                 if (is_identifier_start_character (c)) {
3362                                         tokens_seen = true;
3363                                         return consume_identifier (c);
3364                                 }
3365
3366                                 if (char.IsWhiteSpace ((char) c))
3367                                         continue;
3368
3369                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3370                         }
3371
3372                         if (CompleteOnEOF){
3373                                 if (generated)
3374                                         return Token.COMPLETE_COMPLETION;
3375
3376                                 generated = true;
3377                                 return Token.GENERATE_COMPLETION;
3378                         }
3379
3380
3381                         return Token.EOF;
3382                 }
3383
3384                 int TokenizeBackslash ()
3385                 {
3386 #if FULL_AST
3387                         int read_start = reader.Position;
3388 #endif
3389                         Location start_location = Location;
3390                         int c = get_char ();
3391                         tokens_seen = true;
3392                         if (c == '\'') {
3393                                 val = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3394                                 Report.Error (1011, start_location, "Empty character literal");
3395                                 return Token.LITERAL;
3396                         }
3397
3398                         if (c == '\n') {
3399                                 Report.Error (1010, start_location, "Newline in constant");
3400                                 return Token.ERROR;
3401                         }
3402
3403                         int d;
3404                         c = escape (c, out d);
3405                         if (c == -1)
3406                                 return Token.ERROR;
3407                         if (d != 0)
3408                                 throw new NotImplementedException ();
3409
3410                         ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3411                         val = res;
3412                         c = get_char ();
3413
3414                         if (c != '\'') {
3415                                 Report.Error (1012, start_location, "Too many characters in character literal");
3416
3417                                 // Try to recover, read until newline or next "'"
3418                                 while ((c = get_char ()) != -1) {
3419                                         if (c == '\n' || c == '\'')
3420                                                 break;
3421                                 }
3422                         }
3423
3424 #if FULL_AST
3425                         res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position);
3426 #endif
3427
3428                         return Token.LITERAL;
3429                 }
3430
3431                 int TokenizeLessThan ()
3432                 {
3433                         int d;
3434                         if (handle_typeof) {
3435                                 PushPosition ();
3436                                 if (parse_generic_dimension (out d)) {
3437                                         val = d;
3438                                         DiscardPosition ();
3439                                         return Token.GENERIC_DIMENSION;
3440                                 }
3441                                 PopPosition ();
3442                         }
3443
3444                         // Save current position and parse next token.
3445                         PushPosition ();
3446                         if (parse_less_than ()) {
3447                                 if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) {
3448                                         d = Token.OP_GENERICS_LT_DECL;
3449                                 } else {
3450                                         d = Token.OP_GENERICS_LT;
3451                                 }
3452                                 PopPosition ();
3453                                 return d;
3454                         }
3455
3456                         PopPosition ();
3457                         parsing_generic_less_than = 0;
3458
3459                         d = peek_char ();
3460                         if (d == '<') {
3461                                 get_char ();
3462                                 d = peek_char ();
3463
3464                                 if (d == '=') {
3465                                         get_char ();
3466                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3467                                 }
3468                                 return Token.OP_SHIFT_LEFT;
3469                         }
3470
3471                         if (d == '=') {
3472                                 get_char ();
3473                                 return Token.OP_LE;
3474                         }
3475                         return Token.OP_LT;
3476                 }
3477
3478                 //
3479                 // Handles one line xml comment
3480                 //
3481                 private void handle_one_line_xml_comment ()
3482                 {
3483                         int c;
3484                         while ((c = peek_char ()) == ' ')
3485                                 get_char (); // skip heading whitespaces.
3486                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3487                                 xml_comment_buffer.Append ((char) get_char ());
3488                         }
3489                         if (c == '\r' || c == '\n')
3490                                 xml_comment_buffer.Append (Environment.NewLine);
3491                 }
3492
3493                 //
3494                 // Remove heading "*" in Javadoc-like xml documentation.
3495                 //
3496                 private void update_formatted_doc_comment (int current_comment_start)
3497                 {
3498                         int length = xml_comment_buffer.Length - current_comment_start;
3499                         string [] lines = xml_comment_buffer.ToString (
3500                                 current_comment_start,
3501                                 length).Replace ("\r", "").Split ('\n');
3502
3503                         // The first line starts with /**, thus it is not target
3504                         // for the format check.
3505                         for (int i = 1; i < lines.Length; i++) {
3506                                 string s = lines [i];
3507                                 int idx = s.IndexOf ('*');
3508                                 string head = null;
3509                                 if (idx < 0) {
3510                                         if (i < lines.Length - 1)
3511                                                 return;
3512                                         head = s;
3513                                 } else
3514                                         head = s.Substring (0, idx);
3515                                 foreach (char c in head)
3516                                         if (c != ' ')
3517                                                 return;
3518                                 lines [i] = s.Substring (idx + 1);
3519                         }
3520                         xml_comment_buffer.Remove (current_comment_start, length);
3521                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3522                 }
3523
3524                 //
3525                 // Checks if there was incorrect doc comments and raise
3526                 // warnings.
3527                 //
3528                 public void check_incorrect_doc_comment ()
3529                 {
3530                         if (xml_comment_buffer.Length > 0)
3531                                 WarningMisplacedComment (Location);
3532                 }
3533
3534                 //
3535                 // Consumes the saved xml comment lines (if any)
3536                 // as for current target member or type.
3537                 //
3538                 public string consume_doc_comment ()
3539                 {
3540                         if (xml_comment_buffer.Length > 0) {
3541                                 string ret = xml_comment_buffer.ToString ();
3542                                 reset_doc_comment ();
3543                                 return ret;
3544                         }
3545                         return null;
3546                 }
3547
3548                 Report Report {
3549                         get { return context.Report; }
3550                 }
3551
3552                 void reset_doc_comment ()
3553                 {
3554                         xml_comment_buffer.Length = 0;
3555                 }
3556
3557                 public void cleanup ()
3558                 {
3559                         if (ifstack != null && ifstack.Count >= 1) {
3560                                 int state = ifstack.Pop ();
3561                                 if ((state & REGION) != 0)
3562                                         Report.Error (1038, Location, "#endregion directive expected");
3563                                 else
3564                                         Report.Error (1027, Location, "Expected `#endif' directive");
3565                         }
3566                 }
3567         }
3568
3569         //
3570         // Indicates whether it accepts XML documentation or not.
3571         //
3572         public enum XmlCommentState {
3573                 // comment is allowed in this state.
3574                 Allowed,
3575                 // comment is not allowed in this state.
3576                 NotAllowed,
3577                 // once comments appeared when it is NotAllowed, then the
3578                 // state is changed to it, until the state is changed to
3579                 // .Allowed.
3580                 Error
3581         }
3582 }
3583