mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@gmail.com)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 // Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20 using System.Collections;
  21
  22 namespace Mono.CSharp
  23 {
  24         //
  25         // This class has to be used by parser only, it reuses token
  26         // details once a file is parsed
  27         //
  28         public class LocatedToken
  29         {
  30                 public int row, column;
  31                 public string value;
  32                 public SourceFile file;
  33
  34                 public LocatedToken ()
  35                 {
  36                 }
  37
  38                 public LocatedToken (string value, Location loc)
  39                 {
  40                         this.value = value;
  41                         file = loc.SourceFile;
  42                         row = loc.Row;
  43                         column = loc.Column;
  44                 }
  45
  46                 public override string ToString ()
  47                 {
  48                         return string.Format ("Token '{0}' at {1},{2}", Value, row, column);
  49                 }
  50
  51                 public Location Location
  52                 {
  53                         get { return new Location (file, row, column); }
  54                 }
  55
  56                 public string Value
  57                 {
  58                         get { return value; }
  59                 }
  60         }
  61
  62         /// <summary>
  63         ///    Tokenizer for C# source code.
  64         /// </summary>
  65         public class Tokenizer : yyParser.yyInput
  66         {
  67                 class KeywordEntry<T>
  68                 {
  69                         public readonly T Token;
  70                         public KeywordEntry<T> Next;
  71                         public readonly char[] Value;
  72
  73                         public KeywordEntry (string value, T token)
  74                         {
  75                                 this.Value = value.ToCharArray ();
  76                                 this.Token = token;
  77                         }
  78                 }
  79
  80                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  81                 {
  82                         readonly int length;
  83
  84                         public IdentifiersComparer (int length)
  85                         {
  86                                 this.length = length;
  87                         }
  88
  89                         public bool Equals (char[] x, char[] y)
  90                         {
  91                                 for (int i = 0; i < length; ++i)
  92                                         if (x [i] != y [i])
  93                                                 return false;
  94
  95                                 return true;
  96                         }
  97
  98                         public int GetHashCode (char[] obj)
  99                         {
 100                                 int h = 0;
 101                                 for (int i = 0; i < length; ++i)
 102                                         h = (h << 5) - h + obj [i];
 103
 104                                 return h;
 105                         }
 106                 }
 107
 108                 public class LocatedTokenBuffer
 109                 {
 110                         readonly LocatedToken[] buffer;
 111                         public int pos;
 112
 113                         public LocatedTokenBuffer ()
 114                         {
 115                                 buffer = new LocatedToken[0];
 116                         }
 117
 118                         public LocatedTokenBuffer (LocatedToken[] buffer)
 119                         {
 120                                 this.buffer = buffer ?? new LocatedToken[0];
 121                         }
 122
 123                         public LocatedToken Create (SourceFile file, int row, int column)
 124                         {
 125                                 return Create (null, file, row, column);
 126                         }
 127
 128                         public LocatedToken Create (string value, SourceFile file, int row, int column)
 129                         {
 130                                 //
 131                                 // TODO: I am not very happy about the logic but it's the best
 132                                 // what I could come up with for now.
 133                                 // Ideally we should be using just tiny buffer (256 elements) which
 134                                 // is enough to hold all details for currect stack and recycle elements
 135                                 // poped from the stack but there is a trick needed to recycle
 136                                 // them properly.
 137                                 //
 138                                 LocatedToken entry;
 139                                 if (pos >= buffer.Length) {
 140                                         entry = new LocatedToken ();
 141                                 } else {
 142                                         entry = buffer[pos];
 143                                         if (entry == null) {
 144                                                 entry = new LocatedToken ();
 145                                                 buffer[pos] = entry;
 146                                         }
 147
 148                                         ++pos;
 149                                 }
 150                                 entry.value = value;
 151                                 entry.file = file;
 152                                 entry.row = row;
 153                                 entry.column = column;
 154                                 return entry;
 155                         }
 156
 157                         //
 158                         // Used for token not required by expression evaluator
 159                         //
 160                         [Conditional ("FULL_AST")]
 161                         public void CreateOptional (SourceFile file, int row, int col, ref object token)
 162                         {
 163                                 token = Create (file, row, col);
 164                         }
 165                 }
 166
 167                 public enum PreprocessorDirective
 168                 {
 169                         Invalid = 0,
 170
 171                         Region = 1,
 172                         Endregion = 2,
 173                         If = 3 | RequiresArgument,
 174                         Endif = 4,
 175                         Elif = 5 | RequiresArgument,
 176                         Else = 6,
 177                         Define = 7 | RequiresArgument,
 178                         Undef = 8 | RequiresArgument,
 179                         Error = 9,
 180                         Warning = 10,
 181                         Pragma = 11 | CustomArgumentsParsing,
 182                         Line = 12 | CustomArgumentsParsing,
 183
 184                         CustomArgumentsParsing = 1 << 10,
 185                         RequiresArgument = 1 << 11
 186                 }
 187
 188                 readonly SeekableStreamReader reader;
 189                 readonly CompilationSourceFile source_file;
 190                 readonly CompilerContext context;
 191                 readonly Report Report;
 192
 193
 194                 SourceFile current_source;
 195                 Location hidden_block_start;
 196                 int ref_line = 1;
 197                 int line = 1;
 198                 int col = 0;
 199                 int previous_col;
 200                 int current_token;
 201                 readonly int tab_size;
 202                 bool handle_get_set = false;
 203                 bool handle_remove_add = false;
 204                 bool handle_where;
 205                 bool lambda_arguments_parsing;
 206                 List<Location> escaped_identifiers;
 207                 int parsing_generic_less_than;
 208                 readonly bool doc_processing;
 209                 readonly LocatedTokenBuffer ltb;
 210
 211                 //
 212                 // Used mainly for parser optimizations. Some expressions for instance
 213                 // can appear only in block (including initializer, base initializer)
 214                 // scope only
 215                 //
 216                 public int parsing_block;
 217                 internal bool query_parsing;
 218
 219                 //
 220                 // When parsing type only, useful for ambiguous nullable types
 221                 //
 222                 public int parsing_type;
 223
 224                 //
 225                 // Set when parsing generic declaration (type or method header)
 226                 //
 227                 public bool parsing_generic_declaration;
 228                 public bool parsing_generic_declaration_doc;
 229
 230                 //
 231                 // The value indicates that we have not reach any declaration or
 232                 // namespace yet
 233                 //
 234                 public int parsing_declaration;
 235
 236                 public bool parsing_attribute_section;
 237
 238                 public bool parsing_modifiers;
 239
 240                 public bool parsing_catch_when;
 241
 242                 int parsing_string_interpolation;
 243
 244                 public bool parsing_interpolation_format;
 245
 246                 //
 247                 // The special characters to inject on streams to run the unit parser
 248                 // in the special expression mode. Using private characters from
 249                 // Plane Sixteen (U+100000 to U+10FFFD)
 250                 //
 251                 // This character is only tested just before the tokenizer is about to report
 252                 // an error;   So on the regular operation mode, this addition will have no
 253                 // impact on the tokenizer's performance.
 254                 //
 255
 256                 public const int EvalStatementParserCharacter = 0x100000;
 257                 public const int EvalCompilationUnitParserCharacter = 0x100001;
 258                 public const int EvalUsingDeclarationsParserCharacter = 0x100002;
 259                 public const int DocumentationXref = 0x100003;
 260
 261                 const int UnicodeLS = 0x2028;
 262                 const int UnicodePS = 0x2029;
 263
 264                 //
 265                 // XML documentation buffer. The save point is used to divide
 266                 // comments on types and comments on members.
 267                 //
 268                 StringBuilder xml_comment_buffer;
 269
 270                 //
 271                 // See comment on XmlCommentState enumeration.
 272                 //
 273                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 274
 275                 //
 276                 // Whether tokens have been seen on this line
 277                 //
 278                 bool tokens_seen = false;
 279
 280                 //
 281                 // Set to true once the GENERATE_COMPLETION token has bee
 282                 // returned.   This helps produce one GENERATE_COMPLETION,
 283                 // as many COMPLETE_COMPLETION as necessary to complete the
 284                 // AST tree and one final EOF.
 285                 //
 286                 bool generated;
 287
 288                 //
 289                 // Whether a token has been seen on the file
 290                 // This is needed because `define' is not allowed to be used
 291                 // after a token has been seen.
 292                 //
 293                 bool any_token_seen;
 294
 295                 //
 296                 // Class variables
 297                 //
 298                 static readonly KeywordEntry<int>[][] keywords;
 299                 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 300                 static readonly HashSet<string> keyword_strings;
 301                 static readonly NumberStyles styles;
 302                 static readonly NumberFormatInfo csharp_format_info;
 303
 304                 // Pragma arguments
 305                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 306                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 307                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 308                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 309                 static readonly char[] line_hidden = "hidden".ToCharArray ();
 310                 static readonly char[] line_default = "default".ToCharArray ();
 311
 312                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 313
 314                 public bool PropertyParsing {
 315                         get { return handle_get_set; }
 316                         set { handle_get_set = value; }
 317                 }
 318
 319                 public bool EventParsing {
 320                         get { return handle_remove_add; }
 321                         set { handle_remove_add = value; }
 322                 }
 323
 324                 public bool ConstraintsParsing {
 325                         get { return handle_where; }
 326                         set { handle_where = value; }
 327                 }
 328
 329                 public XmlCommentState doc_state {
 330                         get { return xml_doc_state; }
 331                         set {
 332                                 if (value == XmlCommentState.Allowed) {
 333                                         check_incorrect_doc_comment ();
 334                                         reset_doc_comment ();
 335                                 }
 336                                 xml_doc_state = value;
 337                         }
 338                 }
 339
 340                 //
 341                 // This is used to trigger completion generation on the parser
 342                 public bool CompleteOnEOF;
 343
 344                 void AddEscapedIdentifier (Location loc)
 345                 {
 346                         if (escaped_identifiers == null)
 347                                 escaped_identifiers = new List<Location> ();
 348
 349                         escaped_identifiers.Add (loc);
 350                 }
 351
 352                 public bool IsEscapedIdentifier (ATypeNameExpression name)
 353                 {
 354                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 355                 }
 356
 357                 //
 358                 // Values for the associated token returned
 359                 //
 360                 internal int putback_char;      // Used by repl only
 361                 object val;
 362
 363                 //
 364                 // Pre-processor
 365                 //
 366                 const int TAKING        = 1;
 367                 const int ELSE_SEEN     = 4;
 368                 const int PARENT_TAKING = 8;
 369                 const int REGION        = 16;
 370
 371                 //
 372                 // pre-processor if stack state:
 373                 //
 374                 Stack<int> ifstack;
 375
 376                 public const int MaxIdentifierLength = 512;
 377                 public const int MaxNumberLength = 512;
 378
 379                 readonly char[] id_builder;
 380                 readonly Dictionary<char[], string>[] identifiers;
 381                 readonly char[] number_builder;
 382                 int number_pos;
 383
 384                 char[] value_builder = new char[64];
 385
 386                 public int Line {
 387                         get {
 388                                 return ref_line;
 389                         }
 390                 }
 391
 392                 //
 393                 // This is used when the tokenizer needs to save
 394                 // the current position as it needs to do some parsing
 395                 // on its own to deamiguate a token in behalf of the
 396                 // parser.
 397                 //
 398                 Stack<Position> position_stack = new Stack<Position> (2);
 399
 400                 class Position {
 401                         public int position;
 402                         public int line;
 403                         public int ref_line;
 404                         public int col;
 405                         public Location hidden;
 406                         public int putback_char;
 407                         public int previous_col;
 408                         public Stack<int> ifstack;
 409                         public int parsing_generic_less_than;
 410                         public int current_token;
 411                         public object val;
 412
 413                         public Position (Tokenizer t)
 414                         {
 415                                 position = t.reader.Position;
 416                                 line = t.line;
 417                                 ref_line = t.ref_line;
 418                                 col = t.col;
 419                                 hidden = t.hidden_block_start;
 420                                 putback_char = t.putback_char;
 421                                 previous_col = t.previous_col;
 422                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 423                                         // There is no simple way to clone Stack<T> all
 424                                         // methods reverse the order
 425                                         var clone = t.ifstack.ToArray ();
 426                                         Array.Reverse (clone);
 427                                         ifstack = new Stack<int> (clone);
 428                                 }
 429                                 parsing_generic_less_than = t.parsing_generic_less_than;
 430                                 current_token = t.current_token;
 431                                 val = t.val;
 432                         }
 433                 }
 434
 435                 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, ParserSession session, Report report)
 436                 {
 437                         this.source_file = file;
 438                         this.context = file.Compiler;
 439                         this.current_source = file.SourceFile;
 440                         this.identifiers = session.Identifiers;
 441                         this.id_builder = session.IDBuilder;
 442                         this.number_builder = session.NumberBuilder;
 443                         this.ltb = new LocatedTokenBuffer (session.LocatedTokens);
 444                         this.Report = report;
 445
 446                         reader = input;
 447
 448                         putback_char = -1;
 449
 450                         xml_comment_buffer = new StringBuilder ();
 451                         doc_processing = context.Settings.DocumentationFile != null;
 452
 453                         tab_size = context.Settings.TabSize;
 454                 }
 455
 456                 public void PushPosition ()
 457                 {
 458                         position_stack.Push (new Position (this));
 459                 }
 460
 461                 public void PopPosition ()
 462                 {
 463                         Position p = position_stack.Pop ();
 464
 465                         reader.Position = p.position;
 466                         ref_line = p.ref_line;
 467                         line = p.line;
 468                         col = p.col;
 469                         hidden_block_start = p.hidden;
 470                         putback_char = p.putback_char;
 471                         previous_col = p.previous_col;
 472                         ifstack = p.ifstack;
 473                         parsing_generic_less_than = p.parsing_generic_less_than;
 474                         current_token = p.current_token;
 475                         val = p.val;
 476                 }
 477
 478                 // Do not reset the position, ignore it.
 479                 public void DiscardPosition ()
 480                 {
 481                         position_stack.Pop ();
 482                 }
 483
 484                 static void AddKeyword (string kw, int token)
 485                 {
 486                         keyword_strings.Add (kw);
 487
 488                         AddKeyword (keywords, kw, token);
 489                 }
 490
 491                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 492                 {
 493                         AddKeyword (keywords_preprocessor, kw, directive);
 494                 }
 495
 496                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 497                 {
 498                         int length = kw.Length;
 499                         if (keywords[length] == null) {
 500                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 501                         }
 502
 503                         int char_index = kw[0] - '_';
 504                         var kwe = keywords[length][char_index];
 505                         if (kwe == null) {
 506                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 507                                 return;
 508                         }
 509
 510                         while (kwe.Next != null) {
 511                                 kwe = kwe.Next;
 512                         }
 513
 514                         kwe.Next = new KeywordEntry<T> (kw, token);
 515                 }
 516
 517                 //
 518                 // Class initializer
 519                 //
 520                 static Tokenizer ()
 521                 {
 522                         keyword_strings = new HashSet<string> ();
 523
 524                         // 11 is the length of the longest keyword for now
 525                         keywords = new KeywordEntry<int>[11][];
 526
 527                         AddKeyword ("__arglist", Token.ARGLIST);
 528                         AddKeyword ("__makeref", Token.MAKEREF);
 529                         AddKeyword ("__reftype", Token.REFTYPE);
 530                         AddKeyword ("__refvalue", Token.REFVALUE);
 531                         AddKeyword ("abstract", Token.ABSTRACT);
 532                         AddKeyword ("as", Token.AS);
 533                         AddKeyword ("add", Token.ADD);
 534                         AddKeyword ("base", Token.BASE);
 535                         AddKeyword ("bool", Token.BOOL);
 536                         AddKeyword ("break", Token.BREAK);
 537                         AddKeyword ("byte", Token.BYTE);
 538                         AddKeyword ("case", Token.CASE);
 539                         AddKeyword ("catch", Token.CATCH);
 540                         AddKeyword ("char", Token.CHAR);
 541                         AddKeyword ("checked", Token.CHECKED);
 542                         AddKeyword ("class", Token.CLASS);
 543                         AddKeyword ("const", Token.CONST);
 544                         AddKeyword ("continue", Token.CONTINUE);
 545                         AddKeyword ("decimal", Token.DECIMAL);
 546                         AddKeyword ("default", Token.DEFAULT);
 547                         AddKeyword ("delegate", Token.DELEGATE);
 548                         AddKeyword ("do", Token.DO);
 549                         AddKeyword ("double", Token.DOUBLE);
 550                         AddKeyword ("else", Token.ELSE);
 551                         AddKeyword ("enum", Token.ENUM);
 552                         AddKeyword ("event", Token.EVENT);
 553                         AddKeyword ("explicit", Token.EXPLICIT);
 554                         AddKeyword ("extern", Token.EXTERN);
 555                         AddKeyword ("false", Token.FALSE);
 556                         AddKeyword ("finally", Token.FINALLY);
 557                         AddKeyword ("fixed", Token.FIXED);
 558                         AddKeyword ("float", Token.FLOAT);
 559                         AddKeyword ("for", Token.FOR);
 560                         AddKeyword ("foreach", Token.FOREACH);
 561                         AddKeyword ("goto", Token.GOTO);
 562                         AddKeyword ("get", Token.GET);
 563                         AddKeyword ("if", Token.IF);
 564                         AddKeyword ("implicit", Token.IMPLICIT);
 565                         AddKeyword ("in", Token.IN);
 566                         AddKeyword ("int", Token.INT);
 567                         AddKeyword ("interface", Token.INTERFACE);
 568                         AddKeyword ("internal", Token.INTERNAL);
 569                         AddKeyword ("is", Token.IS);
 570                         AddKeyword ("lock", Token.LOCK);
 571                         AddKeyword ("long", Token.LONG);
 572                         AddKeyword ("namespace", Token.NAMESPACE);
 573                         AddKeyword ("new", Token.NEW);
 574                         AddKeyword ("null", Token.NULL);
 575                         AddKeyword ("object", Token.OBJECT);
 576                         AddKeyword ("operator", Token.OPERATOR);
 577                         AddKeyword ("out", Token.OUT);
 578                         AddKeyword ("override", Token.OVERRIDE);
 579                         AddKeyword ("params", Token.PARAMS);
 580                         AddKeyword ("private", Token.PRIVATE);
 581                         AddKeyword ("protected", Token.PROTECTED);
 582                         AddKeyword ("public", Token.PUBLIC);
 583                         AddKeyword ("readonly", Token.READONLY);
 584                         AddKeyword ("ref", Token.REF);
 585                         AddKeyword ("remove", Token.REMOVE);
 586                         AddKeyword ("return", Token.RETURN);
 587                         AddKeyword ("sbyte", Token.SBYTE);
 588                         AddKeyword ("sealed", Token.SEALED);
 589                         AddKeyword ("set", Token.SET);
 590                         AddKeyword ("short", Token.SHORT);
 591                         AddKeyword ("sizeof", Token.SIZEOF);
 592                         AddKeyword ("stackalloc", Token.STACKALLOC);
 593                         AddKeyword ("static", Token.STATIC);
 594                         AddKeyword ("string", Token.STRING);
 595                         AddKeyword ("struct", Token.STRUCT);
 596                         AddKeyword ("switch", Token.SWITCH);
 597                         AddKeyword ("this", Token.THIS);
 598                         AddKeyword ("throw", Token.THROW);
 599                         AddKeyword ("true", Token.TRUE);
 600                         AddKeyword ("try", Token.TRY);
 601                         AddKeyword ("typeof", Token.TYPEOF);
 602                         AddKeyword ("uint", Token.UINT);
 603                         AddKeyword ("ulong", Token.ULONG);
 604                         AddKeyword ("unchecked", Token.UNCHECKED);
 605                         AddKeyword ("unsafe", Token.UNSAFE);
 606                         AddKeyword ("ushort", Token.USHORT);
 607                         AddKeyword ("using", Token.USING);
 608                         AddKeyword ("virtual", Token.VIRTUAL);
 609                         AddKeyword ("void", Token.VOID);
 610                         AddKeyword ("volatile", Token.VOLATILE);
 611                         AddKeyword ("while", Token.WHILE);
 612                         AddKeyword ("partial", Token.PARTIAL);
 613                         AddKeyword ("where", Token.WHERE);
 614
 615                         // LINQ keywords
 616                         AddKeyword ("from", Token.FROM);
 617                         AddKeyword ("join", Token.JOIN);
 618                         AddKeyword ("on", Token.ON);
 619                         AddKeyword ("equals", Token.EQUALS);
 620                         AddKeyword ("select", Token.SELECT);
 621                         AddKeyword ("group", Token.GROUP);
 622                         AddKeyword ("by", Token.BY);
 623                         AddKeyword ("let", Token.LET);
 624                         AddKeyword ("orderby", Token.ORDERBY);
 625                         AddKeyword ("ascending", Token.ASCENDING);
 626                         AddKeyword ("descending", Token.DESCENDING);
 627                         AddKeyword ("into", Token.INTO);
 628
 629                         // Contextual async keywords
 630                         AddKeyword ("async", Token.ASYNC);
 631                         AddKeyword ("await", Token.AWAIT);
 632
 633                         // Contextual filter catch keyword
 634                         AddKeyword ("when", Token.WHEN);
 635
 636                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 637
 638                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 639                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 640                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 641                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 642                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 643                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 644                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 645                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 646                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 647                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 648                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 649                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 650
 651                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 652                         styles = NumberStyles.Float;
 653                 }
 654
 655                 int GetKeyword (char[] id, int id_len)
 656                 {
 657                         //
 658                         // Keywords are stored in an array of arrays grouped by their
 659                         // length and then by the first character
 660                         //
 661                         if (id_len >= keywords.Length || keywords [id_len] == null)
 662                                 return -1;
 663
 664                         int first_index = id [0] - '_';
 665                         if (first_index > 'z' - '_')
 666                                 return -1;
 667
 668                         var kwe = keywords [id_len] [first_index];
 669                         if (kwe == null)
 670                                 return -1;
 671
 672                         int res;
 673                         do {
 674                                 res = kwe.Token;
 675                                 for (int i = 1; i < id_len; ++i) {
 676                                         if (id [i] != kwe.Value [i]) {
 677                                                 res = 0;
 678                                                 kwe = kwe.Next;
 679                                                 break;
 680                                         }
 681                                 }
 682                         } while (res == 0 && kwe != null);
 683
 684                         if (res == 0)
 685                                 return -1;
 686
 687                         int next_token;
 688                         switch (res) {
 689                         case Token.GET:
 690                         case Token.SET:
 691                                 if (!handle_get_set)
 692                                         res = -1;
 693                                 break;
 694                         case Token.REMOVE:
 695                         case Token.ADD:
 696                                 if (!handle_remove_add)
 697                                         res = -1;
 698                                 break;
 699                         case Token.EXTERN:
 700                                 if (parsing_declaration == 0)
 701                                         res = Token.EXTERN_ALIAS;
 702                                 break;
 703                         case Token.DEFAULT:
 704                                 if (peek_token () == Token.COLON) {
 705                                         token ();
 706                                         res = Token.DEFAULT_COLON;
 707                                 }
 708                                 break;
 709                         case Token.WHEN:
 710                                 if (current_token != Token.CATCH && !parsing_catch_when)
 711                                         res = -1;
 712                                 break;
 713                         case Token.WHERE:
 714                                 if (!(handle_where && current_token != Token.COLON) && !query_parsing)
 715                                         res = -1;
 716                                 break;
 717                         case Token.FROM:
 718                                 //
 719                                 // A query expression is any expression that starts with `from identifier'
 720                                 // followed by any token except ; , =
 721                                 //
 722                                 if (!query_parsing) {
 723                                         if (lambda_arguments_parsing || parsing_block == 0) {
 724                                                 res = -1;
 725                                                 break;
 726                                         }
 727
 728                                         PushPosition ();
 729                                         // HACK: to disable generics micro-parser, because PushPosition does not
 730                                         // store identifiers array
 731                                         parsing_generic_less_than = 1;
 732                                         switch (xtoken ()) {
 733                                         case Token.IDENTIFIER:
 734                                         case Token.INT:
 735                                         case Token.BOOL:
 736                                         case Token.BYTE:
 737                                         case Token.CHAR:
 738                                         case Token.DECIMAL:
 739                                         case Token.DOUBLE:
 740                                         case Token.FLOAT:
 741                                         case Token.LONG:
 742                                         case Token.OBJECT:
 743                                         case Token.STRING:
 744                                         case Token.UINT:
 745                                         case Token.ULONG:
 746                                                 next_token = xtoken ();
 747                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS || next_token == Token.ASSIGN)
 748                                                         goto default;
 749
 750                                                 res = Token.FROM_FIRST;
 751                                                 query_parsing = true;
 752                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 753                                                         Report.FeatureIsNotAvailable (context, Location, "query expressions");
 754                                                 break;
 755                                         case Token.VOID:
 756                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 757                                                 break;
 758                                         default:
 759                                                 PopPosition ();
 760                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 761                                                 // which has been overwritten before we grabbed the identifier
 762                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 763                                                 return -1;
 764                                         }
 765                                         PopPosition ();
 766                                 }
 767                                 break;
 768                         case Token.JOIN:
 769                         case Token.ON:
 770                         case Token.EQUALS:
 771                         case Token.SELECT:
 772                         case Token.GROUP:
 773                         case Token.BY:
 774                         case Token.LET:
 775                         case Token.ORDERBY:
 776                         case Token.ASCENDING:
 777                         case Token.DESCENDING:
 778                         case Token.INTO:
 779                                 if (!query_parsing)
 780                                         res = -1;
 781                                 break;
 782
 783                         case Token.USING:
 784                         case Token.NAMESPACE:
 785                                 // TODO: some explanation needed
 786                                 check_incorrect_doc_comment ();
 787                                 parsing_modifiers = false;
 788                                 break;
 789
 790                         case Token.PARTIAL:
 791                                 if (parsing_block > 0) {
 792                                         res = -1;
 793                                         break;
 794                                 }
 795
 796                                 // Save current position and parse next token.
 797                                 PushPosition ();
 798
 799                                 next_token = token ();
 800                                 bool ok = (next_token == Token.CLASS) ||
 801                                         (next_token == Token.STRUCT) ||
 802                                         (next_token == Token.INTERFACE) ||
 803                                         (next_token == Token.VOID);
 804
 805                                 PopPosition ();
 806
 807                                 if (ok) {
 808                                         if (next_token == Token.VOID) {
 809                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 810                                                         Report.FeatureIsNotAvailable (context, Location, "partial methods");
 811                                         } else if (context.Settings.Version == LanguageVersion.ISO_1)
 812                                                 Report.FeatureIsNotAvailable (context, Location, "partial types");
 813
 814                                         return res;
 815                                 }
 816
 817                                 if (next_token < Token.LAST_KEYWORD) {
 818                                         Report.Error (267, Location,
 819                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 820                                         return token ();
 821                                 }
 822
 823                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 824                                 // which has been overwritten before we grabbed the identifier
 825                                 id_builder[0] = 'p';
 826                                 id_builder[1] = 'a';
 827                                 id_builder[2] = 'r';
 828                                 id_builder[3] = 't';
 829                                 id_builder[4] = 'i';
 830                                 id_builder[5] = 'a';
 831                                 id_builder[6] = 'l';
 832                                 res = -1;
 833                                 break;
 834
 835                         case Token.ASYNC:
 836                                 if (parsing_modifiers) {
 837                                         //
 838                                         // Skip attributes section or constructor called async
 839                                         //
 840                                         if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) {
 841                                                 res = -1;
 842                                         } else {
 843                                                 // async is keyword
 844                                         }
 845                                 } else if (parsing_block > 0) {
 846                                         switch (peek_token ()) {
 847                                         case Token.DELEGATE:
 848                                         case Token.OPEN_PARENS_LAMBDA:
 849                                                 // async is keyword
 850                                                 break;
 851                                         case Token.IDENTIFIER:
 852                                                 PushPosition ();
 853                                                 xtoken ();
 854                                                 if (xtoken () != Token.ARROW) {
 855                                                         PopPosition ();
 856                                                         goto default;
 857                                                 }
 858
 859                                                 PopPosition ();
 860                                                 break;
 861                                         default:
 862                                                 // peek_token could overwrite id_buffer
 863                                                 id_builder [0] = 'a'; id_builder [1] = 's'; id_builder [2] = 'y'; id_builder [3] = 'n'; id_builder [4] = 'c';
 864                                                 res = -1;
 865                                                 break;
 866                                         }
 867                                 } else {
 868                                         res = -1;
 869                                 }
 870
 871                                 if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) {
 872                                         Report.FeatureIsNotAvailable (context, Location, "asynchronous functions");
 873                                 }
 874
 875                                 break;
 876
 877                         case Token.AWAIT:
 878                                 if (parsing_block == 0)
 879                                         res = -1;
 880
 881                                 break;
 882                         }
 883
 884
 885                         return res;
 886                 }
 887
 888                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 889                 {
 890                         //
 891                         // Keywords are stored in an array of arrays grouped by their
 892                         // length and then by the first character
 893                         //
 894                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 895                                 return PreprocessorDirective.Invalid;
 896
 897                         int first_index = id[0] - '_';
 898                         if (first_index > 'z' - '_')
 899                                 return PreprocessorDirective.Invalid;
 900
 901                         var kwe = keywords_preprocessor[id_len][first_index];
 902                         if (kwe == null)
 903                                 return PreprocessorDirective.Invalid;
 904
 905                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 906                         do {
 907                                 res = kwe.Token;
 908                                 for (int i = 1; i < id_len; ++i) {
 909                                         if (id[i] != kwe.Value[i]) {
 910                                                 res = 0;
 911                                                 kwe = kwe.Next;
 912                                                 break;
 913                                         }
 914                                 }
 915                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 916
 917                         return res;
 918                 }
 919
 920                 public Location Location {
 921                         get {
 922                                 return new Location (current_source, ref_line, col);
 923                         }
 924                 }
 925
 926                 static bool is_identifier_start_character (int c)
 927                 {
 928                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 929                 }
 930
 931                 static bool is_identifier_part_character (char c)
 932                 {
 933                         if (c >= 'a' && c <= 'z')
 934                                 return true;
 935
 936                         if (c >= 'A' && c <= 'Z')
 937                                 return true;
 938
 939                         if (c == '_' || (c >= '0' && c <= '9'))
 940                                 return true;
 941
 942                         if (c < 0x80)
 943                                 return false;
 944
 945                         return is_identifier_part_character_slow_part (c);
 946                 }
 947
 948                 static bool is_identifier_part_character_slow_part (char c)
 949                 {
 950                         if (Char.IsLetter (c))
 951                                 return true;
 952
 953                         switch (Char.GetUnicodeCategory (c)) {
 954                                 case UnicodeCategory.ConnectorPunctuation:
 955
 956                                 // combining-character: A Unicode character of classes Mn or Mc
 957                                 case UnicodeCategory.NonSpacingMark:
 958                                 case UnicodeCategory.SpacingCombiningMark:
 959
 960                                 // decimal-digit-character: A Unicode character of the class Nd
 961                                 case UnicodeCategory.DecimalDigitNumber:
 962                                 return true;
 963                         }
 964
 965                         return false;
 966                 }
 967
 968                 public static bool IsKeyword (string s)
 969                 {
 970                         return keyword_strings.Contains (s);
 971                 }
 972
 973                 //
 974                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 975                 //
 976                 int TokenizeOpenParens ()
 977                 {
 978                         int ptoken;
 979                         current_token = -1;
 980
 981                         int bracket_level = 0;
 982                         bool is_type = false;
 983                         bool can_be_type = false;
 984
 985                         while (true) {
 986                                 ptoken = current_token;
 987                                 token ();
 988
 989                                 switch (current_token) {
 990                                 case Token.CLOSE_PARENS:
 991                                         token ();
 992
 993                                         //
 994                                         // Expression inside parens is lambda, (int i) =>
 995                                         //
 996                                         if (current_token == Token.ARROW)
 997                                                 return Token.OPEN_PARENS_LAMBDA;
 998
 999                                         //
1000                                         // Expression inside parens is single type, (int[])
1001                                         //
1002                                         if (is_type) {
1003                                                 if (current_token == Token.SEMICOLON)
1004                                                         return Token.OPEN_PARENS;
1005
1006                                                 return Token.OPEN_PARENS_CAST;
1007                                         }
1008
1009                                         //
1010                                         // Expression is possible cast, look at next token, (T)null
1011                                         //
1012                                         if (can_be_type) {
1013                                                 switch (current_token) {
1014                                                 case Token.OPEN_PARENS:
1015                                                 case Token.BANG:
1016                                                 case Token.TILDE:
1017                                                 case Token.IDENTIFIER:
1018                                                 case Token.LITERAL:
1019                                                 case Token.BASE:
1020                                                 case Token.CHECKED:
1021                                                 case Token.DELEGATE:
1022                                                 case Token.FALSE:
1023                                                 case Token.FIXED:
1024                                                 case Token.NEW:
1025                                                 case Token.NULL:
1026                                                 case Token.SIZEOF:
1027                                                 case Token.THIS:
1028                                                 case Token.THROW:
1029                                                 case Token.TRUE:
1030                                                 case Token.TYPEOF:
1031                                                 case Token.UNCHECKED:
1032                                                 case Token.UNSAFE:
1033                                                 case Token.DEFAULT:
1034                                                 case Token.AWAIT:
1035
1036                                                 //
1037                                                 // These can be part of a member access
1038                                                 //
1039                                                 case Token.INT:
1040                                                 case Token.UINT:
1041                                                 case Token.SHORT:
1042                                                 case Token.USHORT:
1043                                                 case Token.LONG:
1044                                                 case Token.ULONG:
1045                                                 case Token.DOUBLE:
1046                                                 case Token.FLOAT:
1047                                                 case Token.CHAR:
1048                                                 case Token.BYTE:
1049                                                 case Token.DECIMAL:
1050                                                 case Token.BOOL:
1051                                                 case Token.STRING:
1052                                                         return Token.OPEN_PARENS_CAST;
1053                                                 }
1054                                         }
1055                                         return Token.OPEN_PARENS;
1056
1057                                 case Token.DOT:
1058                                 case Token.DOUBLE_COLON:
1059                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
1060                                                 goto default;
1061
1062                                         continue;
1063
1064                                 case Token.IDENTIFIER:
1065                                 case Token.AWAIT:
1066                                         switch (ptoken) {
1067                                         case Token.DOT:
1068                                                 if (bracket_level == 0) {
1069                                                         is_type = false;
1070                                                         can_be_type = true;
1071                                                 }
1072
1073                                                 continue;
1074                                         case Token.OP_GENERICS_LT:
1075                                         case Token.COMMA:
1076                                         case Token.DOUBLE_COLON:
1077                                         case -1:
1078                                                 if (bracket_level == 0)
1079                                                         can_be_type = true;
1080                                                 continue;
1081                                         default:
1082                                                 can_be_type = is_type = false;
1083                                                 continue;
1084                                         }
1085
1086                                 case Token.OBJECT:
1087                                 case Token.STRING:
1088                                 case Token.BOOL:
1089                                 case Token.DECIMAL:
1090                                 case Token.FLOAT:
1091                                 case Token.DOUBLE:
1092                                 case Token.SBYTE:
1093                                 case Token.BYTE:
1094                                 case Token.SHORT:
1095                                 case Token.USHORT:
1096                                 case Token.INT:
1097                                 case Token.UINT:
1098                                 case Token.LONG:
1099                                 case Token.ULONG:
1100                                 case Token.CHAR:
1101                                 case Token.VOID:
1102                                         if (bracket_level == 0)
1103                                                 is_type = true;
1104                                         continue;
1105
1106                                 case Token.COMMA:
1107                                         if (bracket_level == 0) {
1108                                                 bracket_level = 100;
1109                                                 can_be_type = is_type = false;
1110                                         }
1111                                         continue;
1112
1113                                 case Token.OP_GENERICS_LT:
1114                                 case Token.OPEN_BRACKET:
1115                                         if (bracket_level++ == 0)
1116                                                 is_type = true;
1117                                         continue;
1118
1119                                 case Token.OP_GENERICS_GT:
1120                                 case Token.CLOSE_BRACKET:
1121                                         --bracket_level;
1122                                         continue;
1123
1124                                 case Token.INTERR_NULLABLE:
1125                                 case Token.STAR:
1126                                         if (bracket_level == 0)
1127                                                 is_type = true;
1128                                         continue;
1129
1130                                 case Token.REF:
1131                                 case Token.OUT:
1132                                         can_be_type = is_type = false;
1133                                         continue;
1134
1135                                 default:
1136                                         return Token.OPEN_PARENS;
1137                                 }
1138                         }
1139                 }
1140
1141                 public static bool IsValidIdentifier (string s)
1142                 {
1143                         if (s == null || s.Length == 0)
1144                                 return false;
1145
1146                         if (!is_identifier_start_character (s [0]))
1147                                 return false;
1148
1149                         for (int i = 1; i < s.Length; i ++)
1150                                 if (! is_identifier_part_character (s [i]))
1151                                         return false;
1152
1153                         return true;
1154                 }
1155
1156                 bool parse_less_than (ref int genericDimension)
1157                 {
1158                 start:
1159                         int the_token = token ();
1160                         if (the_token == Token.OPEN_BRACKET) {
1161                                 while (true) {
1162                                         the_token = token ();
1163                                         if (the_token == Token.EOF)
1164                                                 return true;
1165
1166                                         if (the_token == Token.CLOSE_BRACKET)
1167                                                 break;
1168                                 }
1169                                 the_token = token ();
1170                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1171                                 the_token = token ();
1172                         }
1173                         switch (the_token) {
1174                         case Token.IDENTIFIER:
1175                         case Token.OBJECT:
1176                         case Token.STRING:
1177                         case Token.BOOL:
1178                         case Token.DECIMAL:
1179                         case Token.FLOAT:
1180                         case Token.DOUBLE:
1181                         case Token.SBYTE:
1182                         case Token.BYTE:
1183                         case Token.SHORT:
1184                         case Token.USHORT:
1185                         case Token.INT:
1186                         case Token.UINT:
1187                         case Token.LONG:
1188                         case Token.ULONG:
1189                         case Token.CHAR:
1190                         case Token.VOID:
1191                                 break;
1192                         case Token.OP_GENERICS_GT:
1193                                 genericDimension = 1;
1194                                 return true;
1195                         case Token.IN:
1196                         case Token.OUT:
1197                                 return true;
1198                         case Token.COMMA:
1199                                 do {
1200                                         ++genericDimension;
1201                                         the_token = token ();
1202                                 } while (the_token == Token.COMMA);
1203
1204                                 if (the_token == Token.OP_GENERICS_GT) {
1205                                         ++genericDimension;
1206                                         return true;
1207                                 }
1208
1209                                 return false;
1210                         default:
1211                                 return false;
1212                         }
1213                 again:
1214                         the_token = token ();
1215
1216                         if (the_token == Token.OP_GENERICS_GT)
1217                                 return true;
1218                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1219                                 goto start;
1220                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1221                                 goto again;
1222                         else if (the_token == Token.OP_GENERICS_LT) {
1223                                 if (!parse_less_than (ref genericDimension))
1224                                         return false;
1225                                 goto again;
1226                         } else if (the_token == Token.OPEN_BRACKET) {
1227                         rank_specifiers:
1228                                 the_token = token ();
1229                                 if (the_token == Token.CLOSE_BRACKET)
1230                                         goto again;
1231                                 else if (the_token == Token.COMMA)
1232                                         goto rank_specifiers;
1233                                 return false;
1234                         }
1235
1236                         return false;
1237                 }
1238
1239                 public int peek_token ()
1240                 {
1241                         int the_token;
1242
1243                         PushPosition ();
1244                         the_token = token ();
1245                         PopPosition ();
1246
1247                         return the_token;
1248                 }
1249
1250                 //
1251                 // Tonizes `?' using custom disambiguous rules to return one
1252                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1253                 //
1254                 // Tricky expression looks like:
1255                 //
1256                 // Foo ? a = x ? b : c;
1257                 //
1258                 int TokenizePossibleNullableType ()
1259                 {
1260                         if (parsing_block == 0 || parsing_type > 0)
1261                                 return Token.INTERR_NULLABLE;
1262
1263                         int d = peek_char ();
1264                         if (d == '?') {
1265                                 get_char ();
1266                                 return Token.OP_COALESCING;
1267                         }
1268
1269                         if (d == '.') {
1270                                 return Token.INTERR_OPERATOR;
1271                         }
1272
1273                         if (d != ' ') {
1274                                 if (d == ',' || d == ';' || d == '>')
1275                                         return Token.INTERR_NULLABLE;
1276                                 if (d == '*' || (d >= '0' && d <= '9'))
1277                                         return Token.INTERR;
1278                         }
1279
1280                         PushPosition ();
1281                         current_token = Token.NONE;
1282                         int next_token;
1283                         int parens = 0;
1284                         int generics = 0;
1285                         int brackets = 0;
1286
1287                         var nt = xtoken ();
1288                         switch (nt) {
1289                         case Token.DOT:
1290                         case Token.OPEN_BRACKET_EXPR:
1291                                 next_token = Token.INTERR_OPERATOR;
1292                                 break;
1293                         case Token.LITERAL:
1294                         case Token.TRUE:
1295                         case Token.FALSE:
1296                         case Token.NULL:
1297                         case Token.THIS:
1298                         case Token.NEW:
1299                                 next_token = Token.INTERR;
1300                                 break;
1301
1302                         case Token.SEMICOLON:
1303                         case Token.COMMA:
1304                         case Token.CLOSE_PARENS:
1305                         case Token.OPEN_BRACKET:
1306                         case Token.OP_GENERICS_GT:
1307                         case Token.INTERR:
1308                         case Token.OP_COALESCING:
1309                         case Token.COLON:
1310                                 next_token = Token.INTERR_NULLABLE;
1311                                 break;
1312
1313                         case Token.OPEN_PARENS:
1314                         case Token.OPEN_PARENS_CAST:
1315                         case Token.OPEN_PARENS_LAMBDA:
1316                                 next_token = -1;
1317                                 ++parens;
1318                                 break;
1319
1320                         case Token.OP_GENERICS_LT:
1321                         case Token.OP_GENERICS_LT_DECL:
1322                         case Token.GENERIC_DIMENSION:
1323                                 next_token = -1;
1324                                 ++generics;
1325                                 break;
1326
1327                         default:
1328                                 next_token = -1;
1329                                 break;
1330                         }
1331
1332                         if (next_token == -1) {
1333                                 switch (xtoken ()) {
1334                                 case Token.COMMA:
1335                                 case Token.SEMICOLON:
1336                                 case Token.OPEN_BRACE:
1337                                 case Token.IN:
1338                                         next_token = Token.INTERR_NULLABLE;
1339                                         break;
1340
1341                                 case Token.COLON:
1342                                         next_token = Token.INTERR;
1343                                         break;
1344
1345                                 case Token.OPEN_PARENS:
1346                                 case Token.OPEN_PARENS_CAST:
1347                                 case Token.OPEN_PARENS_LAMBDA:
1348                                         ++parens;
1349                                         goto default;
1350
1351                                 case Token.OPEN_BRACKET:
1352                                 case Token.OPEN_BRACKET_EXPR:
1353                                         ++brackets;
1354                                         goto default;
1355
1356                                 case Token.CLOSE_PARENS:
1357                                         --parens;
1358                                         goto default;
1359
1360                                 case Token.OP_GENERICS_LT:
1361                                 case Token.OP_GENERICS_LT_DECL:
1362                                 case Token.GENERIC_DIMENSION:
1363                                         ++generics;
1364                                         goto default;
1365
1366                                 default:
1367                                         int ntoken;
1368                                         int interrs = 1;
1369                                         int colons = 0;
1370                                         int braces = 0;
1371                                         //
1372                                         // All shorcuts failed, do it hard way
1373                                         //
1374                                         while ((ntoken = xtoken ()) != Token.EOF) {
1375                                                 switch (ntoken) {
1376                                                 case Token.OPEN_BRACE:
1377                                                         ++braces;
1378                                                         continue;
1379                                                 case Token.OPEN_PARENS:
1380                                                 case Token.OPEN_PARENS_CAST:
1381                                                 case Token.OPEN_PARENS_LAMBDA:
1382                                                         ++parens;
1383                                                         continue;
1384                                                 case Token.CLOSE_BRACE:
1385                                                         --braces;
1386                                                         continue;
1387                                                 case Token.OP_GENERICS_LT:
1388                                                 case Token.OP_GENERICS_LT_DECL:
1389                                                 case Token.GENERIC_DIMENSION:
1390                                                         ++generics;
1391                                                         continue;
1392                                                 case Token.OPEN_BRACKET:
1393                                                 case Token.OPEN_BRACKET_EXPR:
1394                                                         ++brackets;
1395                                                         continue;
1396                                                 case Token.CLOSE_BRACKET:
1397                                                         --brackets;
1398                                                         continue;
1399                                                 case Token.CLOSE_PARENS:
1400                                                         if (parens > 0) {
1401                                                                 --parens;
1402                                                                 continue;
1403                                                         }
1404
1405                                                         PopPosition ();
1406                                                         return Token.INTERR_NULLABLE;
1407
1408                                                 case Token.OP_GENERICS_GT:
1409                                                         if (generics > 0) {
1410                                                                 --generics;
1411                                                                 continue;
1412                                                         }
1413
1414                                                         PopPosition ();
1415                                                         return Token.INTERR_NULLABLE;
1416                                                 }
1417
1418                                                 if (braces != 0)
1419                                                         continue;
1420
1421                                                 if (ntoken == Token.SEMICOLON)
1422                                                         break;
1423
1424                                                 if (parens != 0)
1425                                                         continue;
1426
1427                                                 if (ntoken == Token.COMMA) {
1428                                                         if (generics != 0 || brackets != 0)
1429                                                                 continue;
1430
1431                                                         PopPosition ();
1432                                                         return Token.INTERR_NULLABLE;
1433                                                 }
1434
1435                                                 if (ntoken == Token.COLON) {
1436                                                         if (++colons == interrs)
1437                                                                 break;
1438                                                         continue;
1439                                                 }
1440
1441                                                 if (ntoken == Token.INTERR) {
1442                                                         ++interrs;
1443                                                         continue;
1444                                                 }
1445                                         }
1446
1447                                         next_token = colons != interrs && braces == 0 ? Token.INTERR_NULLABLE : Token.INTERR;
1448                                         break;
1449                                 }
1450                         }
1451
1452                         PopPosition ();
1453                         return next_token;
1454                 }
1455
1456                 bool decimal_digits (int c)
1457                 {
1458                         int d;
1459                         bool seen_digits = false;
1460
1461                         if (c != -1){
1462                                 if (number_pos == MaxNumberLength)
1463                                         Error_NumericConstantTooLong ();
1464                                 number_builder [number_pos++] = (char) c;
1465                         }
1466
1467                         //
1468                         // We use peek_char2, because decimal_digits needs to do a
1469                         // 2-character look-ahead (5.ToString for example).
1470                         //
1471                         while ((d = peek_char2 ()) != -1){
1472                                 if (d >= '0' && d <= '9'){
1473                                         if (number_pos == MaxNumberLength)
1474                                                 Error_NumericConstantTooLong ();
1475                                         number_builder [number_pos++] = (char) d;
1476                                         get_char ();
1477                                         seen_digits = true;
1478                                 } else
1479                                         break;
1480                         }
1481
1482                         return seen_digits;
1483                 }
1484
1485                 static bool is_hex (int e)
1486                 {
1487                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1488                 }
1489
1490                 static TypeCode real_type_suffix (int c)
1491                 {
1492                         switch (c){
1493                         case 'F': case 'f':
1494                                 return TypeCode.Single;
1495                         case 'D': case 'd':
1496                                 return TypeCode.Double;
1497                         case 'M': case 'm':
1498                                 return TypeCode.Decimal;
1499                         default:
1500                                 return TypeCode.Empty;
1501                         }
1502                 }
1503
1504                 ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc)
1505                 {
1506                         bool is_unsigned = false;
1507                         bool is_long = false;
1508
1509                         if (c != -1){
1510                                 bool scanning = true;
1511                                 do {
1512                                         switch (c){
1513                                         case 'U': case 'u':
1514                                                 if (is_unsigned)
1515                                                         scanning = false;
1516                                                 is_unsigned = true;
1517                                                 get_char ();
1518                                                 break;
1519
1520                                         case 'l':
1521                                                 if (!is_unsigned){
1522                                                         //
1523                                                         // if we have not seen anything in between
1524                                                         // report this error
1525                                                         //
1526                                                         Report.Warning (78, 4, Location, "The `l' suffix is easily confused with the digit `1' (use `L' for clarity)");
1527                                                 }
1528
1529                                                 goto case 'L';
1530
1531                                         case 'L':
1532                                                 if (is_long)
1533                                                         scanning = false;
1534                                                 is_long = true;
1535                                                 get_char ();
1536                                                 break;
1537
1538                                         default:
1539                                                 scanning = false;
1540                                                 break;
1541                                         }
1542                                         c = peek_char ();
1543                                 } while (scanning);
1544                         }
1545
1546                         if (is_long && is_unsigned){
1547                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1548                         }
1549
1550                         if (is_unsigned){
1551                                 // uint if possible, or ulong else.
1552
1553                                 if ((ul & 0xffffffff00000000) == 0)
1554                                         return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc);
1555                                 else
1556                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1557                         } else if (is_long){
1558                                 // long if possible, ulong otherwise
1559                                 if ((ul & 0x8000000000000000) != 0)
1560                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1561                                 else
1562                                         return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1563                         } else {
1564                                 // int, uint, long or ulong in that order
1565                                 if ((ul & 0xffffffff00000000) == 0){
1566                                         uint ui = (uint) ul;
1567
1568                                         if ((ui & 0x80000000) != 0)
1569                                                 return new UIntLiteral (context.BuiltinTypes, ui, loc);
1570                                         else
1571                                                 return new IntLiteral (context.BuiltinTypes, (int) ui, loc);
1572                                 } else {
1573                                         if ((ul & 0x8000000000000000) != 0)
1574                                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1575                                         else
1576                                                 return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1577                                 }
1578                         }
1579                 }
1580
1581                 //
1582                 // given `c' as the next char in the input decide whether
1583                 // we need to convert to a special type, and then choose
1584                 // the best representation for the integer
1585                 //
1586                 ILiteralConstant adjust_int (int c, Location loc)
1587                 {
1588                         try {
1589                                 if (number_pos > 9){
1590                                         ulong ul = (uint) (number_builder [0] - '0');
1591
1592                                         for (int i = 1; i < number_pos; i++){
1593                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1594                                         }
1595
1596                                         return integer_type_suffix (ul, c, loc);
1597                                 } else {
1598                                         uint ui = (uint) (number_builder [0] - '0');
1599
1600                                         for (int i = 1; i < number_pos; i++){
1601                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1602                                         }
1603
1604                                         return integer_type_suffix (ui, c, loc);
1605                                 }
1606                         } catch (OverflowException) {
1607                                 Error_NumericConstantTooLong ();
1608                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1609                         }
1610                         catch (FormatException) {
1611                                 Report.Error (1013, Location, "Invalid number");
1612                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1613                         }
1614                 }
1615
1616                 ILiteralConstant adjust_real (TypeCode t, Location loc)
1617                 {
1618                         string s = new string (number_builder, 0, number_pos);
1619                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1620
1621                         switch (t){
1622                         case TypeCode.Decimal:
1623                                 try {
1624                                         return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc);
1625                                 } catch (OverflowException) {
1626                                         Report.Error (594, Location, error_details, "decimal");
1627                                         return new DecimalLiteral (context.BuiltinTypes, 0, loc);
1628                                 }
1629                         case TypeCode.Single:
1630                                 try {
1631                                         return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc);
1632                                 } catch (OverflowException) {
1633                                         Report.Error (594, Location, error_details, "float");
1634                                         return new FloatLiteral (context.BuiltinTypes, 0, loc);
1635                                 }
1636                         default:
1637                                 try {
1638                                         return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc);
1639                                 } catch (OverflowException) {
1640                                         Report.Error (594, loc, error_details, "double");
1641                                         return new DoubleLiteral (context.BuiltinTypes, 0, loc);
1642                                 }
1643                         }
1644                 }
1645
1646                 ILiteralConstant handle_hex (Location loc)
1647                 {
1648                         int d;
1649                         ulong ul;
1650
1651                         get_char ();
1652                         while ((d = peek_char ()) != -1){
1653                                 if (is_hex (d)){
1654                                         number_builder [number_pos++] = (char) d;
1655                                         get_char ();
1656                                 } else
1657                                         break;
1658                         }
1659
1660                         string s = new String (number_builder, 0, number_pos);
1661
1662                         try {
1663                                 if (number_pos <= 8)
1664                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1665                                 else
1666                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1667
1668                                 return integer_type_suffix (ul, peek_char (), loc);
1669                         } catch (OverflowException){
1670                                 Error_NumericConstantTooLong ();
1671                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1672                         }
1673                         catch (FormatException) {
1674                                 Report.Error (1013, Location, "Invalid number");
1675                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1676                         }
1677                 }
1678
1679                 //
1680                 // Invoked if we know we have .digits or digits
1681                 //
1682                 int is_number (int c, bool dotLead)
1683                 {
1684                         ILiteralConstant res;
1685
1686 #if FULL_AST
1687                         int read_start = reader.Position - 1;
1688                         if (dotLead) {
1689                                 //
1690                                 // Caller did peek_char
1691                                 //
1692                                 --read_start;
1693                         }
1694 #endif
1695                         number_pos = 0;
1696                         var loc = Location;
1697
1698                         if (!dotLead){
1699                                 if (c == '0'){
1700                                         int peek = peek_char ();
1701
1702                                         if (peek == 'x' || peek == 'X') {
1703                                                 val = res = handle_hex (loc);
1704 #if FULL_AST
1705                                                 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1706 #endif
1707
1708                                                 return Token.LITERAL;
1709                                         }
1710                                 }
1711                                 decimal_digits (c);
1712                                 c = peek_char ();
1713                         }
1714
1715                         //
1716                         // We need to handle the case of
1717                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1718                         //
1719                         bool is_real = false;
1720                         if (c == '.'){
1721                                 if (!dotLead)
1722                                         get_char ();
1723
1724                                 if (decimal_digits ('.')){
1725                                         is_real = true;
1726                                         c = peek_char ();
1727                                 } else {
1728                                         putback ('.');
1729                                         number_pos--;
1730                                         val = res = adjust_int (-1, loc);
1731
1732 #if FULL_AST
1733                                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1734 #endif
1735                                         return Token.LITERAL;
1736                                 }
1737                         }
1738
1739                         if (c == 'e' || c == 'E'){
1740                                 is_real = true;
1741                                 get_char ();
1742                                 if (number_pos == MaxNumberLength)
1743                                         Error_NumericConstantTooLong ();
1744                                 number_builder [number_pos++] = (char) c;
1745                                 c = get_char ();
1746
1747                                 if (c == '+'){
1748                                         if (number_pos == MaxNumberLength)
1749                                                 Error_NumericConstantTooLong ();
1750                                         number_builder [number_pos++] = '+';
1751                                         c = -1;
1752                                 } else if (c == '-') {
1753                                         if (number_pos == MaxNumberLength)
1754                                                 Error_NumericConstantTooLong ();
1755                                         number_builder [number_pos++] = '-';
1756                                         c = -1;
1757                                 } else {
1758                                         if (number_pos == MaxNumberLength)
1759                                                 Error_NumericConstantTooLong ();
1760                                         number_builder [number_pos++] = '+';
1761                                 }
1762
1763                                 decimal_digits (c);
1764                                 c = peek_char ();
1765                         }
1766
1767                         var type = real_type_suffix (c);
1768                         if (type == TypeCode.Empty && !is_real) {
1769                                 res = adjust_int (c, loc);
1770                         } else {
1771                                 is_real = true;
1772
1773                                 if (type != TypeCode.Empty) {
1774                                         get_char ();
1775                                 }
1776
1777                                 res = adjust_real (type, loc);
1778                         }
1779
1780                         val = res;
1781
1782 #if FULL_AST
1783                         var chars = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty && c > 0 ? 1 : 0));
1784                         if (chars[chars.Length - 1] == '\r')
1785                                 Array.Resize (ref chars, chars.Length - 1);
1786                         res.ParsedValue = chars;
1787 #endif
1788
1789                         return Token.LITERAL;
1790                 }
1791
1792                 //
1793                 // Accepts exactly count (4 or 8) hex, no more no less
1794                 //
1795                 int getHex (int count, out int surrogate, out bool error)
1796                 {
1797                         int i;
1798                         int total = 0;
1799                         int c;
1800                         int top = count != -1 ? count : 4;
1801
1802                         get_char ();
1803                         error = false;
1804                         surrogate = 0;
1805                         for (i = 0; i < top; i++){
1806                                 c = get_char ();
1807
1808                                 if (c >= '0' && c <= '9')
1809                                         c = (int) c - (int) '0';
1810                                 else if (c >= 'A' && c <= 'F')
1811                                         c = (int) c - (int) 'A' + 10;
1812                                 else if (c >= 'a' && c <= 'f')
1813                                         c = (int) c - (int) 'a' + 10;
1814                                 else {
1815                                         error = true;
1816                                         return 0;
1817                                 }
1818
1819                                 total = (total * 16) + c;
1820                                 if (count == -1){
1821                                         int p = peek_char ();
1822                                         if (p == -1)
1823                                                 break;
1824                                         if (!is_hex ((char)p))
1825                                                 break;
1826                                 }
1827                         }
1828
1829                         if (top == 8) {
1830                                 if (total > 0x0010FFFF) {
1831                                         error = true;
1832                                         return 0;
1833                                 }
1834
1835                                 if (total >= 0x00010000) {
1836                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1837                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1838                                 }
1839                         }
1840
1841                         return total;
1842                 }
1843
1844                 int escape (int c, out int surrogate)
1845                 {
1846                         bool error;
1847                         int d;
1848                         int v;
1849
1850                         d = peek_char ();
1851                         if (c != '\\') {
1852                                 surrogate = 0;
1853                                 return c;
1854                         }
1855
1856                         switch (d){
1857                         case 'a':
1858                                 v = '\a'; break;
1859                         case 'b':
1860                                 v = '\b'; break;
1861                         case 'n':
1862                                 v = '\n'; break;
1863                         case 't':
1864                                 v = '\t'; break;
1865                         case 'v':
1866                                 v = '\v'; break;
1867                         case 'r':
1868                                 v = '\r'; break;
1869                         case '\\':
1870                                 v = '\\'; break;
1871                         case 'f':
1872                                 v = '\f'; break;
1873                         case '0':
1874                                 v = 0; break;
1875                         case '"':
1876                                 v = '"'; break;
1877                         case '\'':
1878                                 v = '\''; break;
1879                         case 'x':
1880                                 v = getHex (-1, out surrogate, out error);
1881                                 if (error)
1882                                         goto default;
1883                                 return v;
1884                         case 'u':
1885                         case 'U':
1886                                 return EscapeUnicode (d, out surrogate);
1887                         default:
1888                                 surrogate = 0;
1889                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1890                                 return d;
1891                         }
1892
1893                         get_char ();
1894                         surrogate = 0;
1895                         return v;
1896                 }
1897
1898                 int EscapeUnicode (int ch, out int surrogate)
1899                 {
1900                         bool error;
1901                         if (ch == 'U') {
1902                                 ch = getHex (8, out surrogate, out error);
1903                         } else {
1904                                 ch = getHex (4, out surrogate, out error);
1905                         }
1906
1907                         if (error)
1908                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1909
1910                         return ch;
1911                 }
1912
1913                 int get_char ()
1914                 {
1915                         int x;
1916                         if (putback_char != -1) {
1917                                 x = putback_char;
1918                                 putback_char = -1;
1919                         } else {
1920                                 x = reader.Read ();
1921                         }
1922
1923                         if (x <= 13) {
1924                                 if (x == '\r') {
1925                                         if (peek_char () == '\n') {
1926                                                 putback_char = -1;
1927                                         }
1928
1929                                         x = '\n';
1930                                         advance_line ();
1931                                 } else if (x == '\n') {
1932                                         advance_line ();
1933                                 } else {
1934                                         col++;
1935                                 }
1936                         } else if (x >= UnicodeLS && x <= UnicodePS) {
1937                                 advance_line ();
1938                         } else {
1939                                 col++;
1940                         }
1941
1942                         return x;
1943                 }
1944
1945                 void advance_line ()
1946                 {
1947                         line++;
1948                         ref_line++;
1949                         previous_col = col;
1950                         col = 0;
1951                 }
1952
1953                 int peek_char ()
1954                 {
1955                         if (putback_char == -1)
1956                                 putback_char = reader.Read ();
1957                         return putback_char;
1958                 }
1959
1960                 int peek_char2 ()
1961                 {
1962                         if (putback_char != -1)
1963                                 return putback_char;
1964                         return reader.Peek ();
1965                 }
1966
1967                 public void putback (int c)
1968                 {
1969                         if (putback_char != -1) {
1970                                 throw new InternalErrorException (string.Format ("Secondary putback [{0}] putting back [{1}] is not allowed", (char)putback_char, (char) c), Location);
1971                         }
1972
1973                         if (c == '\n' || col == 0 || (c >= UnicodeLS && c <= UnicodePS)) {
1974                                 // It won't happen though.
1975                                 line--;
1976                                 ref_line--;
1977                                 col = previous_col;
1978                         }
1979                         else
1980                                 col--;
1981                         putback_char = c;
1982                 }
1983
1984                 public bool advance ()
1985                 {
1986                         return peek_char () != -1 || CompleteOnEOF;
1987                 }
1988
1989                 public Object Value {
1990                         get {
1991                                 return val;
1992                         }
1993                 }
1994
1995                 public Object value ()
1996                 {
1997                         return val;
1998                 }
1999
2000                 public int token ()
2001                 {
2002                         current_token = xtoken ();
2003                         return current_token;
2004                 }
2005
2006                 int TokenizePreprocessorKeyword (out int c)
2007                 {
2008                         // skip over white space
2009                         do {
2010                                 c = get_char ();
2011                         } while (c == ' ' || c == '\t');
2012
2013
2014                         int pos = 0;
2015                         while (c != -1 && c >= 'a' && c <= 'z') {
2016                                 id_builder[pos++] = (char) c;
2017                                 c = get_char ();
2018                                 if (c == '\\') {
2019                                         int peek = peek_char ();
2020                                         if (peek == 'U' || peek == 'u') {
2021                                                 int surrogate;
2022                                                 c = EscapeUnicode (c, out surrogate);
2023                                                 if (surrogate != 0) {
2024                                                         if (is_identifier_part_character ((char) c)) {
2025                                                                 id_builder[pos++] = (char) c;
2026                                                         }
2027                                                         c = surrogate;
2028                                                 }
2029                                         }
2030                                 }
2031                         }
2032
2033                         return pos;
2034                 }
2035
2036                 PreprocessorDirective get_cmd_arg (out string arg)
2037                 {
2038                         int c;
2039
2040                         tokens_seen = false;
2041                         arg = "";
2042
2043                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorKeyword (out c));
2044
2045                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
2046                                 return cmd;
2047
2048                         // skip over white space
2049                         while (c == ' ' || c == '\t')
2050                                 c = get_char ();
2051
2052                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
2053                         int pos = 0;
2054
2055                         while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
2056                                 if (c == '\\' && has_identifier_argument >= 0) {
2057                                         if (has_identifier_argument != 0) {
2058                                                 has_identifier_argument = 1;
2059
2060                                                 int peek = peek_char ();
2061                                                 if (peek == 'U' || peek == 'u') {
2062                                                         int surrogate;
2063                                                         c = EscapeUnicode (c, out surrogate);
2064                                                         if (surrogate != 0) {
2065                                                                 if (is_identifier_part_character ((char) c)) {
2066                                                                         if (pos == value_builder.Length)
2067                                                                                 Array.Resize (ref value_builder, pos * 2);
2068
2069                                                                         value_builder[pos++] = (char) c;
2070                                                                 }
2071                                                                 c = surrogate;
2072                                                         }
2073                                                 }
2074                                         } else {
2075                                                 has_identifier_argument = -1;
2076                                         }
2077                                 } else if (c == '/' && peek_char () == '/') {
2078                                         //
2079                                         // Eat single-line comments
2080                                         //
2081                                         get_char ();
2082                                         ReadToEndOfLine ();
2083                                         break;
2084                                 }
2085
2086                                 if (pos == value_builder.Length)
2087                                         Array.Resize (ref value_builder, pos * 2);
2088
2089                                 value_builder[pos++] = (char) c;
2090                                 c = get_char ();
2091                         }
2092
2093                         if (pos != 0) {
2094                                 if (pos > MaxIdentifierLength)
2095                                         arg = new string (value_builder, 0, pos);
2096                                 else
2097                                         arg = InternIdentifier (value_builder, pos);
2098
2099                                 // Eat any trailing whitespaces
2100                                 arg = arg.Trim (simple_whitespaces);
2101                         }
2102
2103                         return cmd;
2104                 }
2105
2106                 //
2107                 // Handles the #line directive
2108                 //
2109                 bool PreProcessLine ()
2110                 {
2111                         Location loc = Location;
2112
2113                         int c;
2114
2115                         int length = TokenizePreprocessorKeyword (out c);
2116                         if (length == line_default.Length) {
2117                                 if (!IsTokenIdentifierEqual (line_default))
2118                                         return false;
2119
2120                                 current_source = source_file.SourceFile;
2121                                 if (!hidden_block_start.IsNull) {
2122                                         current_source.RegisterHiddenScope (hidden_block_start, loc);
2123                                         hidden_block_start = Location.Null;
2124                                 }
2125
2126                                 ref_line = line;
2127                                 return true;
2128                         }
2129
2130                         if (length == line_hidden.Length) {
2131                                 if (!IsTokenIdentifierEqual (line_hidden))
2132                                         return false;
2133
2134                                 if (hidden_block_start.IsNull)
2135                                         hidden_block_start = loc;
2136
2137                                 return true;
2138                         }
2139
2140                         if (length != 0 || c < '0' || c > '9') {
2141                                 //
2142                                 // Eat any remaining characters to continue parsing on next line
2143                                 //
2144                                 ReadToEndOfLine ();
2145                                 return false;
2146                         }
2147
2148                         int new_line = TokenizeNumber (c);
2149                         if (new_line < 1) {
2150                                 //
2151                                 // Eat any remaining characters to continue parsing on next line
2152                                 //
2153                                 ReadToEndOfLine ();
2154                                 return new_line != 0;
2155                         }
2156
2157                         c = get_char ();
2158                         if (c == ' ') {
2159                                 // skip over white space
2160                                 do {
2161                                         c = get_char ();
2162                                 } while (c == ' ' || c == '\t');
2163                         } else if (c == '"') {
2164                                 c = 0;
2165                         }
2166
2167                         if (c != '\n' && c != '/' && c != '"' && c != UnicodeLS && c != UnicodePS) {
2168                                 //
2169                                 // Eat any remaining characters to continue parsing on next line
2170                                 //
2171                                 ReadToEndOfLine ();
2172
2173                                 Report.Error (1578, loc, "Filename, single-line comment or end-of-line expected");
2174                                 return true;
2175                         }
2176
2177                         string new_file_name = null;
2178                         if (c == '"') {
2179                                 new_file_name = TokenizeFileName (ref c);
2180
2181                                 // skip over white space
2182                                 while (c == ' ' || c == '\t') {
2183                                         c = get_char ();
2184                                 }
2185                         }
2186
2187                         if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
2188
2189                         } else if (c == '/') {
2190                                 ReadSingleLineComment ();
2191                         } else {
2192                                 //
2193                                 // Eat any remaining characters to continue parsing on next line
2194                                 //
2195                                 ReadToEndOfLine ();
2196
2197                                 Error_EndLineExpected ();
2198                                 return true;
2199                         }
2200
2201                         if (new_file_name != null) {
2202                                 current_source = context.LookupFile (source_file, new_file_name);
2203                                 source_file.AddIncludeFile (current_source);
2204                         }
2205
2206                         if (!hidden_block_start.IsNull) {
2207                                 current_source.RegisterHiddenScope (hidden_block_start, loc);
2208                                 hidden_block_start = Location.Null;
2209                         }
2210
2211                         ref_line = new_line;
2212                         return true;
2213                 }
2214
2215                 //
2216                 // Handles #define and #undef
2217                 //
2218                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
2219                 {
2220                         if (ident.Length == 0 || ident == "true" || ident == "false"){
2221                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
2222                                 return;
2223                         }
2224
2225                         if (ident.IndexOfAny (simple_whitespaces) != -1){
2226                                 Error_EndLineExpected ();
2227                                 return;
2228                         }
2229
2230                         if (!is_identifier_start_character (ident [0]))
2231                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
2232
2233                         foreach (char c in ident.Substring (1)){
2234                                 if (!is_identifier_part_character (c)){
2235                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
2236                                         return;
2237                                 }
2238                         }
2239
2240                         if (!caller_is_taking)
2241                                 return;
2242
2243                         if (is_define) {
2244                                 //
2245                                 // #define ident
2246                                 //
2247                                 if (context.Settings.IsConditionalSymbolDefined (ident))
2248                                         return;
2249
2250                                 source_file.AddDefine (ident);
2251                         } else {
2252                                 //
2253                                 // #undef ident
2254                                 //
2255                                 source_file.AddUndefine (ident);
2256                         }
2257                 }
2258
2259                 byte read_hex (out bool error)
2260                 {
2261                         int total;
2262                         int c = get_char ();
2263
2264                         if ((c >= '0') && (c <= '9'))
2265                                 total = (int) c - (int) '0';
2266                         else if ((c >= 'A') && (c <= 'F'))
2267                                 total = (int) c - (int) 'A' + 10;
2268                         else if ((c >= 'a') && (c <= 'f'))
2269                                 total = (int) c - (int) 'a' + 10;
2270                         else {
2271                                 error = true;
2272                                 return 0;
2273                         }
2274
2275                         total *= 16;
2276                         c = get_char ();
2277
2278                         if ((c >= '0') && (c <= '9'))
2279                                 total += (int) c - (int) '0';
2280                         else if ((c >= 'A') && (c <= 'F'))
2281                                 total += (int) c - (int) 'A' + 10;
2282                         else if ((c >= 'a') && (c <= 'f'))
2283                                 total += (int) c - (int) 'a' + 10;
2284                         else {
2285                                 error = true;
2286                                 return 0;
2287                         }
2288
2289                         error = false;
2290                         return (byte) total;
2291                 }
2292
2293                 //
2294                 // Parses #pragma checksum
2295                 //
2296                 bool ParsePragmaChecksum ()
2297                 {
2298                         //
2299                         // The syntax is ` "foo.txt" "{guid}" "hash"'
2300                         //
2301                         // guid is predefined hash algorithm guid {406ea660-64cf-4c82-b6f0-42d48172a799} for md5
2302                         //
2303                         int c = get_char ();
2304
2305                         if (c != '"')
2306                                 return false;
2307
2308                         string file_name = TokenizeFileName (ref c);
2309
2310                         // TODO: Any white-spaces count
2311                         if (c != ' ')
2312                                 return false;
2313
2314                         SourceFile file = context.LookupFile (source_file, file_name);
2315
2316                         if (get_char () != '"' || get_char () != '{')
2317                                 return false;
2318
2319                         bool error;
2320                         byte[] guid_bytes = new byte [16];
2321                         int i = 0;
2322
2323                         for (; i < 4; i++) {
2324                                 guid_bytes [i] = read_hex (out error);
2325                                 if (error)
2326                                         return false;
2327                         }
2328
2329                         if (get_char () != '-')
2330                                 return false;
2331
2332                         for (; i < 10; i++) {
2333                                 guid_bytes [i] = read_hex (out error);
2334                                 if (error)
2335                                         return false;
2336
2337                                 guid_bytes [i++] = read_hex (out error);
2338                                 if (error)
2339                                         return false;
2340
2341                                 if (get_char () != '-')
2342                                         return false;
2343                         }
2344
2345                         for (; i < 16; i++) {
2346                                 guid_bytes [i] = read_hex (out error);
2347                                 if (error)
2348                                         return false;
2349                         }
2350
2351                         if (get_char () != '}' || get_char () != '"')
2352                                 return false;
2353
2354                         // TODO: Any white-spaces count
2355                         c = get_char ();
2356                         if (c != ' ')
2357                                 return false;
2358
2359                         if (get_char () != '"')
2360                                 return false;
2361
2362                         // Any length of checksum
2363                         List<byte> checksum_bytes = new List<byte> (16);
2364
2365                         var checksum_location = Location;
2366                         c = peek_char ();
2367                         while (c != '"' && c != -1) {
2368                                 checksum_bytes.Add (read_hex (out error));
2369                                 if (error)
2370                                         return false;
2371
2372                                 c = peek_char ();
2373                         }
2374
2375                         if (c == '/') {
2376                                 ReadSingleLineComment ();
2377                         } else if (get_char () != '"') {
2378                                 return false;
2379                         }
2380
2381                         if (context.Settings.GenerateDebugInfo) {
2382                                 var chsum = checksum_bytes.ToArray ();
2383
2384                                 if (file.HasChecksum) {
2385                                         if (!ArrayComparer.IsEqual (file.Checksum, chsum)) {
2386                                                 // TODO: Report.SymbolRelatedToPreviousError
2387                                                 Report.Warning (1697, 1, checksum_location, "Different checksum values specified for file `{0}'", file.Name);
2388                                         }
2389                                 }
2390
2391                                 file.SetChecksum (guid_bytes, chsum);
2392                                 current_source.AutoGenerated = true;
2393                         }
2394
2395                         return true;
2396                 }
2397
2398                 bool IsTokenIdentifierEqual (char[] identifier)
2399                 {
2400                         for (int i = 0; i < identifier.Length; ++i) {
2401                                 if (identifier[i] != id_builder[i])
2402                                         return false;
2403                         }
2404
2405                         return true;
2406                 }
2407
2408                 bool ScanClosingInterpolationBrace ()
2409                 {
2410                         PushPosition ();
2411
2412                         bool? res = null;
2413                         int str_quote = 0;
2414                         do {
2415                                 var c = reader.Read ();
2416                                 switch (c) {
2417                                 case '\"':
2418                                         ++str_quote;
2419                                         break;
2420                                 case -1:
2421                                         res = false;
2422                                         break;
2423                                 case '}':
2424                                         if (str_quote % 2 == 1) {
2425                                                 res = true;
2426                                         }
2427
2428                                         break;
2429                                 }
2430                         } while (res == null);
2431
2432                         PopPosition ();
2433                         return res.Value;
2434                 }
2435
2436                 int TokenizeNumber (int value)
2437                 {
2438                         number_pos = 0;
2439
2440                         decimal_digits (value);
2441                         uint ui = (uint) (number_builder[0] - '0');
2442
2443                         try {
2444                                 for (int i = 1; i < number_pos; i++) {
2445                                         ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2446                                 }
2447
2448                                 return (int) ui;
2449                         } catch (OverflowException) {
2450                                 Error_NumericConstantTooLong ();
2451                                 return -1;
2452                         }
2453                 }
2454
2455                 string TokenizeFileName (ref int c)
2456                 {
2457                         var string_builder = new StringBuilder ();
2458                         while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
2459                                 c = get_char ();
2460                                 if (c == '"') {
2461                                         c = get_char ();
2462                                         break;
2463                                 }
2464
2465                                 string_builder.Append ((char) c);
2466                         }
2467
2468                         if (string_builder.Length == 0) {
2469                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
2470                         }
2471
2472
2473                         return string_builder.ToString ();
2474                 }
2475
2476                 int TokenizePragmaWarningIdentifier (ref int c, ref bool identifier)
2477                 {
2478
2479                         if ((c >= '0' && c <= '9') || is_identifier_start_character (c)) {
2480                                 int number;
2481
2482                                 if (c >= '0' && c <= '9') {
2483                                         number_pos = 0;
2484                                         number = TokenizeNumber (c);
2485
2486                                         c = get_char ();
2487
2488                                         if (c != ' ' && c != '\t' && c != ',' && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS) {
2489                                                 return ReadPragmaWarningComment (c);
2490                                         }
2491                                 } else {
2492                                         //
2493                                         // LAMESPEC v6: No spec what identifier really is in this context, it seems keywords are allowed too
2494                                         //
2495                                         int pos = 0;
2496                                         number = -1;
2497                                         id_builder [pos++] = (char)c;
2498                                         while (c < MaxIdentifierLength) {
2499                                                 c = reader.Read ();
2500                                                 id_builder [pos] = (char)c;
2501
2502                                                 if (c >= '0' && c <= '9') {
2503                                                         if (pos == 6 && id_builder [0] == 'C' && id_builder [1] == 'S') {
2504                                                                 // Recognize CSXXXX as C# XXXX warning
2505                                                                 number = 0;
2506                                                                 int pow = 1000;
2507                                                                 for (int i = 0; i < 4; ++i) {
2508                                                                         var ch = id_builder [i + 2];
2509                                                                         if (ch < '0' || ch > '9') {
2510                                                                                 number = -1;
2511                                                                                 break;
2512                                                                         }
2513
2514                                                                         number += (ch - '0') * pow;
2515                                                                         pow /= 10;
2516                                                                 }
2517                                                         }
2518                                                 } else if ((c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && c != '_') {
2519                                                         break;
2520                                                 }
2521
2522                                                 ++pos;
2523                                         }
2524
2525                                         if (number < 0) {
2526                                                 identifier = true;
2527                                                 number = pos;
2528                                         }
2529                                 }
2530
2531                                 // skip over white space
2532                                 while (c == ' ' || c == '\t')
2533                                         c = get_char ();
2534
2535                                 if (c == ',') {
2536                                         c = get_char ();
2537                                 }
2538
2539                                 // skip over white space
2540                                 while (c == ' ' || c == '\t')
2541                                         c = get_char ();
2542
2543                                 return number;
2544                         }
2545
2546                         return ReadPragmaWarningComment (c);
2547                 }
2548
2549                 int ReadPragmaWarningComment (int c)
2550                 {
2551                         if (c == '/') {
2552                                 ReadSingleLineComment ();
2553                         } else {
2554                                 Report.Warning (1692, 1, Location, "Invalid number");
2555
2556                                 // Read everything till the end of the line or file
2557                                 ReadToEndOfLine ();
2558                         }
2559
2560                         return -1;
2561                 }
2562
2563                 void ReadToEndOfLine ()
2564                 {
2565                         int c;
2566                         do {
2567                                 c = get_char ();
2568                         } while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS);
2569                 }
2570
2571                 void ReadSingleLineComment ()
2572                 {
2573                         if (peek_char () != '/')
2574                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2575
2576                         // Read everything till the end of the line or file
2577                         ReadToEndOfLine ();
2578                 }
2579
2580                 /// <summary>
2581                 /// Handles #pragma directive
2582                 /// </summary>
2583                 void ParsePragmaDirective ()
2584                 {
2585                         int c;
2586                         int length = TokenizePreprocessorKeyword (out c);
2587                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2588                                 length = TokenizePreprocessorKeyword (out c);
2589
2590                                 //
2591                                 // #pragma warning disable
2592                                 // #pragma warning restore
2593                                 //
2594                                 if (length == pragma_warning_disable.Length) {
2595                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2596                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2597                                                 // skip over white space
2598                                                 while (c == ' ' || c == '\t')
2599                                                         c = get_char ();
2600
2601                                                 var loc = Location;
2602
2603                                                 if (c == '\n' || c == '/' || c == UnicodeLS || c == UnicodePS) {
2604                                                         if (c == '/')
2605                                                                 ReadSingleLineComment ();
2606
2607                                                         //
2608                                                         // Disable/Restore all warnings
2609                                                         //
2610                                                         if (disable) {
2611                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2612                                                         } else {
2613                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2614                                                         }
2615                                                 } else {
2616                                                         //
2617                                                         // Disable/Restore a warning or group of warnings
2618                                                         //
2619                                                         int code;
2620                                                         do {
2621                                                                 bool identifier = false;
2622                                                                 code = TokenizePragmaWarningIdentifier (ref c, ref identifier);
2623                                                                 if (code > 0) {
2624                                                                         if (identifier) {
2625                                                                                 // no-op, custom warnings cannot occur in mcs
2626                                                                         } else if (disable) {
2627                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, context.Report);
2628                                                                         } else {
2629                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, context);
2630                                                                         }
2631                                                                 }
2632                                                         } while (code >= 0 && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS);
2633                                                 }
2634
2635                                                 return;
2636                                         }
2637                                 }
2638
2639                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2640
2641                                 // Eat any remaining characters on the line
2642                                 ReadToEndOfLine ();
2643
2644                                 return;
2645                         }
2646
2647                         //
2648                         // #pragma checksum
2649                         //
2650                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2651                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2652                                         Report.Warning (1695, 1, Location,
2653                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2654                                 }
2655
2656                                 return;
2657                         }
2658
2659                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2660
2661                         // Eat any remaining characters on the line
2662                         ReadToEndOfLine ();
2663                 }
2664
2665                 bool eval_val (string s)
2666                 {
2667                         if (s == "true")
2668                                 return true;
2669                         if (s == "false")
2670                                 return false;
2671
2672                         return source_file.IsConditionalDefined (s);
2673                 }
2674
2675                 bool pp_primary (ref string s)
2676                 {
2677                         s = s.Trim ();
2678                         int len = s.Length;
2679
2680                         if (len > 0){
2681                                 char c = s [0];
2682
2683                                 if (c == '('){
2684                                         s = s.Substring (1);
2685                                         bool val = pp_expr (ref s, false);
2686                                         if (s.Length > 0 && s [0] == ')'){
2687                                                 s = s.Substring (1);
2688                                                 return val;
2689                                         }
2690                                         Error_InvalidDirective ();
2691                                         return false;
2692                                 }
2693
2694                                 if (is_identifier_start_character (c)){
2695                                         int j = 1;
2696
2697                                         while (j < len){
2698                                                 c = s [j];
2699
2700                                                 if (is_identifier_part_character (c)){
2701                                                         j++;
2702                                                         continue;
2703                                                 }
2704                                                 bool v = eval_val (s.Substring (0, j));
2705                                                 s = s.Substring (j);
2706                                                 return v;
2707                                         }
2708                                         bool vv = eval_val (s);
2709                                         s = "";
2710                                         return vv;
2711                                 }
2712                         }
2713                         Error_InvalidDirective ();
2714                         return false;
2715                 }
2716
2717                 bool pp_unary (ref string s)
2718                 {
2719                         s = s.Trim ();
2720                         int len = s.Length;
2721
2722                         if (len > 0){
2723                                 if (s [0] == '!'){
2724                                         if (len > 1 && s [1] == '='){
2725                                                 Error_InvalidDirective ();
2726                                                 return false;
2727                                         }
2728                                         s = s.Substring (1);
2729                                         return ! pp_primary (ref s);
2730                                 } else
2731                                         return pp_primary (ref s);
2732                         } else {
2733                                 Error_InvalidDirective ();
2734                                 return false;
2735                         }
2736                 }
2737
2738                 bool pp_eq (ref string s)
2739                 {
2740                         bool va = pp_unary (ref s);
2741
2742                         s = s.Trim ();
2743                         int len = s.Length;
2744                         if (len > 0){
2745                                 if (s [0] == '='){
2746                                         if (len > 2 && s [1] == '='){
2747                                                 s = s.Substring (2);
2748                                                 return va == pp_unary (ref s);
2749                                         } else {
2750                                                 Error_InvalidDirective ();
2751                                                 return false;
2752                                         }
2753                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2754                                         s = s.Substring (2);
2755
2756                                         return va != pp_unary (ref s);
2757
2758                                 }
2759                         }
2760
2761                         return va;
2762
2763                 }
2764
2765                 bool pp_and (ref string s)
2766                 {
2767                         bool va = pp_eq (ref s);
2768
2769                         s = s.Trim ();
2770                         int len = s.Length;
2771                         if (len > 0){
2772                                 if (s [0] == '&'){
2773                                         if (len > 2 && s [1] == '&'){
2774                                                 s = s.Substring (2);
2775                                                 return (va & pp_and (ref s));
2776                                         } else {
2777                                                 Error_InvalidDirective ();
2778                                                 return false;
2779                                         }
2780                                 }
2781                         }
2782                         return va;
2783                 }
2784
2785                 //
2786                 // Evaluates an expression for `#if' or `#elif'
2787                 //
2788                 bool pp_expr (ref string s, bool isTerm)
2789                 {
2790                         bool va = pp_and (ref s);
2791                         s = s.Trim ();
2792                         int len = s.Length;
2793                         if (len > 0){
2794                                 char c = s [0];
2795
2796                                 if (c == '|'){
2797                                         if (len > 2 && s [1] == '|'){
2798                                                 s = s.Substring (2);
2799                                                 return va | pp_expr (ref s, isTerm);
2800                                         } else {
2801                                                 Error_InvalidDirective ();
2802                                                 return false;
2803                                         }
2804                                 }
2805                                 if (isTerm) {
2806                                         Error_EndLineExpected ();
2807                                         return false;
2808                                 }
2809                         }
2810
2811                         return va;
2812                 }
2813
2814                 bool eval (string s)
2815                 {
2816                         bool v = pp_expr (ref s, true);
2817                         s = s.Trim ();
2818                         if (s.Length != 0){
2819                                 return false;
2820                         }
2821
2822                         return v;
2823                 }
2824
2825                 void Error_NumericConstantTooLong ()
2826                 {
2827                         Report.Error (1021, Location, "Integral constant is too large");
2828                 }
2829
2830                 void Error_InvalidDirective ()
2831                 {
2832                         Report.Error (1517, Location, "Invalid preprocessor directive");
2833                 }
2834
2835                 void Error_UnexpectedDirective (string extra)
2836                 {
2837                         Report.Error (
2838                                 1028, Location,
2839                                 "Unexpected processor directive ({0})", extra);
2840                 }
2841
2842                 void Error_TokensSeen ()
2843                 {
2844                         Report.Error (1032, Location,
2845                                 "Cannot define or undefine preprocessor symbols after first token in file");
2846                 }
2847
2848                 void Eror_WrongPreprocessorLocation ()
2849                 {
2850                         Report.Error (1040, Location,
2851                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2852                 }
2853
2854                 void Error_EndLineExpected ()
2855                 {
2856                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2857                 }
2858
2859                 //
2860                 // Raises a warning when tokenizer found documentation comment
2861                 // on unexpected place
2862                 //
2863                 void WarningMisplacedComment (Location loc)
2864                 {
2865                         if (doc_state != XmlCommentState.Error) {
2866                                 doc_state = XmlCommentState.Error;
2867                                 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element");
2868                         }
2869                 }
2870
2871                 //
2872                 // if true, then the code continues processing the code
2873                 // if false, the code stays in a loop until another directive is
2874                 // reached.
2875                 // When caller_is_taking is false we ignore all directives except the ones
2876                 // which can help us to identify where the #if block ends
2877                 bool ParsePreprocessingDirective (bool caller_is_taking)
2878                 {
2879                         string arg;
2880                         bool region_directive = false;
2881
2882                         var directive = get_cmd_arg (out arg);
2883
2884                         //
2885                         // The first group of pre-processing instructions is always processed
2886                         //
2887                         switch (directive) {
2888                         case PreprocessorDirective.Region:
2889                                 region_directive = true;
2890                                 arg = "true";
2891                                 goto case PreprocessorDirective.If;
2892
2893                         case PreprocessorDirective.Endregion:
2894                                 if (ifstack == null || ifstack.Count == 0){
2895                                         Error_UnexpectedDirective ("no #region for this #endregion");
2896                                         return true;
2897                                 }
2898                                 int pop = ifstack.Pop ();
2899
2900                                 if ((pop & REGION) == 0)
2901                                         Report.Error (1027, Location, "Expected `#endif' directive");
2902
2903                                 return caller_is_taking;
2904
2905                         case PreprocessorDirective.If:
2906                                 if (ifstack == null)
2907                                         ifstack = new Stack<int> (2);
2908
2909                                 int flags = region_directive ? REGION : 0;
2910                                 if (ifstack.Count == 0){
2911                                         flags |= PARENT_TAKING;
2912                                 } else {
2913                                         int state = ifstack.Peek ();
2914                                         if ((state & TAKING) != 0) {
2915                                                 flags |= PARENT_TAKING;
2916                                         }
2917                                 }
2918
2919                                 if (eval (arg) && caller_is_taking) {
2920                                         ifstack.Push (flags | TAKING);
2921                                         return true;
2922                                 }
2923                                 ifstack.Push (flags);
2924                                 return false;
2925
2926                         case PreprocessorDirective.Endif:
2927                                 if (ifstack == null || ifstack.Count == 0){
2928                                         Error_UnexpectedDirective ("no #if for this #endif");
2929                                         return true;
2930                                 } else {
2931                                         pop = ifstack.Pop ();
2932
2933                                         if ((pop & REGION) != 0)
2934                                                 Report.Error (1038, Location, "#endregion directive expected");
2935
2936                                         if (arg.Length != 0) {
2937                                                 Error_EndLineExpected ();
2938                                         }
2939
2940                                         if (ifstack.Count == 0)
2941                                                 return true;
2942
2943                                         int state = ifstack.Peek ();
2944                                         return (state & TAKING) != 0;
2945                                 }
2946
2947                         case PreprocessorDirective.Elif:
2948                                 if (ifstack == null || ifstack.Count == 0){
2949                                         Error_UnexpectedDirective ("no #if for this #elif");
2950                                         return true;
2951                                 } else {
2952                                         int state = ifstack.Pop ();
2953
2954                                         if ((state & REGION) != 0) {
2955                                                 Report.Error (1038, Location, "#endregion directive expected");
2956                                                 return true;
2957                                         }
2958
2959                                         if ((state & ELSE_SEEN) != 0){
2960                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2961                                                 return true;
2962                                         }
2963
2964                                         if ((state & TAKING) != 0) {
2965                                                 ifstack.Push (0);
2966                                                 return false;
2967                                         }
2968
2969                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2970                                                 ifstack.Push (state | TAKING);
2971                                                 return true;
2972                                         }
2973
2974                                         ifstack.Push (state);
2975                                         return false;
2976                                 }
2977
2978                         case PreprocessorDirective.Else:
2979                                 if (ifstack == null || ifstack.Count == 0){
2980                                         Error_UnexpectedDirective ("no #if for this #else");
2981                                         return true;
2982                                 } else {
2983                                         int state = ifstack.Peek ();
2984
2985                                         if ((state & REGION) != 0) {
2986                                                 Report.Error (1038, Location, "#endregion directive expected");
2987                                                 return true;
2988                                         }
2989
2990                                         if ((state & ELSE_SEEN) != 0){
2991                                                 Error_UnexpectedDirective ("#else within #else");
2992                                                 return true;
2993                                         }
2994
2995                                         ifstack.Pop ();
2996
2997                                         if (arg.Length != 0) {
2998                                                 Error_EndLineExpected ();
2999                                                 return true;
3000                                         }
3001
3002                                         bool ret = false;
3003                                         if ((state & PARENT_TAKING) != 0) {
3004                                                 ret = (state & TAKING) == 0;
3005
3006                                                 if (ret)
3007                                                         state |= TAKING;
3008                                                 else
3009                                                         state &= ~TAKING;
3010                                         }
3011
3012                                         ifstack.Push (state | ELSE_SEEN);
3013
3014                                         return ret;
3015                                 }
3016                         case PreprocessorDirective.Define:
3017                                 if (any_token_seen){
3018                                         if (caller_is_taking)
3019                                                 Error_TokensSeen ();
3020                                         return caller_is_taking;
3021                                 }
3022                                 PreProcessDefinition (true, arg, caller_is_taking);
3023                                 return caller_is_taking;
3024
3025                         case PreprocessorDirective.Undef:
3026                                 if (any_token_seen){
3027                                         if (caller_is_taking)
3028                                                 Error_TokensSeen ();
3029                                         return caller_is_taking;
3030                                 }
3031                                 PreProcessDefinition (false, arg, caller_is_taking);
3032                                 return caller_is_taking;
3033
3034                         case PreprocessorDirective.Invalid:
3035                                 Report.Error (1024, Location, "Wrong preprocessor directive");
3036                                 return true;
3037                         }
3038
3039                         //
3040                         // These are only processed if we are in a `taking' block
3041                         //
3042                         if (!caller_is_taking)
3043                                 return false;
3044
3045                         switch (directive){
3046                         case PreprocessorDirective.Error:
3047                                 Report.Error (1029, Location, "#error: '{0}'", arg);
3048                                 return true;
3049
3050                         case PreprocessorDirective.Warning:
3051                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
3052                                 return true;
3053
3054                         case PreprocessorDirective.Pragma:
3055                                 if (context.Settings.Version == LanguageVersion.ISO_1) {
3056                                         Report.FeatureIsNotAvailable (context, Location, "#pragma");
3057                                 }
3058
3059                                 ParsePragmaDirective ();
3060                                 return true;
3061
3062                         case PreprocessorDirective.Line:
3063                                 Location loc = Location;
3064                                 if (!PreProcessLine ())
3065                                         Report.Error (1576, loc, "The line number specified for #line directive is missing or invalid");
3066
3067                                 return caller_is_taking;
3068                         }
3069
3070                         throw new NotImplementedException (directive.ToString ());
3071                 }
3072
3073                 int consume_string (bool quoted)
3074                 {
3075                         int c;
3076                         int pos = 0;
3077                         Location start_location = Location;
3078                         if (quoted)
3079                                 start_location = start_location - 1;
3080
3081 #if FULL_AST
3082                         int reader_pos = reader.Position;
3083 #endif
3084
3085                         while (true){
3086                                 // Cannot use get_char because of \r in quoted strings
3087                                 if (putback_char != -1) {
3088                                         c = putback_char;
3089                                         putback_char = -1;
3090                                 } else {
3091                                         c = reader.Read ();
3092                                 }
3093
3094                                 if (c == '"') {
3095                                         ++col;
3096
3097                                         if (quoted && peek_char () == '"') {
3098                                                 if (pos == value_builder.Length)
3099                                                         Array.Resize (ref value_builder, pos * 2);
3100
3101                                                 value_builder[pos++] = (char) c;
3102                                                 get_char ();
3103                                                 continue;
3104                                         }
3105
3106                                         ILiteralConstant res = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location);
3107                                         val = res;
3108 #if FULL_AST
3109                                         res.ParsedValue = quoted ?
3110                                                 reader.ReadChars (reader_pos - 2, reader.Position - 1) :
3111                                                 reader.ReadChars (reader_pos - 1, reader.Position);
3112 #endif
3113
3114                                         return Token.LITERAL;
3115                                 }
3116
3117                                 if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
3118                                         if (!quoted) {
3119                                                 Report.Error (1010, Location, "Newline in constant");
3120
3121                                                 advance_line ();
3122
3123                                                 // Don't add \r to string literal
3124                                                 if (pos > 1 && value_builder [pos - 1] == '\r')
3125                                                         --pos;
3126
3127                                                 val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location);
3128                                                 return Token.LITERAL;
3129                                         }
3130
3131                                         advance_line ();
3132                                 } else if (c == '\\' && !quoted) {
3133                                         ++col;
3134                                         int surrogate;
3135                                         c = escape (c, out surrogate);
3136                                         if (c == -1)
3137                                                 return Token.ERROR;
3138                                         if (surrogate != 0) {
3139                                                 if (pos == value_builder.Length)
3140                                                         Array.Resize (ref value_builder, pos * 2);
3141
3142                                                 value_builder[pos++] = (char) c;
3143                                                 c = surrogate;
3144                                         }
3145                                 } else if (c == -1) {
3146                                         Report.Error (1039, Location, "Unterminated string literal");
3147                                         return Token.EOF;
3148                                 } else {
3149                                         ++col;
3150                                 }
3151
3152                                 if (pos == value_builder.Length)
3153                                         Array.Resize (ref value_builder, pos * 2);
3154
3155                                 value_builder[pos++] = (char) c;
3156                         }
3157                 }
3158
3159                 private int consume_identifier (int s)
3160                 {
3161                         int res = consume_identifier (s, false);
3162
3163                         if (doc_state == XmlCommentState.Allowed)
3164                                 doc_state = XmlCommentState.NotAllowed;
3165
3166                         return res;
3167                 }
3168
3169                 int consume_identifier (int c, bool quoted)
3170                 {
3171                         //
3172                         // This method is very performance sensitive. It accounts
3173                         // for approximately 25% of all parser time
3174                         //
3175
3176                         int pos = 0;
3177                         int column = col;
3178                         if (quoted)
3179                                 --column;
3180
3181                         if (c == '\\') {
3182                                 int surrogate;
3183                                 c = escape (c, out surrogate);
3184                                 if (quoted || is_identifier_start_character (c)) {
3185                                         // it's added bellow
3186                                 } else if (surrogate != 0) {
3187                                         id_builder [pos++] = (char)c;
3188                                         c = surrogate;
3189                                 } else {
3190                                         Report.Error (1056, Location, "Unexpected character `\\{0}'", c.ToString ("x4"));
3191                                         return Token.ERROR;
3192                                 }
3193                         }
3194
3195                         id_builder [pos++] = (char) c;
3196
3197                         try {
3198                                 while (true) {
3199                                         c = reader.Read ();
3200
3201                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
3202                                                 id_builder [pos++] = (char) c;
3203                                                 continue;
3204                                         }
3205
3206                                         if (c < 0x80) {
3207                                                 if (c == '\\') {
3208                                                         int surrogate;
3209                                                         c = escape (c, out surrogate);
3210                                                         if (is_identifier_part_character ((char) c))
3211                                                                 id_builder[pos++] = (char) c;
3212                                                         else if (surrogate != 0) {
3213                                                                 c = surrogate;
3214                                                         } else {
3215                                                                 switch (c) {
3216                                                                 // TODO: Probably need more whitespace characters
3217                                                                 case 0xFEFF:
3218                                                                         putback_char = c;
3219                                                                         break;
3220                                                                 default:
3221                                                                         Report.Error (1056, Location, "Unexpected character `\\{0}'", c.ToString ("x4"));
3222                                                                         return Token.ERROR;
3223                                                                 }
3224                                                         }
3225
3226                                                         continue;
3227                                                 }
3228                                         } else if (is_identifier_part_character_slow_part ((char) c)) {
3229                                                 id_builder [pos++] = (char) c;
3230                                                 continue;
3231                                         }
3232
3233                                         putback_char = c;
3234                                         break;
3235                                 }
3236                         } catch (IndexOutOfRangeException) {
3237                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
3238                                 --pos;
3239                                 col += pos;
3240                         }
3241
3242                         col += pos - 1;
3243
3244                         //
3245                         // Optimization: avoids doing the keyword lookup
3246                         // on uppercase letters
3247                         //
3248                         if (id_builder [0] >= '_' && !quoted) {
3249                                 int keyword = GetKeyword (id_builder, pos);
3250                                 if (keyword != -1) {
3251                                         val = ltb.Create (keyword == Token.AWAIT ? "await" : null, current_source, ref_line, column);
3252                                         return keyword;
3253                                 }
3254                         }
3255
3256                         string s = InternIdentifier (id_builder, pos);
3257                         val = ltb.Create (s, current_source, ref_line, column);
3258                         if (quoted && parsing_attribute_section)
3259                                 AddEscapedIdentifier (((LocatedToken) val).Location);
3260
3261                         return Token.IDENTIFIER;
3262                 }
3263
3264                 string InternIdentifier (char[] charBuffer, int length)
3265                 {
3266                         //
3267                         // Keep identifiers in an array of hashtables to avoid needless
3268                         // allocations
3269                         //
3270                         var identifiers_group = identifiers[length];
3271                         string s;
3272                         if (identifiers_group != null) {
3273                                 if (identifiers_group.TryGetValue (charBuffer, out s)) {
3274                                         return s;
3275                                 }
3276                         } else {
3277                                 // TODO: this should be number of files dependant
3278                                 // corlib compilation peaks at 1000 and System.Core at 150
3279                                 int capacity = length > 20 ? 10 : 100;
3280                                 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length));
3281                                 identifiers[length] = identifiers_group;
3282                         }
3283
3284                         char[] chars = new char[length];
3285                         Array.Copy (charBuffer, chars, length);
3286
3287                         s = new string (charBuffer, 0, length);
3288                         identifiers_group.Add (chars, s);
3289                         return s;
3290                 }
3291
3292                 public int xtoken ()
3293                 {
3294                         if (parsing_interpolation_format) {
3295                                 return TokenizeInterpolationFormat ();
3296                         }
3297
3298                         int d, c;
3299
3300                         // Whether we have seen comments on the current line
3301                         bool comments_seen = false;
3302                         while ((c = get_char ()) != -1) {
3303                                 switch (c) {
3304                                 case '\t':
3305                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
3306                                         continue;
3307
3308                                 case ' ':
3309                                 case '\f':
3310                                 case '\v':
3311                                 case 0xa0:
3312                                 case 0:
3313                                 case 0xFEFF:    // Ignore BOM anywhere in the file
3314                                         continue;
3315
3316 /*                              This is required for compatibility with .NET
3317                                 case 0xEF:
3318                                         if (peek_char () == 0xBB) {
3319                                                 PushPosition ();
3320                                                 get_char ();
3321                                                 if (get_char () == 0xBF)
3322                                                         continue;
3323                                                 PopPosition ();
3324                                         }
3325                                         break;
3326 */
3327                                 case '\\':
3328                                         tokens_seen = true;
3329                                         return consume_identifier (c);
3330
3331                                 case '{':
3332                                         val = ltb.Create (current_source, ref_line, col);
3333                                         return Token.OPEN_BRACE;
3334                                 case '}':
3335                                         if (parsing_string_interpolation > 0) {
3336                                                 if (peek_char () != '}') {
3337                                                         --parsing_string_interpolation;
3338                                                         return TokenizeInterpolatedString ();
3339                                                 }
3340
3341                                                 continue;
3342                                         }
3343
3344                                         val = ltb.Create (current_source, ref_line, col);
3345                                         return Token.CLOSE_BRACE;
3346                                 case '[':
3347                                         // To block doccomment inside attribute declaration.
3348                                         if (doc_state == XmlCommentState.Allowed)
3349                                                 doc_state = XmlCommentState.NotAllowed;
3350
3351                                         val = ltb.Create (current_source, ref_line, col);
3352
3353                                         if (parsing_block == 0 || lambda_arguments_parsing)
3354                                                 return Token.OPEN_BRACKET;
3355
3356                                         int next = peek_char ();
3357                                         switch (next) {
3358                                         case ']':
3359                                         case ',':
3360                                                 return Token.OPEN_BRACKET;
3361
3362                                         case ' ':
3363                                         case '\f':
3364                                         case '\v':
3365                                         case '\r':
3366                                         case '\n':
3367                                         case UnicodeLS:
3368                                         case UnicodePS:
3369                                         case '/':
3370                                                 next = peek_token ();
3371                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
3372                                                         return Token.OPEN_BRACKET;
3373
3374                                                 return Token.OPEN_BRACKET_EXPR;
3375                                         default:
3376                                                 return Token.OPEN_BRACKET_EXPR;
3377                                         }
3378                                 case ']':
3379                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3380                                         return Token.CLOSE_BRACKET;
3381                                 case '(':
3382                                         val = ltb.Create (current_source, ref_line, col);
3383                                         //
3384                                         // An expression versions of parens can appear in block context only
3385                                         //
3386                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
3387
3388                                                 //
3389                                                 // Optmize most common case where we know that parens
3390                                                 // is not special
3391                                                 //
3392                                                 switch (current_token) {
3393                                                 case Token.IDENTIFIER:
3394                                                 case Token.IF:
3395                                                 case Token.FOR:
3396                                                 case Token.FOREACH:
3397                                                 case Token.TYPEOF:
3398                                                 case Token.WHILE:
3399                                                 case Token.SWITCH:
3400                                                 case Token.USING:
3401                                                 case Token.DEFAULT:
3402                                                 case Token.DELEGATE:
3403                                                 case Token.OP_GENERICS_GT:
3404                                                         return Token.OPEN_PARENS;
3405                                                 }
3406
3407                                                 // Optimize using peek
3408                                                 int xx = peek_char ();
3409                                                 switch (xx) {
3410                                                 case '(':
3411                                                 case '\'':
3412                                                 case '"':
3413                                                 case '0':
3414                                                 case '1':
3415                                                         return Token.OPEN_PARENS;
3416                                                 }
3417
3418                                                 lambda_arguments_parsing = true;
3419                                                 PushPosition ();
3420                                                 d = TokenizeOpenParens ();
3421                                                 PopPosition ();
3422                                                 lambda_arguments_parsing = false;
3423                                                 return d;
3424                                         }
3425
3426                                         return Token.OPEN_PARENS;
3427                                 case ')':
3428                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3429                                         return Token.CLOSE_PARENS;
3430                                 case ',':
3431                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3432                                         return Token.COMMA;
3433                                 case ';':
3434                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3435                                         return Token.SEMICOLON;
3436                                 case '~':
3437                                         val = ltb.Create (current_source, ref_line, col);
3438                                         return Token.TILDE;
3439                                 case '?':
3440                                         val = ltb.Create (current_source, ref_line, col);
3441                                         return TokenizePossibleNullableType ();
3442                                 case '<':
3443                                         val = ltb.Create (current_source, ref_line, col);
3444                                         if (parsing_generic_less_than++ > 0)
3445                                                 return Token.OP_GENERICS_LT;
3446
3447                                         return TokenizeLessThan ();
3448
3449                                 case '>':
3450                                         val = ltb.Create (current_source, ref_line, col);
3451                                         d = peek_char ();
3452
3453                                         if (d == '='){
3454                                                 get_char ();
3455                                                 return Token.OP_GE;
3456                                         }
3457
3458                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
3459                                                 parsing_generic_less_than--;
3460                                                 return Token.OP_GENERICS_GT;
3461                                         }
3462
3463                                         if (d == '>') {
3464                                                 get_char ();
3465                                                 d = peek_char ();
3466
3467                                                 if (d == '=') {
3468                                                         get_char ();
3469                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
3470                                                 }
3471                                                 return Token.OP_SHIFT_RIGHT;
3472                                         }
3473
3474                                         return Token.OP_GT;
3475
3476                                 case '+':
3477                                         val = ltb.Create (current_source, ref_line, col);
3478                                         d = peek_char ();
3479                                         if (d == '+') {
3480                                                 d = Token.OP_INC;
3481                                         } else if (d == '=') {
3482                                                 d = Token.OP_ADD_ASSIGN;
3483                                         } else {
3484                                                 return Token.PLUS;
3485                                         }
3486                                         get_char ();
3487                                         return d;
3488
3489                                 case '-':
3490                                         val = ltb.Create (current_source, ref_line, col);
3491                                         d = peek_char ();
3492                                         if (d == '-') {
3493                                                 d = Token.OP_DEC;
3494                                         } else if (d == '=')
3495                                                 d = Token.OP_SUB_ASSIGN;
3496                                         else if (d == '>')
3497                                                 d = Token.OP_PTR;
3498                                         else {
3499                                                 return Token.MINUS;
3500                                         }
3501                                         get_char ();
3502                                         return d;
3503
3504                                 case '!':
3505                                         val = ltb.Create (current_source, ref_line, col);
3506                                         if (peek_char () == '='){
3507                                                 get_char ();
3508                                                 return Token.OP_NE;
3509                                         }
3510                                         return Token.BANG;
3511
3512                                 case '=':
3513                                         val = ltb.Create (current_source, ref_line, col);
3514                                         d = peek_char ();
3515                                         if (d == '='){
3516                                                 get_char ();
3517                                                 return Token.OP_EQ;
3518                                         }
3519                                         if (d == '>'){
3520                                                 get_char ();
3521                                                 return Token.ARROW;
3522                                         }
3523
3524                                         return Token.ASSIGN;
3525
3526                                 case '&':
3527                                         val = ltb.Create (current_source, ref_line, col);
3528                                         d = peek_char ();
3529                                         if (d == '&'){
3530                                                 get_char ();
3531                                                 return Token.OP_AND;
3532                                         }
3533                                         if (d == '='){
3534                                                 get_char ();
3535                                                 return Token.OP_AND_ASSIGN;
3536                                         }
3537                                         return Token.BITWISE_AND;
3538
3539                                 case '|':
3540                                         val = ltb.Create (current_source, ref_line, col);
3541                                         d = peek_char ();
3542                                         if (d == '|'){
3543                                                 get_char ();
3544                                                 return Token.OP_OR;
3545                                         }
3546                                         if (d == '='){
3547                                                 get_char ();
3548                                                 return Token.OP_OR_ASSIGN;
3549                                         }
3550                                         return Token.BITWISE_OR;
3551
3552                                 case '*':
3553                                         val = ltb.Create (current_source, ref_line, col);
3554                                         if (peek_char () == '='){
3555                                                 get_char ();
3556                                                 return Token.OP_MULT_ASSIGN;
3557                                         }
3558                                         return Token.STAR;
3559
3560                                 case '/':
3561                                         d = peek_char ();
3562                                         if (d == '='){
3563                                                 val = ltb.Create (current_source, ref_line, col);
3564                                                 get_char ();
3565                                                 return Token.OP_DIV_ASSIGN;
3566                                         }
3567
3568                                         // Handle double-slash comments.
3569                                         if (d == '/'){
3570                                                 if (parsing_string_interpolation > 0) {
3571                                                         Report.Error (8077, Location, "A single-line comment may not be used in an interpolated string");
3572                                                         goto case '}';
3573                                                 }
3574
3575                                                 get_char ();
3576                                                 if (doc_processing) {
3577                                                         if (peek_char () == '/') {
3578                                                                 get_char ();
3579                                                                 // Don't allow ////.
3580                                                                 if ((d = peek_char ()) != '/') {
3581                                                                         if (doc_state == XmlCommentState.Allowed)
3582                                                                                 handle_one_line_xml_comment ();
3583                                                                         else if (doc_state == XmlCommentState.NotAllowed)
3584                                                                                 WarningMisplacedComment (Location - 3);
3585                                                                 }
3586                                                         } else {
3587                                                                 if (xml_comment_buffer.Length > 0)
3588                                                                         doc_state = XmlCommentState.NotAllowed;
3589                                                         }
3590                                                 }
3591
3592                                                 ReadToEndOfLine ();
3593
3594                                                 any_token_seen |= tokens_seen;
3595                                                 tokens_seen = false;
3596                                                 comments_seen = false;
3597                                                 continue;
3598                                         } else if (d == '*'){
3599                                                 get_char ();
3600                                                 bool docAppend = false;
3601                                                 if (doc_processing && peek_char () == '*') {
3602                                                         get_char ();
3603                                                         // But when it is /**/, just do nothing.
3604                                                         if (peek_char () == '/') {
3605                                                                 get_char ();
3606                                                                 continue;
3607                                                         }
3608                                                         if (doc_state == XmlCommentState.Allowed)
3609                                                                 docAppend = true;
3610                                                         else if (doc_state == XmlCommentState.NotAllowed) {
3611                                                                 WarningMisplacedComment (Location - 2);
3612                                                         }
3613                                                 }
3614
3615                                                 int current_comment_start = 0;
3616                                                 if (docAppend) {
3617                                                         current_comment_start = xml_comment_buffer.Length;
3618                                                         xml_comment_buffer.Append (Environment.NewLine);
3619                                                 }
3620
3621                                                 while ((d = get_char ()) != -1){
3622                                                         if (d == '*' && peek_char () == '/'){
3623                                                                 get_char ();
3624                                                                 comments_seen = true;
3625                                                                 break;
3626                                                         }
3627                                                         if (docAppend)
3628                                                                 xml_comment_buffer.Append ((char) d);
3629
3630                                                         if (d == '\n' || d == UnicodeLS || d == UnicodePS){
3631                                                                 any_token_seen |= tokens_seen;
3632                                                                 tokens_seen = false;
3633                                                                 //
3634                                                                 // Reset 'comments_seen' just to be consistent.
3635                                                                 // It doesn't matter either way, here.
3636                                                                 //
3637                                                                 comments_seen = false;
3638                                                         }
3639                                                 }
3640                                                 if (!comments_seen)
3641                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3642
3643                                                 if (docAppend)
3644                                                         update_formatted_doc_comment (current_comment_start);
3645                                                 continue;
3646                                         }
3647                                         val = ltb.Create (current_source, ref_line, col);
3648                                         return Token.DIV;
3649
3650                                 case '%':
3651                                         val = ltb.Create (current_source, ref_line, col);
3652                                         if (peek_char () == '='){
3653                                                 get_char ();
3654                                                 return Token.OP_MOD_ASSIGN;
3655                                         }
3656                                         return Token.PERCENT;
3657
3658                                 case '^':
3659                                         val = ltb.Create (current_source, ref_line, col);
3660                                         if (peek_char () == '='){
3661                                                 get_char ();
3662                                                 return Token.OP_XOR_ASSIGN;
3663                                         }
3664                                         return Token.CARRET;
3665
3666                                 case ':':
3667                                         val = ltb.Create (current_source, ref_line, col);
3668                                         if (peek_char () == ':') {
3669                                                 get_char ();
3670                                                 return Token.DOUBLE_COLON;
3671                                         }
3672                                         return Token.COLON;
3673
3674                                 case '0': case '1': case '2': case '3': case '4':
3675                                 case '5': case '6': case '7': case '8': case '9':
3676                                         tokens_seen = true;
3677                                         return is_number (c, false);
3678
3679                                 case '\n': // white space
3680                                 case UnicodeLS:
3681                                 case UnicodePS:
3682                                         any_token_seen |= tokens_seen;
3683                                         tokens_seen = false;
3684                                         comments_seen = false;
3685                                         continue;
3686
3687                                 case '.':
3688                                         tokens_seen = true;
3689                                         d = peek_char ();
3690                                         if (d >= '0' && d <= '9')
3691                                                 return is_number (c, true);
3692
3693                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3694                                         return Token.DOT;
3695
3696                                 case '#':
3697                                         if (tokens_seen || comments_seen) {
3698                                                 Eror_WrongPreprocessorLocation ();
3699                                                 return Token.ERROR;
3700                                         }
3701
3702                                         if (ParsePreprocessingDirective (true))
3703                                                 continue;
3704
3705                                         bool directive_expected = false;
3706                                         while ((c = get_char ()) != -1) {
3707                                                 if (col == 1) {
3708                                                         directive_expected = true;
3709                                                 } else if (!directive_expected) {
3710                                                         // TODO: Implement comment support for disabled code and uncomment this code
3711 //                                                      if (c == '#') {
3712 //                                                              Eror_WrongPreprocessorLocation ();
3713 //                                                              return Token.ERROR;
3714 //                                                      }
3715                                                         continue;
3716                                                 }
3717
3718                                                 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' || c == UnicodeLS || c == UnicodePS)
3719                                                         continue;
3720
3721                                                 if (c == '#') {
3722                                                         if (ParsePreprocessingDirective (false))
3723                                                                 break;
3724                                                 }
3725                                                 directive_expected = false;
3726                                         }
3727
3728                                         if (c != -1) {
3729                                                 tokens_seen = false;
3730                                                 continue;
3731                                         }
3732
3733                                         return Token.EOF;
3734
3735                                 case '"':
3736                                         if (parsing_string_interpolation > 0 && !ScanClosingInterpolationBrace ()) {
3737                                                 parsing_string_interpolation = 0;
3738                                                 Report.Error (8076, Location, "Missing close delimiter `}' for interpolated expression");
3739                                                 val = new StringLiteral (context.BuiltinTypes, "", Location);
3740                                                 return Token.INTERPOLATED_STRING_END;
3741                                         }
3742
3743                                         return consume_string (false);
3744
3745                                 case '\'':
3746                                         return TokenizeBackslash ();
3747
3748                                 case '@':
3749                                         c = get_char ();
3750                                         if (c == '"') {
3751                                                 tokens_seen = true;
3752                                                 return consume_string (true);
3753                                         }
3754
3755                                         if (is_identifier_start_character (c)){
3756                                                 return consume_identifier (c, true);
3757                                         }
3758
3759                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3760                                         return Token.ERROR;
3761
3762                                 case '$':
3763                                         if (peek_char () == '"') {
3764                                                 get_char ();
3765                                                 return TokenizeInterpolatedString ();
3766                                         }
3767
3768                                         break;
3769                                 case EvalStatementParserCharacter:
3770                                         return Token.EVAL_STATEMENT_PARSER;
3771                                 case EvalCompilationUnitParserCharacter:
3772                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3773                                 case EvalUsingDeclarationsParserCharacter:
3774                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3775                                 case DocumentationXref:
3776                                         return Token.DOC_SEE;
3777                                 }
3778
3779                                 if (is_identifier_start_character (c)) {
3780                                         tokens_seen = true;
3781                                         return consume_identifier (c);
3782                                 }
3783
3784                                 if (char.IsWhiteSpace ((char) c))
3785                                         continue;
3786
3787                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3788                         }
3789
3790                         if (CompleteOnEOF){
3791                                 if (generated)
3792                                         return Token.COMPLETE_COMPLETION;
3793
3794                                 generated = true;
3795                                 return Token.GENERATE_COMPLETION;
3796                         }
3797
3798
3799                         return Token.EOF;
3800                 }
3801
3802                 int TokenizeBackslash ()
3803                 {
3804 #if FULL_AST
3805                         int read_start = reader.Position;
3806 #endif
3807                         Location start_location = Location;
3808                         int c = get_char ();
3809                         tokens_seen = true;
3810                         if (c == '\'') {
3811                                 val = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3812                                 Report.Error (1011, start_location, "Empty character literal");
3813                                 return Token.LITERAL;
3814                         }
3815
3816                         if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
3817                                 Report.Error (1010, start_location, "Newline in constant");
3818                                 return Token.ERROR;
3819                         }
3820
3821                         int d;
3822                         c = escape (c, out d);
3823                         if (c == -1)
3824                                 return Token.ERROR;
3825                         if (d != 0)
3826                                 throw new NotImplementedException ();
3827
3828                         ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3829                         val = res;
3830                         c = get_char ();
3831
3832                         if (c != '\'') {
3833                                 Report.Error (1012, start_location, "Too many characters in character literal");
3834
3835                                 // Try to recover, read until newline or next "'"
3836                                 while ((c = get_char ()) != -1) {
3837                                         if (c == '\n' || c == '\'' || c == UnicodeLS || c == UnicodePS)
3838                                                 break;
3839                                 }
3840                         }
3841
3842 #if FULL_AST
3843                         res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position);
3844 #endif
3845
3846                         return Token.LITERAL;
3847                 }
3848
3849                 int TokenizeLessThan ()
3850                 {
3851                         int d;
3852
3853                         // Save current position and parse next token.
3854                         PushPosition ();
3855                         int generic_dimension = 0;
3856                         if (parse_less_than (ref generic_dimension)) {
3857                                 if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) {
3858                                         d = Token.OP_GENERICS_LT_DECL;
3859                                 } else {
3860                                         if (generic_dimension > 0) {
3861                                                 val = generic_dimension;
3862                                                 DiscardPosition ();
3863                                                 return Token.GENERIC_DIMENSION;
3864                                         }
3865
3866                                         d = Token.OP_GENERICS_LT;
3867                                 }
3868                                 PopPosition ();
3869                                 return d;
3870                         }
3871
3872                         PopPosition ();
3873                         parsing_generic_less_than = 0;
3874
3875                         d = peek_char ();
3876                         if (d == '<') {
3877                                 get_char ();
3878                                 d = peek_char ();
3879
3880                                 if (d == '=') {
3881                                         get_char ();
3882                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3883                                 }
3884                                 return Token.OP_SHIFT_LEFT;
3885                         }
3886
3887                         if (d == '=') {
3888                                 get_char ();
3889                                 return Token.OP_LE;
3890                         }
3891                         return Token.OP_LT;
3892                 }
3893
3894                 int TokenizeInterpolatedString ()
3895                 {
3896                         int pos = 0;
3897                         var start_location = Location;
3898
3899                         while (true) {
3900                                 var ch = get_char ();
3901                                 switch (ch) {
3902                                 case '"':
3903                                         val = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location);
3904                                         return Token.INTERPOLATED_STRING_END;
3905                                 case '{':
3906                                         if (peek_char () == '{') {
3907                                                 value_builder [pos++] = (char)ch;
3908                                                 get_char ();
3909                                                 break;
3910                                         }
3911
3912                                         ++parsing_string_interpolation;
3913                                         val = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location);
3914                                         return Token.INTERPOLATED_STRING;
3915                                 case '\\':
3916                                         ++col;
3917                                         int surrogate;
3918                                         ch = escape (ch, out surrogate);
3919                                         if (ch == -1)
3920                                                 return Token.ERROR;
3921
3922                                         if (ch == '{' || ch == '}') {
3923                                                 Report.Error (8087, Location, "A `{0}' character may only be escaped by doubling `{0}{0}' in an interpolated string", ((char) ch).ToString ());
3924                                         }
3925
3926                                         if (surrogate != 0) {
3927                                                 if (pos == value_builder.Length)
3928                                                         Array.Resize (ref value_builder, pos * 2);
3929
3930                                                 value_builder [pos++] = (char)ch;
3931                                                 ch = surrogate;
3932                                         }
3933
3934                                         break;
3935                                 case -1:
3936                                         return Token.EOF;
3937                                 }
3938
3939                                 ++col;
3940                                 value_builder[pos++] = (char) ch;
3941                         }
3942                 }
3943
3944                 int TokenizeInterpolationFormat ()
3945                 {
3946                         int pos = 0;
3947                         int braces = 0;
3948                         while (true) {
3949                                 var ch = get_char ();
3950                                 switch (ch) {
3951                                 case '{':
3952                                         ++braces;
3953                                         break;
3954                                 case '}':
3955                                         if (braces == 0) {
3956                                                 putback_char = ch;
3957                                                 if (pos == 0) {
3958                                                         Report.Error (8089, Location, "Empty interpolated expression format specifier");
3959                                                 } else if (Array.IndexOf (simple_whitespaces, value_builder [pos - 1]) >= 0) {
3960                                                         Report.Error (8088, Location, "A interpolated expression format specifier may not contain trailing whitespace");
3961                                                 }
3962
3963                                                 val = CreateStringFromBuilder (pos);
3964                                                 return Token.LITERAL;
3965                                         }
3966
3967                                         --braces;
3968                                         break;
3969                                 case '\\':
3970                                         ++col;
3971                                         int surrogate;
3972                                         ch = escape (ch, out surrogate);
3973                                         if (ch == -1)
3974                                                 return Token.ERROR;
3975
3976                                         if (ch == '{' || ch == '}') {
3977                                                 Report.Error (8087, Location, "A `{0}' character may only be escaped by doubling `{0}{0}' in an interpolated string", ((char) ch).ToString ());
3978                                         }
3979
3980                                         if (surrogate != 0) {
3981                                                 if (pos == value_builder.Length)
3982                                                         Array.Resize (ref value_builder, pos * 2);
3983
3984                                                 value_builder [pos++] = (char)ch;
3985                                                 ch = surrogate;
3986                                         }
3987
3988                                         break;
3989                                 case -1:
3990                                         return Token.EOF;
3991                                 }
3992
3993                                 ++col;
3994                                 value_builder[pos++] = (char) ch;
3995                         }
3996                 }
3997
3998                 string CreateStringFromBuilder (int pos)
3999                 {
4000                         if (pos == 0)
4001                                 return string.Empty;
4002                         if (pos <= 4)
4003                                 return InternIdentifier (value_builder, pos);
4004
4005                         return new string (value_builder, 0, pos);
4006                 }
4007
4008                 //
4009                 // Handles one line xml comment
4010                 //
4011                 private void handle_one_line_xml_comment ()
4012                 {
4013                         int c;
4014                         while ((c = peek_char ()) == ' ')
4015                                 get_char (); // skip heading whitespaces.
4016                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
4017                                 xml_comment_buffer.Append ((char) get_char ());
4018                         }
4019                         if (c == '\r' || c == '\n')
4020                                 xml_comment_buffer.Append (Environment.NewLine);
4021                 }
4022
4023                 //
4024                 // Remove heading "*" in Javadoc-like xml documentation.
4025                 //
4026                 private void update_formatted_doc_comment (int current_comment_start)
4027                 {
4028                         int length = xml_comment_buffer.Length - current_comment_start;
4029                         string [] lines = xml_comment_buffer.ToString (
4030                                 current_comment_start,
4031                                 length).Replace ("\r", "").Split ('\n');
4032
4033                         // The first line starts with /**, thus it is not target
4034                         // for the format check.
4035                         for (int i = 1; i < lines.Length; i++) {
4036                                 string s = lines [i];
4037                                 int idx = s.IndexOf ('*');
4038                                 string head = null;
4039                                 if (idx < 0) {
4040                                         if (i < lines.Length - 1)
4041                                                 return;
4042                                         head = s;
4043                                 } else
4044                                         head = s.Substring (0, idx);
4045                                 foreach (char c in head)
4046                                         if (c != ' ')
4047                                                 return;
4048                                 lines [i] = s.Substring (idx + 1);
4049                         }
4050                         xml_comment_buffer.Remove (current_comment_start, length);
4051                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
4052                 }
4053
4054                 //
4055                 // Checks if there was incorrect doc comments and raise
4056                 // warnings.
4057                 //
4058                 public void check_incorrect_doc_comment ()
4059                 {
4060                         if (xml_comment_buffer.Length > 0)
4061                                 WarningMisplacedComment (Location);
4062                 }
4063
4064                 //
4065                 // Consumes the saved xml comment lines (if any)
4066                 // as for current target member or type.
4067                 //
4068                 public string consume_doc_comment ()
4069                 {
4070                         if (xml_comment_buffer.Length > 0) {
4071                                 string ret = xml_comment_buffer.ToString ();
4072                                 reset_doc_comment ();
4073                                 return ret;
4074                         }
4075                         return null;
4076                 }
4077
4078                 void reset_doc_comment ()
4079                 {
4080                         xml_comment_buffer.Length = 0;
4081                 }
4082
4083                 public void cleanup ()
4084                 {
4085                         if (ifstack != null && ifstack.Count >= 1) {
4086                                 int state = ifstack.Pop ();
4087                                 if ((state & REGION) != 0)
4088                                         Report.Error (1038, Location, "#endregion directive expected");
4089                                 else
4090                                         Report.Error (1027, Location, "Expected `#endif' directive");
4091                         }
4092                 }
4093         }
4094
4095         //
4096         // Indicates whether it accepts XML documentation or not.
4097         //
4098         public enum XmlCommentState {
4099                 // comment is allowed in this state.
4100                 Allowed,
4101                 // comment is not allowed in this state.
4102                 NotAllowed,
4103                 // once comments appeared when it is NotAllowed, then the
4104                 // state is changed to it, until the state is changed to
4105                 // .Allowed.
4106                 Error
4107         }
4108 }
4109