mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@gmail.com)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 // Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20 using System.Collections;
  21
  22 namespace Mono.CSharp
  23 {
  24         //
  25         // This class has to be used by parser only, it reuses token
  26         // details once a file is parsed
  27         //
  28         public class LocatedToken
  29         {
  30                 public int row, column;
  31                 public string value;
  32                 public SourceFile file;
  33
  34                 public LocatedToken ()
  35                 {
  36                 }
  37
  38                 public LocatedToken (string value, Location loc)
  39                 {
  40                         this.value = value;
  41                         file = loc.SourceFile;
  42                         row = loc.Row;
  43                         column = loc.Column;
  44                 }
  45
  46                 public override string ToString ()
  47                 {
  48                         return string.Format ("Token '{0}' at {1},{2}", Value, row, column);
  49                 }
  50
  51                 public Location Location
  52                 {
  53                         get { return new Location (file, row, column); }
  54                 }
  55
  56                 public string Value
  57                 {
  58                         get { return value; }
  59                 }
  60         }
  61
  62         /// <summary>
  63         ///    Tokenizer for C# source code.
  64         /// </summary>
  65         public class Tokenizer : yyParser.yyInput
  66         {
  67                 class KeywordEntry<T>
  68                 {
  69                         public readonly T Token;
  70                         public KeywordEntry<T> Next;
  71                         public readonly char[] Value;
  72
  73                         public KeywordEntry (string value, T token)
  74                         {
  75                                 this.Value = value.ToCharArray ();
  76                                 this.Token = token;
  77                         }
  78                 }
  79
  80                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  81                 {
  82                         readonly int length;
  83
  84                         public IdentifiersComparer (int length)
  85                         {
  86                                 this.length = length;
  87                         }
  88
  89                         public bool Equals (char[] x, char[] y)
  90                         {
  91                                 for (int i = 0; i < length; ++i)
  92                                         if (x [i] != y [i])
  93                                                 return false;
  94
  95                                 return true;
  96                         }
  97
  98                         public int GetHashCode (char[] obj)
  99                         {
 100                                 int h = 0;
 101                                 for (int i = 0; i < length; ++i)
 102                                         h = (h << 5) - h + obj [i];
 103
 104                                 return h;
 105                         }
 106                 }
 107
 108                 public class LocatedTokenBuffer
 109                 {
 110                         readonly LocatedToken[] buffer;
 111                         public int pos;
 112
 113                         public LocatedTokenBuffer ()
 114                         {
 115                                 buffer = new LocatedToken[0];
 116                         }
 117
 118                         public LocatedTokenBuffer (LocatedToken[] buffer)
 119                         {
 120                                 this.buffer = buffer ?? new LocatedToken[0];
 121                         }
 122
 123                         public LocatedToken Create (SourceFile file, int row, int column)
 124                         {
 125                                 return Create (null, file, row, column);
 126                         }
 127
 128                         public LocatedToken Create (string value, SourceFile file, int row, int column)
 129                         {
 130                                 //
 131                                 // TODO: I am not very happy about the logic but it's the best
 132                                 // what I could come up with for now.
 133                                 // Ideally we should be using just tiny buffer (256 elements) which
 134                                 // is enough to hold all details for currect stack and recycle elements
 135                                 // poped from the stack but there is a trick needed to recycle
 136                                 // them properly.
 137                                 //
 138                                 LocatedToken entry;
 139                                 if (pos >= buffer.Length) {
 140                                         entry = new LocatedToken ();
 141                                 } else {
 142                                         entry = buffer[pos];
 143                                         if (entry == null) {
 144                                                 entry = new LocatedToken ();
 145                                                 buffer[pos] = entry;
 146                                         }
 147
 148                                         ++pos;
 149                                 }
 150                                 entry.value = value;
 151                                 entry.file = file;
 152                                 entry.row = row;
 153                                 entry.column = column;
 154                                 return entry;
 155                         }
 156
 157                         //
 158                         // Used for token not required by expression evaluator
 159                         //
 160                         [Conditional ("FULL_AST")]
 161                         public void CreateOptional (SourceFile file, int row, int col, ref object token)
 162                         {
 163                                 token = Create (file, row, col);
 164                         }
 165                 }
 166
 167                 public enum PreprocessorDirective
 168                 {
 169                         Invalid = 0,
 170
 171                         Region = 1,
 172                         Endregion = 2,
 173                         If = 3 | RequiresArgument,
 174                         Endif = 4,
 175                         Elif = 5 | RequiresArgument,
 176                         Else = 6,
 177                         Define = 7 | RequiresArgument,
 178                         Undef = 8 | RequiresArgument,
 179                         Error = 9,
 180                         Warning = 10,
 181                         Pragma = 11 | CustomArgumentsParsing,
 182                         Line = 12 | CustomArgumentsParsing,
 183
 184                         CustomArgumentsParsing = 1 << 10,
 185                         RequiresArgument = 1 << 11
 186                 }
 187
 188                 readonly SeekableStreamReader reader;
 189                 readonly CompilationSourceFile source_file;
 190                 readonly CompilerContext context;
 191                 readonly Report Report;
 192
 193
 194                 SourceFile current_source;
 195                 Location hidden_block_start;
 196                 int ref_line = 1;
 197                 int line = 1;
 198                 int col = 0;
 199                 int previous_col;
 200                 int current_token;
 201                 readonly int tab_size;
 202                 bool handle_get_set = false;
 203                 bool handle_remove_add = false;
 204                 bool handle_where;
 205                 bool lambda_arguments_parsing;
 206                 List<Location> escaped_identifiers;
 207                 int parsing_generic_less_than;
 208                 readonly bool doc_processing;
 209                 readonly LocatedTokenBuffer ltb;
 210
 211                 //
 212                 // Used mainly for parser optimizations. Some expressions for instance
 213                 // can appear only in block (including initializer, base initializer)
 214                 // scope only
 215                 //
 216                 public int parsing_block;
 217                 internal bool query_parsing;
 218
 219                 //
 220                 // When parsing type only, useful for ambiguous nullable types
 221                 //
 222                 public int parsing_type;
 223
 224                 //
 225                 // Set when parsing generic declaration (type or method header)
 226                 //
 227                 public bool parsing_generic_declaration;
 228                 public bool parsing_generic_declaration_doc;
 229
 230                 //
 231                 // The value indicates that we have not reach any declaration or
 232                 // namespace yet
 233                 //
 234                 public int parsing_declaration;
 235
 236                 public bool parsing_attribute_section;
 237
 238                 public bool parsing_modifiers;
 239
 240                 //
 241                 // The special characters to inject on streams to run the unit parser
 242                 // in the special expression mode. Using private characters from
 243                 // Plane Sixteen (U+100000 to U+10FFFD)
 244                 //
 245                 // This character is only tested just before the tokenizer is about to report
 246                 // an error;   So on the regular operation mode, this addition will have no
 247                 // impact on the tokenizer's performance.
 248                 //
 249
 250                 public const int EvalStatementParserCharacter = 0x100000;
 251                 public const int EvalCompilationUnitParserCharacter = 0x100001;
 252                 public const int EvalUsingDeclarationsParserCharacter = 0x100002;
 253                 public const int DocumentationXref = 0x100003;
 254
 255                 const int UnicodeLS = 0x2028;
 256                 const int UnicodePS = 0x2029;
 257
 258                 //
 259                 // XML documentation buffer. The save point is used to divide
 260                 // comments on types and comments on members.
 261                 //
 262                 StringBuilder xml_comment_buffer;
 263
 264                 //
 265                 // See comment on XmlCommentState enumeration.
 266                 //
 267                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 268
 269                 //
 270                 // Whether tokens have been seen on this line
 271                 //
 272                 bool tokens_seen = false;
 273
 274                 //
 275                 // Set to true once the GENERATE_COMPLETION token has bee
 276                 // returned.   This helps produce one GENERATE_COMPLETION,
 277                 // as many COMPLETE_COMPLETION as necessary to complete the
 278                 // AST tree and one final EOF.
 279                 //
 280                 bool generated;
 281
 282                 //
 283                 // Whether a token has been seen on the file
 284                 // This is needed because `define' is not allowed to be used
 285                 // after a token has been seen.
 286                 //
 287                 bool any_token_seen;
 288
 289                 //
 290                 // Class variables
 291                 //
 292                 static readonly KeywordEntry<int>[][] keywords;
 293                 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 294                 static readonly HashSet<string> keyword_strings;
 295                 static readonly NumberStyles styles;
 296                 static readonly NumberFormatInfo csharp_format_info;
 297
 298                 // Pragma arguments
 299                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 300                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 301                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 302                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 303                 static readonly char[] line_hidden = "hidden".ToCharArray ();
 304                 static readonly char[] line_default = "default".ToCharArray ();
 305
 306                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 307
 308                 public bool PropertyParsing {
 309                         get { return handle_get_set; }
 310                         set { handle_get_set = value; }
 311                 }
 312
 313                 public bool EventParsing {
 314                         get { return handle_remove_add; }
 315                         set { handle_remove_add = value; }
 316                 }
 317
 318                 public bool ConstraintsParsing {
 319                         get { return handle_where; }
 320                         set { handle_where = value; }
 321                 }
 322
 323                 public XmlCommentState doc_state {
 324                         get { return xml_doc_state; }
 325                         set {
 326                                 if (value == XmlCommentState.Allowed) {
 327                                         check_incorrect_doc_comment ();
 328                                         reset_doc_comment ();
 329                                 }
 330                                 xml_doc_state = value;
 331                         }
 332                 }
 333
 334                 //
 335                 // This is used to trigger completion generation on the parser
 336                 public bool CompleteOnEOF;
 337
 338                 void AddEscapedIdentifier (Location loc)
 339                 {
 340                         if (escaped_identifiers == null)
 341                                 escaped_identifiers = new List<Location> ();
 342
 343                         escaped_identifiers.Add (loc);
 344                 }
 345
 346                 public bool IsEscapedIdentifier (ATypeNameExpression name)
 347                 {
 348                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 349                 }
 350
 351                 //
 352                 // Values for the associated token returned
 353                 //
 354                 internal int putback_char;      // Used by repl only
 355                 object val;
 356
 357                 //
 358                 // Pre-processor
 359                 //
 360                 const int TAKING        = 1;
 361                 const int ELSE_SEEN     = 4;
 362                 const int PARENT_TAKING = 8;
 363                 const int REGION        = 16;
 364
 365                 //
 366                 // pre-processor if stack state:
 367                 //
 368                 Stack<int> ifstack;
 369
 370                 public const int MaxIdentifierLength = 512;
 371                 public const int MaxNumberLength = 512;
 372
 373                 readonly char[] id_builder;
 374                 readonly Dictionary<char[], string>[] identifiers;
 375                 readonly char[] number_builder;
 376                 int number_pos;
 377
 378                 char[] value_builder = new char[64];
 379
 380                 public int Line {
 381                         get {
 382                                 return ref_line;
 383                         }
 384                 }
 385
 386                 //
 387                 // This is used when the tokenizer needs to save
 388                 // the current position as it needs to do some parsing
 389                 // on its own to deamiguate a token in behalf of the
 390                 // parser.
 391                 //
 392                 Stack<Position> position_stack = new Stack<Position> (2);
 393
 394                 class Position {
 395                         public int position;
 396                         public int line;
 397                         public int ref_line;
 398                         public int col;
 399                         public Location hidden;
 400                         public int putback_char;
 401                         public int previous_col;
 402                         public Stack<int> ifstack;
 403                         public int parsing_generic_less_than;
 404                         public int current_token;
 405                         public object val;
 406
 407                         public Position (Tokenizer t)
 408                         {
 409                                 position = t.reader.Position;
 410                                 line = t.line;
 411                                 ref_line = t.ref_line;
 412                                 col = t.col;
 413                                 hidden = t.hidden_block_start;
 414                                 putback_char = t.putback_char;
 415                                 previous_col = t.previous_col;
 416                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 417                                         // There is no simple way to clone Stack<T> all
 418                                         // methods reverse the order
 419                                         var clone = t.ifstack.ToArray ();
 420                                         Array.Reverse (clone);
 421                                         ifstack = new Stack<int> (clone);
 422                                 }
 423                                 parsing_generic_less_than = t.parsing_generic_less_than;
 424                                 current_token = t.current_token;
 425                                 val = t.val;
 426                         }
 427                 }
 428
 429                 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, ParserSession session, Report report)
 430                 {
 431                         this.source_file = file;
 432                         this.context = file.Compiler;
 433                         this.current_source = file.SourceFile;
 434                         this.identifiers = session.Identifiers;
 435                         this.id_builder = session.IDBuilder;
 436                         this.number_builder = session.NumberBuilder;
 437                         this.ltb = new LocatedTokenBuffer (session.LocatedTokens);
 438                         this.Report = report;
 439
 440                         reader = input;
 441
 442                         putback_char = -1;
 443
 444                         xml_comment_buffer = new StringBuilder ();
 445                         doc_processing = context.Settings.DocumentationFile != null;
 446
 447                         tab_size = context.Settings.TabSize;
 448                 }
 449
 450                 public void PushPosition ()
 451                 {
 452                         position_stack.Push (new Position (this));
 453                 }
 454
 455                 public void PopPosition ()
 456                 {
 457                         Position p = position_stack.Pop ();
 458
 459                         reader.Position = p.position;
 460                         ref_line = p.ref_line;
 461                         line = p.line;
 462                         col = p.col;
 463                         hidden_block_start = p.hidden;
 464                         putback_char = p.putback_char;
 465                         previous_col = p.previous_col;
 466                         ifstack = p.ifstack;
 467                         parsing_generic_less_than = p.parsing_generic_less_than;
 468                         current_token = p.current_token;
 469                         val = p.val;
 470                 }
 471
 472                 // Do not reset the position, ignore it.
 473                 public void DiscardPosition ()
 474                 {
 475                         position_stack.Pop ();
 476                 }
 477
 478                 static void AddKeyword (string kw, int token)
 479                 {
 480                         keyword_strings.Add (kw);
 481
 482                         AddKeyword (keywords, kw, token);
 483                 }
 484
 485                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 486                 {
 487                         AddKeyword (keywords_preprocessor, kw, directive);
 488                 }
 489
 490                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 491                 {
 492                         int length = kw.Length;
 493                         if (keywords[length] == null) {
 494                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 495                         }
 496
 497                         int char_index = kw[0] - '_';
 498                         var kwe = keywords[length][char_index];
 499                         if (kwe == null) {
 500                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 501                                 return;
 502                         }
 503
 504                         while (kwe.Next != null) {
 505                                 kwe = kwe.Next;
 506                         }
 507
 508                         kwe.Next = new KeywordEntry<T> (kw, token);
 509                 }
 510
 511                 //
 512                 // Class initializer
 513                 //
 514                 static Tokenizer ()
 515                 {
 516                         keyword_strings = new HashSet<string> ();
 517
 518                         // 11 is the length of the longest keyword for now
 519                         keywords = new KeywordEntry<int>[11][];
 520
 521                         AddKeyword ("__arglist", Token.ARGLIST);
 522                         AddKeyword ("__makeref", Token.MAKEREF);
 523                         AddKeyword ("__reftype", Token.REFTYPE);
 524                         AddKeyword ("__refvalue", Token.REFVALUE);
 525                         AddKeyword ("abstract", Token.ABSTRACT);
 526                         AddKeyword ("as", Token.AS);
 527                         AddKeyword ("add", Token.ADD);
 528                         AddKeyword ("base", Token.BASE);
 529                         AddKeyword ("bool", Token.BOOL);
 530                         AddKeyword ("break", Token.BREAK);
 531                         AddKeyword ("byte", Token.BYTE);
 532                         AddKeyword ("case", Token.CASE);
 533                         AddKeyword ("catch", Token.CATCH);
 534                         AddKeyword ("char", Token.CHAR);
 535                         AddKeyword ("checked", Token.CHECKED);
 536                         AddKeyword ("class", Token.CLASS);
 537                         AddKeyword ("const", Token.CONST);
 538                         AddKeyword ("continue", Token.CONTINUE);
 539                         AddKeyword ("decimal", Token.DECIMAL);
 540                         AddKeyword ("default", Token.DEFAULT);
 541                         AddKeyword ("delegate", Token.DELEGATE);
 542                         AddKeyword ("do", Token.DO);
 543                         AddKeyword ("double", Token.DOUBLE);
 544                         AddKeyword ("else", Token.ELSE);
 545                         AddKeyword ("enum", Token.ENUM);
 546                         AddKeyword ("event", Token.EVENT);
 547                         AddKeyword ("explicit", Token.EXPLICIT);
 548                         AddKeyword ("extern", Token.EXTERN);
 549                         AddKeyword ("false", Token.FALSE);
 550                         AddKeyword ("finally", Token.FINALLY);
 551                         AddKeyword ("fixed", Token.FIXED);
 552                         AddKeyword ("float", Token.FLOAT);
 553                         AddKeyword ("for", Token.FOR);
 554                         AddKeyword ("foreach", Token.FOREACH);
 555                         AddKeyword ("goto", Token.GOTO);
 556                         AddKeyword ("get", Token.GET);
 557                         AddKeyword ("if", Token.IF);
 558                         AddKeyword ("implicit", Token.IMPLICIT);
 559                         AddKeyword ("in", Token.IN);
 560                         AddKeyword ("int", Token.INT);
 561                         AddKeyword ("interface", Token.INTERFACE);
 562                         AddKeyword ("internal", Token.INTERNAL);
 563                         AddKeyword ("is", Token.IS);
 564                         AddKeyword ("lock", Token.LOCK);
 565                         AddKeyword ("long", Token.LONG);
 566                         AddKeyword ("namespace", Token.NAMESPACE);
 567                         AddKeyword ("new", Token.NEW);
 568                         AddKeyword ("null", Token.NULL);
 569                         AddKeyword ("object", Token.OBJECT);
 570                         AddKeyword ("operator", Token.OPERATOR);
 571                         AddKeyword ("out", Token.OUT);
 572                         AddKeyword ("override", Token.OVERRIDE);
 573                         AddKeyword ("params", Token.PARAMS);
 574                         AddKeyword ("private", Token.PRIVATE);
 575                         AddKeyword ("protected", Token.PROTECTED);
 576                         AddKeyword ("public", Token.PUBLIC);
 577                         AddKeyword ("readonly", Token.READONLY);
 578                         AddKeyword ("ref", Token.REF);
 579                         AddKeyword ("remove", Token.REMOVE);
 580                         AddKeyword ("return", Token.RETURN);
 581                         AddKeyword ("sbyte", Token.SBYTE);
 582                         AddKeyword ("sealed", Token.SEALED);
 583                         AddKeyword ("set", Token.SET);
 584                         AddKeyword ("short", Token.SHORT);
 585                         AddKeyword ("sizeof", Token.SIZEOF);
 586                         AddKeyword ("stackalloc", Token.STACKALLOC);
 587                         AddKeyword ("static", Token.STATIC);
 588                         AddKeyword ("string", Token.STRING);
 589                         AddKeyword ("struct", Token.STRUCT);
 590                         AddKeyword ("switch", Token.SWITCH);
 591                         AddKeyword ("this", Token.THIS);
 592                         AddKeyword ("throw", Token.THROW);
 593                         AddKeyword ("true", Token.TRUE);
 594                         AddKeyword ("try", Token.TRY);
 595                         AddKeyword ("typeof", Token.TYPEOF);
 596                         AddKeyword ("uint", Token.UINT);
 597                         AddKeyword ("ulong", Token.ULONG);
 598                         AddKeyword ("unchecked", Token.UNCHECKED);
 599                         AddKeyword ("unsafe", Token.UNSAFE);
 600                         AddKeyword ("ushort", Token.USHORT);
 601                         AddKeyword ("using", Token.USING);
 602                         AddKeyword ("virtual", Token.VIRTUAL);
 603                         AddKeyword ("void", Token.VOID);
 604                         AddKeyword ("volatile", Token.VOLATILE);
 605                         AddKeyword ("while", Token.WHILE);
 606                         AddKeyword ("partial", Token.PARTIAL);
 607                         AddKeyword ("where", Token.WHERE);
 608
 609                         // LINQ keywords
 610                         AddKeyword ("from", Token.FROM);
 611                         AddKeyword ("join", Token.JOIN);
 612                         AddKeyword ("on", Token.ON);
 613                         AddKeyword ("equals", Token.EQUALS);
 614                         AddKeyword ("select", Token.SELECT);
 615                         AddKeyword ("group", Token.GROUP);
 616                         AddKeyword ("by", Token.BY);
 617                         AddKeyword ("let", Token.LET);
 618                         AddKeyword ("orderby", Token.ORDERBY);
 619                         AddKeyword ("ascending", Token.ASCENDING);
 620                         AddKeyword ("descending", Token.DESCENDING);
 621                         AddKeyword ("into", Token.INTO);
 622
 623                         // Contextual async keywords
 624                         AddKeyword ("async", Token.ASYNC);
 625                         AddKeyword ("await", Token.AWAIT);
 626
 627                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 628
 629                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 630                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 631                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 632                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 633                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 634                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 635                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 636                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 637                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 638                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 639                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 640                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 641
 642                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 643                         styles = NumberStyles.Float;
 644                 }
 645
 646                 int GetKeyword (char[] id, int id_len)
 647                 {
 648                         //
 649                         // Keywords are stored in an array of arrays grouped by their
 650                         // length and then by the first character
 651                         //
 652                         if (id_len >= keywords.Length || keywords [id_len] == null)
 653                                 return -1;
 654
 655                         int first_index = id [0] - '_';
 656                         if (first_index > 'z' - '_')
 657                                 return -1;
 658
 659                         var kwe = keywords [id_len] [first_index];
 660                         if (kwe == null)
 661                                 return -1;
 662
 663                         int res;
 664                         do {
 665                                 res = kwe.Token;
 666                                 for (int i = 1; i < id_len; ++i) {
 667                                         if (id [i] != kwe.Value [i]) {
 668                                                 res = 0;
 669                                                 kwe = kwe.Next;
 670                                                 break;
 671                                         }
 672                                 }
 673                         } while (res == 0 && kwe != null);
 674
 675                         if (res == 0)
 676                                 return -1;
 677
 678                         int next_token;
 679                         switch (res) {
 680                         case Token.GET:
 681                         case Token.SET:
 682                                 if (!handle_get_set)
 683                                         res = -1;
 684                                 break;
 685                         case Token.REMOVE:
 686                         case Token.ADD:
 687                                 if (!handle_remove_add)
 688                                         res = -1;
 689                                 break;
 690                         case Token.EXTERN:
 691                                 if (parsing_declaration == 0)
 692                                         res = Token.EXTERN_ALIAS;
 693                                 break;
 694                         case Token.DEFAULT:
 695                                 if (peek_token () == Token.COLON) {
 696                                         token ();
 697                                         res = Token.DEFAULT_COLON;
 698                                 }
 699                                 break;
 700                         case Token.WHERE:
 701                                 if (!(handle_where && current_token != Token.COLON) && !query_parsing)
 702                                         res = -1;
 703                                 break;
 704                         case Token.FROM:
 705                                 //
 706                                 // A query expression is any expression that starts with `from identifier'
 707                                 // followed by any token except ; , =
 708                                 //
 709                                 if (!query_parsing) {
 710                                         if (lambda_arguments_parsing || parsing_block == 0) {
 711                                                 res = -1;
 712                                                 break;
 713                                         }
 714
 715                                         PushPosition ();
 716                                         // HACK: to disable generics micro-parser, because PushPosition does not
 717                                         // store identifiers array
 718                                         parsing_generic_less_than = 1;
 719                                         switch (xtoken ()) {
 720                                         case Token.IDENTIFIER:
 721                                         case Token.INT:
 722                                         case Token.BOOL:
 723                                         case Token.BYTE:
 724                                         case Token.CHAR:
 725                                         case Token.DECIMAL:
 726                                         case Token.DOUBLE:
 727                                         case Token.FLOAT:
 728                                         case Token.LONG:
 729                                         case Token.OBJECT:
 730                                         case Token.STRING:
 731                                         case Token.UINT:
 732                                         case Token.ULONG:
 733                                                 next_token = xtoken ();
 734                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS || next_token == Token.ASSIGN)
 735                                                         goto default;
 736
 737                                                 res = Token.FROM_FIRST;
 738                                                 query_parsing = true;
 739                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 740                                                         Report.FeatureIsNotAvailable (context, Location, "query expressions");
 741                                                 break;
 742                                         case Token.VOID:
 743                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 744                                                 break;
 745                                         default:
 746                                                 PopPosition ();
 747                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 748                                                 // which has been overwritten before we grabbed the identifier
 749                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 750                                                 return -1;
 751                                         }
 752                                         PopPosition ();
 753                                 }
 754                                 break;
 755                         case Token.JOIN:
 756                         case Token.ON:
 757                         case Token.EQUALS:
 758                         case Token.SELECT:
 759                         case Token.GROUP:
 760                         case Token.BY:
 761                         case Token.LET:
 762                         case Token.ORDERBY:
 763                         case Token.ASCENDING:
 764                         case Token.DESCENDING:
 765                         case Token.INTO:
 766                                 if (!query_parsing)
 767                                         res = -1;
 768                                 break;
 769
 770                         case Token.USING:
 771                         case Token.NAMESPACE:
 772                                 // TODO: some explanation needed
 773                                 check_incorrect_doc_comment ();
 774                                 parsing_modifiers = false;
 775                                 break;
 776
 777                         case Token.PARTIAL:
 778                                 if (parsing_block > 0) {
 779                                         res = -1;
 780                                         break;
 781                                 }
 782
 783                                 // Save current position and parse next token.
 784                                 PushPosition ();
 785
 786                                 next_token = token ();
 787                                 bool ok = (next_token == Token.CLASS) ||
 788                                         (next_token == Token.STRUCT) ||
 789                                         (next_token == Token.INTERFACE) ||
 790                                         (next_token == Token.VOID);
 791
 792                                 PopPosition ();
 793
 794                                 if (ok) {
 795                                         if (next_token == Token.VOID) {
 796                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 797                                                         Report.FeatureIsNotAvailable (context, Location, "partial methods");
 798                                         } else if (context.Settings.Version == LanguageVersion.ISO_1)
 799                                                 Report.FeatureIsNotAvailable (context, Location, "partial types");
 800
 801                                         return res;
 802                                 }
 803
 804                                 if (next_token < Token.LAST_KEYWORD) {
 805                                         Report.Error (267, Location,
 806                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 807                                         return token ();
 808                                 }
 809
 810                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 811                                 // which has been overwritten before we grabbed the identifier
 812                                 id_builder[0] = 'p';
 813                                 id_builder[1] = 'a';
 814                                 id_builder[2] = 'r';
 815                                 id_builder[3] = 't';
 816                                 id_builder[4] = 'i';
 817                                 id_builder[5] = 'a';
 818                                 id_builder[6] = 'l';
 819                                 res = -1;
 820                                 break;
 821
 822                         case Token.ASYNC:
 823                                 if (parsing_modifiers) {
 824                                         //
 825                                         // Skip attributes section or constructor called async
 826                                         //
 827                                         if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) {
 828                                                 res = -1;
 829                                         } else {
 830                                                 // async is keyword
 831                                         }
 832                                 } else if (parsing_block > 0) {
 833                                         switch (peek_token ()) {
 834                                         case Token.DELEGATE:
 835                                         case Token.OPEN_PARENS_LAMBDA:
 836                                                 // async is keyword
 837                                                 break;
 838                                         case Token.IDENTIFIER:
 839                                                 PushPosition ();
 840                                                 xtoken ();
 841                                                 if (xtoken () != Token.ARROW) {
 842                                                         PopPosition ();
 843                                                         goto default;
 844                                                 }
 845
 846                                                 PopPosition ();
 847                                                 break;
 848                                         default:
 849                                                 // peek_token could overwrite id_buffer
 850                                                 id_builder [0] = 'a'; id_builder [1] = 's'; id_builder [2] = 'y'; id_builder [3] = 'n'; id_builder [4] = 'c';
 851                                                 res = -1;
 852                                                 break;
 853                                         }
 854                                 } else {
 855                                         res = -1;
 856                                 }
 857
 858                                 if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) {
 859                                         Report.FeatureIsNotAvailable (context, Location, "asynchronous functions");
 860                                 }
 861
 862                                 break;
 863
 864                         case Token.AWAIT:
 865                                 if (parsing_block == 0)
 866                                         res = -1;
 867
 868                                 break;
 869                         }
 870
 871
 872                         return res;
 873                 }
 874
 875                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 876                 {
 877                         //
 878                         // Keywords are stored in an array of arrays grouped by their
 879                         // length and then by the first character
 880                         //
 881                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 882                                 return PreprocessorDirective.Invalid;
 883
 884                         int first_index = id[0] - '_';
 885                         if (first_index > 'z' - '_')
 886                                 return PreprocessorDirective.Invalid;
 887
 888                         var kwe = keywords_preprocessor[id_len][first_index];
 889                         if (kwe == null)
 890                                 return PreprocessorDirective.Invalid;
 891
 892                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 893                         do {
 894                                 res = kwe.Token;
 895                                 for (int i = 1; i < id_len; ++i) {
 896                                         if (id[i] != kwe.Value[i]) {
 897                                                 res = 0;
 898                                                 kwe = kwe.Next;
 899                                                 break;
 900                                         }
 901                                 }
 902                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 903
 904                         return res;
 905                 }
 906
 907                 public Location Location {
 908                         get {
 909                                 return new Location (current_source, ref_line, col);
 910                         }
 911                 }
 912
 913                 static bool is_identifier_start_character (int c)
 914                 {
 915                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 916                 }
 917
 918                 static bool is_identifier_part_character (char c)
 919                 {
 920                         if (c >= 'a' && c <= 'z')
 921                                 return true;
 922
 923                         if (c >= 'A' && c <= 'Z')
 924                                 return true;
 925
 926                         if (c == '_' || (c >= '0' && c <= '9'))
 927                                 return true;
 928
 929                         if (c < 0x80)
 930                                 return false;
 931
 932                         return is_identifier_part_character_slow_part (c);
 933                 }
 934
 935                 static bool is_identifier_part_character_slow_part (char c)
 936                 {
 937                         if (Char.IsLetter (c))
 938                                 return true;
 939
 940                         switch (Char.GetUnicodeCategory (c)) {
 941                                 case UnicodeCategory.ConnectorPunctuation:
 942
 943                                 // combining-character: A Unicode character of classes Mn or Mc
 944                                 case UnicodeCategory.NonSpacingMark:
 945                                 case UnicodeCategory.SpacingCombiningMark:
 946
 947                                 // decimal-digit-character: A Unicode character of the class Nd
 948                                 case UnicodeCategory.DecimalDigitNumber:
 949                                 return true;
 950                         }
 951
 952                         return false;
 953                 }
 954
 955                 public static bool IsKeyword (string s)
 956                 {
 957                         return keyword_strings.Contains (s);
 958                 }
 959
 960                 //
 961                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 962                 //
 963                 int TokenizeOpenParens ()
 964                 {
 965                         int ptoken;
 966                         current_token = -1;
 967
 968                         int bracket_level = 0;
 969                         bool is_type = false;
 970                         bool can_be_type = false;
 971
 972                         while (true) {
 973                                 ptoken = current_token;
 974                                 token ();
 975
 976                                 switch (current_token) {
 977                                 case Token.CLOSE_PARENS:
 978                                         token ();
 979
 980                                         //
 981                                         // Expression inside parens is lambda, (int i) =>
 982                                         //
 983                                         if (current_token == Token.ARROW)
 984                                                 return Token.OPEN_PARENS_LAMBDA;
 985
 986                                         //
 987                                         // Expression inside parens is single type, (int[])
 988                                         //
 989                                         if (is_type) {
 990                                                 if (current_token == Token.SEMICOLON)
 991                                                         return Token.OPEN_PARENS;
 992
 993                                                 return Token.OPEN_PARENS_CAST;
 994                                         }
 995
 996                                         //
 997                                         // Expression is possible cast, look at next token, (T)null
 998                                         //
 999                                         if (can_be_type) {
1000                                                 switch (current_token) {
1001                                                 case Token.OPEN_PARENS:
1002                                                 case Token.BANG:
1003                                                 case Token.TILDE:
1004                                                 case Token.IDENTIFIER:
1005                                                 case Token.LITERAL:
1006                                                 case Token.BASE:
1007                                                 case Token.CHECKED:
1008                                                 case Token.DELEGATE:
1009                                                 case Token.FALSE:
1010                                                 case Token.FIXED:
1011                                                 case Token.NEW:
1012                                                 case Token.NULL:
1013                                                 case Token.SIZEOF:
1014                                                 case Token.THIS:
1015                                                 case Token.THROW:
1016                                                 case Token.TRUE:
1017                                                 case Token.TYPEOF:
1018                                                 case Token.UNCHECKED:
1019                                                 case Token.UNSAFE:
1020                                                 case Token.DEFAULT:
1021                                                 case Token.AWAIT:
1022
1023                                                 //
1024                                                 // These can be part of a member access
1025                                                 //
1026                                                 case Token.INT:
1027                                                 case Token.UINT:
1028                                                 case Token.SHORT:
1029                                                 case Token.USHORT:
1030                                                 case Token.LONG:
1031                                                 case Token.ULONG:
1032                                                 case Token.DOUBLE:
1033                                                 case Token.FLOAT:
1034                                                 case Token.CHAR:
1035                                                 case Token.BYTE:
1036                                                 case Token.DECIMAL:
1037                                                 case Token.BOOL:
1038                                                 case Token.STRING:
1039                                                         return Token.OPEN_PARENS_CAST;
1040                                                 }
1041                                         }
1042                                         return Token.OPEN_PARENS;
1043
1044                                 case Token.DOT:
1045                                 case Token.DOUBLE_COLON:
1046                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
1047                                                 goto default;
1048
1049                                         continue;
1050
1051                                 case Token.IDENTIFIER:
1052                                 case Token.AWAIT:
1053                                         switch (ptoken) {
1054                                         case Token.DOT:
1055                                                 if (bracket_level == 0) {
1056                                                         is_type = false;
1057                                                         can_be_type = true;
1058                                                 }
1059
1060                                                 continue;
1061                                         case Token.OP_GENERICS_LT:
1062                                         case Token.COMMA:
1063                                         case Token.DOUBLE_COLON:
1064                                         case -1:
1065                                                 if (bracket_level == 0)
1066                                                         can_be_type = true;
1067                                                 continue;
1068                                         default:
1069                                                 can_be_type = is_type = false;
1070                                                 continue;
1071                                         }
1072
1073                                 case Token.OBJECT:
1074                                 case Token.STRING:
1075                                 case Token.BOOL:
1076                                 case Token.DECIMAL:
1077                                 case Token.FLOAT:
1078                                 case Token.DOUBLE:
1079                                 case Token.SBYTE:
1080                                 case Token.BYTE:
1081                                 case Token.SHORT:
1082                                 case Token.USHORT:
1083                                 case Token.INT:
1084                                 case Token.UINT:
1085                                 case Token.LONG:
1086                                 case Token.ULONG:
1087                                 case Token.CHAR:
1088                                 case Token.VOID:
1089                                         if (bracket_level == 0)
1090                                                 is_type = true;
1091                                         continue;
1092
1093                                 case Token.COMMA:
1094                                         if (bracket_level == 0) {
1095                                                 bracket_level = 100;
1096                                                 can_be_type = is_type = false;
1097                                         }
1098                                         continue;
1099
1100                                 case Token.OP_GENERICS_LT:
1101                                 case Token.OPEN_BRACKET:
1102                                         if (bracket_level++ == 0)
1103                                                 is_type = true;
1104                                         continue;
1105
1106                                 case Token.OP_GENERICS_GT:
1107                                 case Token.CLOSE_BRACKET:
1108                                         --bracket_level;
1109                                         continue;
1110
1111                                 case Token.INTERR_NULLABLE:
1112                                 case Token.STAR:
1113                                         if (bracket_level == 0)
1114                                                 is_type = true;
1115                                         continue;
1116
1117                                 case Token.REF:
1118                                 case Token.OUT:
1119                                         can_be_type = is_type = false;
1120                                         continue;
1121
1122                                 default:
1123                                         return Token.OPEN_PARENS;
1124                                 }
1125                         }
1126                 }
1127
1128                 public static bool IsValidIdentifier (string s)
1129                 {
1130                         if (s == null || s.Length == 0)
1131                                 return false;
1132
1133                         if (!is_identifier_start_character (s [0]))
1134                                 return false;
1135
1136                         for (int i = 1; i < s.Length; i ++)
1137                                 if (! is_identifier_part_character (s [i]))
1138                                         return false;
1139
1140                         return true;
1141                 }
1142
1143                 bool parse_less_than (ref int genericDimension)
1144                 {
1145                 start:
1146                         int the_token = token ();
1147                         if (the_token == Token.OPEN_BRACKET) {
1148                                 while (true) {
1149                                         the_token = token ();
1150                                         if (the_token == Token.EOF)
1151                                                 return true;
1152
1153                                         if (the_token == Token.CLOSE_BRACKET)
1154                                                 break;
1155                                 }
1156                                 the_token = token ();
1157                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1158                                 the_token = token ();
1159                         }
1160                         switch (the_token) {
1161                         case Token.IDENTIFIER:
1162                         case Token.OBJECT:
1163                         case Token.STRING:
1164                         case Token.BOOL:
1165                         case Token.DECIMAL:
1166                         case Token.FLOAT:
1167                         case Token.DOUBLE:
1168                         case Token.SBYTE:
1169                         case Token.BYTE:
1170                         case Token.SHORT:
1171                         case Token.USHORT:
1172                         case Token.INT:
1173                         case Token.UINT:
1174                         case Token.LONG:
1175                         case Token.ULONG:
1176                         case Token.CHAR:
1177                         case Token.VOID:
1178                                 break;
1179                         case Token.OP_GENERICS_GT:
1180                                 genericDimension = 1;
1181                                 return true;
1182                         case Token.IN:
1183                         case Token.OUT:
1184                                 return true;
1185                         case Token.COMMA:
1186                                 do {
1187                                         ++genericDimension;
1188                                         the_token = token ();
1189                                 } while (the_token == Token.COMMA);
1190
1191                                 if (the_token == Token.OP_GENERICS_GT) {
1192                                         ++genericDimension;
1193                                         return true;
1194                                 }
1195
1196                                 return false;
1197                         default:
1198                                 return false;
1199                         }
1200                 again:
1201                         the_token = token ();
1202
1203                         if (the_token == Token.OP_GENERICS_GT)
1204                                 return true;
1205                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1206                                 goto start;
1207                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1208                                 goto again;
1209                         else if (the_token == Token.OP_GENERICS_LT) {
1210                                 if (!parse_less_than (ref genericDimension))
1211                                         return false;
1212                                 goto again;
1213                         } else if (the_token == Token.OPEN_BRACKET) {
1214                         rank_specifiers:
1215                                 the_token = token ();
1216                                 if (the_token == Token.CLOSE_BRACKET)
1217                                         goto again;
1218                                 else if (the_token == Token.COMMA)
1219                                         goto rank_specifiers;
1220                                 return false;
1221                         }
1222
1223                         return false;
1224                 }
1225
1226                 public int peek_token ()
1227                 {
1228                         int the_token;
1229
1230                         PushPosition ();
1231                         the_token = token ();
1232                         PopPosition ();
1233
1234                         return the_token;
1235                 }
1236
1237                 //
1238                 // Tonizes `?' using custom disambiguous rules to return one
1239                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1240                 //
1241                 // Tricky expression looks like:
1242                 //
1243                 // Foo ? a = x ? b : c;
1244                 //
1245                 int TokenizePossibleNullableType ()
1246                 {
1247                         if (parsing_block == 0 || parsing_type > 0)
1248                                 return Token.INTERR_NULLABLE;
1249
1250                         int d = peek_char ();
1251                         if (d == '?') {
1252                                 get_char ();
1253                                 return Token.OP_COALESCING;
1254                         }
1255
1256                         if (d == '.') {
1257                                 return Token.INTERR_OPERATOR;
1258                         }
1259
1260                         if (d != ' ') {
1261                                 if (d == ',' || d == ';' || d == '>')
1262                                         return Token.INTERR_NULLABLE;
1263                                 if (d == '*' || (d >= '0' && d <= '9'))
1264                                         return Token.INTERR;
1265                         }
1266
1267                         PushPosition ();
1268                         current_token = Token.NONE;
1269                         int next_token;
1270                         int parens = 0;
1271                         int generics = 0;
1272                         int brackets = 0;
1273
1274                         var nt = xtoken ();
1275                         switch (nt) {
1276                         case Token.DOT:
1277                         case Token.OPEN_BRACKET_EXPR:
1278                                 next_token = Token.INTERR_OPERATOR;
1279                                 break;
1280                         case Token.LITERAL:
1281                         case Token.TRUE:
1282                         case Token.FALSE:
1283                         case Token.NULL:
1284                         case Token.THIS:
1285                         case Token.NEW:
1286                                 next_token = Token.INTERR;
1287                                 break;
1288
1289                         case Token.SEMICOLON:
1290                         case Token.COMMA:
1291                         case Token.CLOSE_PARENS:
1292                         case Token.OPEN_BRACKET:
1293                         case Token.OP_GENERICS_GT:
1294                         case Token.INTERR:
1295                         case Token.OP_COALESCING:
1296                         case Token.COLON:
1297                                 next_token = Token.INTERR_NULLABLE;
1298                                 break;
1299
1300                         case Token.OPEN_PARENS:
1301                         case Token.OPEN_PARENS_CAST:
1302                         case Token.OPEN_PARENS_LAMBDA:
1303                                 next_token = -1;
1304                                 ++parens;
1305                                 break;
1306
1307                         case Token.OP_GENERICS_LT:
1308                         case Token.OP_GENERICS_LT_DECL:
1309                         case Token.GENERIC_DIMENSION:
1310                                 next_token = -1;
1311                                 ++generics;
1312                                 break;
1313
1314                         default:
1315                                 next_token = -1;
1316                                 break;
1317                         }
1318
1319                         if (next_token == -1) {
1320                                 switch (xtoken ()) {
1321                                 case Token.COMMA:
1322                                 case Token.SEMICOLON:
1323                                 case Token.OPEN_BRACE:
1324                                 case Token.IN:
1325                                         next_token = Token.INTERR_NULLABLE;
1326                                         break;
1327
1328                                 case Token.COLON:
1329                                         next_token = Token.INTERR;
1330                                         break;
1331
1332                                 case Token.OPEN_PARENS:
1333                                 case Token.OPEN_PARENS_CAST:
1334                                 case Token.OPEN_PARENS_LAMBDA:
1335                                         ++parens;
1336                                         goto default;
1337
1338                                 case Token.OPEN_BRACKET:
1339                                 case Token.OPEN_BRACKET_EXPR:
1340                                         ++brackets;
1341                                         goto default;
1342
1343                                 case Token.CLOSE_PARENS:
1344                                         --parens;
1345                                         goto default;
1346
1347                                 case Token.OP_GENERICS_LT:
1348                                 case Token.OP_GENERICS_LT_DECL:
1349                                 case Token.GENERIC_DIMENSION:
1350                                         ++generics;
1351                                         goto default;
1352
1353                                 default:
1354                                         int ntoken;
1355                                         int interrs = 1;
1356                                         int colons = 0;
1357                                         int braces = 0;
1358                                         //
1359                                         // All shorcuts failed, do it hard way
1360                                         //
1361                                         while ((ntoken = xtoken ()) != Token.EOF) {
1362                                                 switch (ntoken) {
1363                                                 case Token.OPEN_BRACE:
1364                                                         ++braces;
1365                                                         continue;
1366                                                 case Token.OPEN_PARENS:
1367                                                 case Token.OPEN_PARENS_CAST:
1368                                                 case Token.OPEN_PARENS_LAMBDA:
1369                                                         ++parens;
1370                                                         continue;
1371                                                 case Token.CLOSE_BRACE:
1372                                                         --braces;
1373                                                         continue;
1374                                                 case Token.OP_GENERICS_LT:
1375                                                 case Token.OP_GENERICS_LT_DECL:
1376                                                 case Token.GENERIC_DIMENSION:
1377                                                         ++generics;
1378                                                         continue;
1379                                                 case Token.OPEN_BRACKET:
1380                                                 case Token.OPEN_BRACKET_EXPR:
1381                                                         ++brackets;
1382                                                         continue;
1383                                                 case Token.CLOSE_BRACKET:
1384                                                         --brackets;
1385                                                         continue;
1386                                                 case Token.CLOSE_PARENS:
1387                                                         if (parens > 0) {
1388                                                                 --parens;
1389                                                                 continue;
1390                                                         }
1391
1392                                                         PopPosition ();
1393                                                         return Token.INTERR_NULLABLE;
1394
1395                                                 case Token.OP_GENERICS_GT:
1396                                                         if (generics > 0) {
1397                                                                 --generics;
1398                                                                 continue;
1399                                                         }
1400
1401                                                         PopPosition ();
1402                                                         return Token.INTERR_NULLABLE;
1403                                                 }
1404
1405                                                 if (braces != 0)
1406                                                         continue;
1407
1408                                                 if (ntoken == Token.SEMICOLON)
1409                                                         break;
1410
1411                                                 if (parens != 0)
1412                                                         continue;
1413
1414                                                 if (ntoken == Token.COMMA) {
1415                                                         if (generics != 0 || brackets != 0)
1416                                                                 continue;
1417
1418                                                         PopPosition ();
1419                                                         return Token.INTERR_NULLABLE;
1420                                                 }
1421
1422                                                 if (ntoken == Token.COLON) {
1423                                                         if (++colons == interrs)
1424                                                                 break;
1425                                                         continue;
1426                                                 }
1427
1428                                                 if (ntoken == Token.INTERR) {
1429                                                         ++interrs;
1430                                                         continue;
1431                                                 }
1432                                         }
1433
1434                                         next_token = colons != interrs && braces == 0 ? Token.INTERR_NULLABLE : Token.INTERR;
1435                                         break;
1436                                 }
1437                         }
1438
1439                         PopPosition ();
1440                         return next_token;
1441                 }
1442
1443                 bool decimal_digits (int c)
1444                 {
1445                         int d;
1446                         bool seen_digits = false;
1447
1448                         if (c != -1){
1449                                 if (number_pos == MaxNumberLength)
1450                                         Error_NumericConstantTooLong ();
1451                                 number_builder [number_pos++] = (char) c;
1452                         }
1453
1454                         //
1455                         // We use peek_char2, because decimal_digits needs to do a
1456                         // 2-character look-ahead (5.ToString for example).
1457                         //
1458                         while ((d = peek_char2 ()) != -1){
1459                                 if (d >= '0' && d <= '9'){
1460                                         if (number_pos == MaxNumberLength)
1461                                                 Error_NumericConstantTooLong ();
1462                                         number_builder [number_pos++] = (char) d;
1463                                         get_char ();
1464                                         seen_digits = true;
1465                                 } else
1466                                         break;
1467                         }
1468
1469                         return seen_digits;
1470                 }
1471
1472                 static bool is_hex (int e)
1473                 {
1474                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1475                 }
1476
1477                 static TypeCode real_type_suffix (int c)
1478                 {
1479                         switch (c){
1480                         case 'F': case 'f':
1481                                 return TypeCode.Single;
1482                         case 'D': case 'd':
1483                                 return TypeCode.Double;
1484                         case 'M': case 'm':
1485                                 return TypeCode.Decimal;
1486                         default:
1487                                 return TypeCode.Empty;
1488                         }
1489                 }
1490
1491                 ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc)
1492                 {
1493                         bool is_unsigned = false;
1494                         bool is_long = false;
1495
1496                         if (c != -1){
1497                                 bool scanning = true;
1498                                 do {
1499                                         switch (c){
1500                                         case 'U': case 'u':
1501                                                 if (is_unsigned)
1502                                                         scanning = false;
1503                                                 is_unsigned = true;
1504                                                 get_char ();
1505                                                 break;
1506
1507                                         case 'l':
1508                                                 if (!is_unsigned){
1509                                                         //
1510                                                         // if we have not seen anything in between
1511                                                         // report this error
1512                                                         //
1513                                                         Report.Warning (78, 4, Location, "The `l' suffix is easily confused with the digit `1' (use `L' for clarity)");
1514                                                 }
1515
1516                                                 goto case 'L';
1517
1518                                         case 'L':
1519                                                 if (is_long)
1520                                                         scanning = false;
1521                                                 is_long = true;
1522                                                 get_char ();
1523                                                 break;
1524
1525                                         default:
1526                                                 scanning = false;
1527                                                 break;
1528                                         }
1529                                         c = peek_char ();
1530                                 } while (scanning);
1531                         }
1532
1533                         if (is_long && is_unsigned){
1534                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1535                         }
1536
1537                         if (is_unsigned){
1538                                 // uint if possible, or ulong else.
1539
1540                                 if ((ul & 0xffffffff00000000) == 0)
1541                                         return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc);
1542                                 else
1543                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1544                         } else if (is_long){
1545                                 // long if possible, ulong otherwise
1546                                 if ((ul & 0x8000000000000000) != 0)
1547                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1548                                 else
1549                                         return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1550                         } else {
1551                                 // int, uint, long or ulong in that order
1552                                 if ((ul & 0xffffffff00000000) == 0){
1553                                         uint ui = (uint) ul;
1554
1555                                         if ((ui & 0x80000000) != 0)
1556                                                 return new UIntLiteral (context.BuiltinTypes, ui, loc);
1557                                         else
1558                                                 return new IntLiteral (context.BuiltinTypes, (int) ui, loc);
1559                                 } else {
1560                                         if ((ul & 0x8000000000000000) != 0)
1561                                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1562                                         else
1563                                                 return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1564                                 }
1565                         }
1566                 }
1567
1568                 //
1569                 // given `c' as the next char in the input decide whether
1570                 // we need to convert to a special type, and then choose
1571                 // the best representation for the integer
1572                 //
1573                 ILiteralConstant adjust_int (int c, Location loc)
1574                 {
1575                         try {
1576                                 if (number_pos > 9){
1577                                         ulong ul = (uint) (number_builder [0] - '0');
1578
1579                                         for (int i = 1; i < number_pos; i++){
1580                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1581                                         }
1582
1583                                         return integer_type_suffix (ul, c, loc);
1584                                 } else {
1585                                         uint ui = (uint) (number_builder [0] - '0');
1586
1587                                         for (int i = 1; i < number_pos; i++){
1588                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1589                                         }
1590
1591                                         return integer_type_suffix (ui, c, loc);
1592                                 }
1593                         } catch (OverflowException) {
1594                                 Error_NumericConstantTooLong ();
1595                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1596                         }
1597                         catch (FormatException) {
1598                                 Report.Error (1013, Location, "Invalid number");
1599                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1600                         }
1601                 }
1602
1603                 ILiteralConstant adjust_real (TypeCode t, Location loc)
1604                 {
1605                         string s = new string (number_builder, 0, number_pos);
1606                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1607
1608                         switch (t){
1609                         case TypeCode.Decimal:
1610                                 try {
1611                                         return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc);
1612                                 } catch (OverflowException) {
1613                                         Report.Error (594, Location, error_details, "decimal");
1614                                         return new DecimalLiteral (context.BuiltinTypes, 0, loc);
1615                                 }
1616                         case TypeCode.Single:
1617                                 try {
1618                                         return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc);
1619                                 } catch (OverflowException) {
1620                                         Report.Error (594, Location, error_details, "float");
1621                                         return new FloatLiteral (context.BuiltinTypes, 0, loc);
1622                                 }
1623                         default:
1624                                 try {
1625                                         return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc);
1626                                 } catch (OverflowException) {
1627                                         Report.Error (594, loc, error_details, "double");
1628                                         return new DoubleLiteral (context.BuiltinTypes, 0, loc);
1629                                 }
1630                         }
1631                 }
1632
1633                 ILiteralConstant handle_hex (Location loc)
1634                 {
1635                         int d;
1636                         ulong ul;
1637
1638                         get_char ();
1639                         while ((d = peek_char ()) != -1){
1640                                 if (is_hex (d)){
1641                                         number_builder [number_pos++] = (char) d;
1642                                         get_char ();
1643                                 } else
1644                                         break;
1645                         }
1646
1647                         string s = new String (number_builder, 0, number_pos);
1648
1649                         try {
1650                                 if (number_pos <= 8)
1651                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1652                                 else
1653                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1654
1655                                 return integer_type_suffix (ul, peek_char (), loc);
1656                         } catch (OverflowException){
1657                                 Error_NumericConstantTooLong ();
1658                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1659                         }
1660                         catch (FormatException) {
1661                                 Report.Error (1013, Location, "Invalid number");
1662                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1663                         }
1664                 }
1665
1666                 //
1667                 // Invoked if we know we have .digits or digits
1668                 //
1669                 int is_number (int c, bool dotLead)
1670                 {
1671                         ILiteralConstant res;
1672
1673 #if FULL_AST
1674                         int read_start = reader.Position - 1;
1675                         if (dotLead) {
1676                                 //
1677                                 // Caller did peek_char
1678                                 //
1679                                 --read_start;
1680                         }
1681 #endif
1682                         number_pos = 0;
1683                         var loc = Location;
1684
1685                         if (!dotLead){
1686                                 if (c == '0'){
1687                                         int peek = peek_char ();
1688
1689                                         if (peek == 'x' || peek == 'X') {
1690                                                 val = res = handle_hex (loc);
1691 #if FULL_AST
1692                                                 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1693 #endif
1694
1695                                                 return Token.LITERAL;
1696                                         }
1697                                 }
1698                                 decimal_digits (c);
1699                                 c = peek_char ();
1700                         }
1701
1702                         //
1703                         // We need to handle the case of
1704                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1705                         //
1706                         bool is_real = false;
1707                         if (c == '.'){
1708                                 if (!dotLead)
1709                                         get_char ();
1710
1711                                 if (decimal_digits ('.')){
1712                                         is_real = true;
1713                                         c = peek_char ();
1714                                 } else {
1715                                         putback ('.');
1716                                         number_pos--;
1717                                         val = res = adjust_int (-1, loc);
1718
1719 #if FULL_AST
1720                                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1721 #endif
1722                                         return Token.LITERAL;
1723                                 }
1724                         }
1725
1726                         if (c == 'e' || c == 'E'){
1727                                 is_real = true;
1728                                 get_char ();
1729                                 if (number_pos == MaxNumberLength)
1730                                         Error_NumericConstantTooLong ();
1731                                 number_builder [number_pos++] = (char) c;
1732                                 c = get_char ();
1733
1734                                 if (c == '+'){
1735                                         if (number_pos == MaxNumberLength)
1736                                                 Error_NumericConstantTooLong ();
1737                                         number_builder [number_pos++] = '+';
1738                                         c = -1;
1739                                 } else if (c == '-') {
1740                                         if (number_pos == MaxNumberLength)
1741                                                 Error_NumericConstantTooLong ();
1742                                         number_builder [number_pos++] = '-';
1743                                         c = -1;
1744                                 } else {
1745                                         if (number_pos == MaxNumberLength)
1746                                                 Error_NumericConstantTooLong ();
1747                                         number_builder [number_pos++] = '+';
1748                                 }
1749
1750                                 decimal_digits (c);
1751                                 c = peek_char ();
1752                         }
1753
1754                         var type = real_type_suffix (c);
1755                         if (type == TypeCode.Empty && !is_real) {
1756                                 res = adjust_int (c, loc);
1757                         } else {
1758                                 is_real = true;
1759
1760                                 if (type != TypeCode.Empty) {
1761                                         get_char ();
1762                                 }
1763
1764                                 res = adjust_real (type, loc);
1765                         }
1766
1767                         val = res;
1768
1769 #if FULL_AST
1770                         var chars = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty && c > 0 ? 1 : 0));
1771                         if (chars[chars.Length - 1] == '\r')
1772                                 Array.Resize (ref chars, chars.Length - 1);
1773                         res.ParsedValue = chars;
1774 #endif
1775
1776                         return Token.LITERAL;
1777                 }
1778
1779                 //
1780                 // Accepts exactly count (4 or 8) hex, no more no less
1781                 //
1782                 int getHex (int count, out int surrogate, out bool error)
1783                 {
1784                         int i;
1785                         int total = 0;
1786                         int c;
1787                         int top = count != -1 ? count : 4;
1788
1789                         get_char ();
1790                         error = false;
1791                         surrogate = 0;
1792                         for (i = 0; i < top; i++){
1793                                 c = get_char ();
1794
1795                                 if (c >= '0' && c <= '9')
1796                                         c = (int) c - (int) '0';
1797                                 else if (c >= 'A' && c <= 'F')
1798                                         c = (int) c - (int) 'A' + 10;
1799                                 else if (c >= 'a' && c <= 'f')
1800                                         c = (int) c - (int) 'a' + 10;
1801                                 else {
1802                                         error = true;
1803                                         return 0;
1804                                 }
1805
1806                                 total = (total * 16) + c;
1807                                 if (count == -1){
1808                                         int p = peek_char ();
1809                                         if (p == -1)
1810                                                 break;
1811                                         if (!is_hex ((char)p))
1812                                                 break;
1813                                 }
1814                         }
1815
1816                         if (top == 8) {
1817                                 if (total > 0x0010FFFF) {
1818                                         error = true;
1819                                         return 0;
1820                                 }
1821
1822                                 if (total >= 0x00010000) {
1823                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1824                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1825                                 }
1826                         }
1827
1828                         return total;
1829                 }
1830
1831                 int escape (int c, out int surrogate)
1832                 {
1833                         bool error;
1834                         int d;
1835                         int v;
1836
1837                         d = peek_char ();
1838                         if (c != '\\') {
1839                                 surrogate = 0;
1840                                 return c;
1841                         }
1842
1843                         switch (d){
1844                         case 'a':
1845                                 v = '\a'; break;
1846                         case 'b':
1847                                 v = '\b'; break;
1848                         case 'n':
1849                                 v = '\n'; break;
1850                         case 't':
1851                                 v = '\t'; break;
1852                         case 'v':
1853                                 v = '\v'; break;
1854                         case 'r':
1855                                 v = '\r'; break;
1856                         case '\\':
1857                                 v = '\\'; break;
1858                         case 'f':
1859                                 v = '\f'; break;
1860                         case '0':
1861                                 v = 0; break;
1862                         case '"':
1863                                 v = '"'; break;
1864                         case '\'':
1865                                 v = '\''; break;
1866                         case 'x':
1867                                 v = getHex (-1, out surrogate, out error);
1868                                 if (error)
1869                                         goto default;
1870                                 return v;
1871                         case 'u':
1872                         case 'U':
1873                                 return EscapeUnicode (d, out surrogate);
1874                         default:
1875                                 surrogate = 0;
1876                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1877                                 return d;
1878                         }
1879
1880                         get_char ();
1881                         surrogate = 0;
1882                         return v;
1883                 }
1884
1885                 int EscapeUnicode (int ch, out int surrogate)
1886                 {
1887                         bool error;
1888                         if (ch == 'U') {
1889                                 ch = getHex (8, out surrogate, out error);
1890                         } else {
1891                                 ch = getHex (4, out surrogate, out error);
1892                         }
1893
1894                         if (error)
1895                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1896
1897                         return ch;
1898                 }
1899
1900                 int get_char ()
1901                 {
1902                         int x;
1903                         if (putback_char != -1) {
1904                                 x = putback_char;
1905                                 putback_char = -1;
1906                         } else {
1907                                 x = reader.Read ();
1908                         }
1909
1910                         if (x <= 13) {
1911                                 if (x == '\r') {
1912                                         if (peek_char () == '\n') {
1913                                                 putback_char = -1;
1914                                         }
1915
1916                                         x = '\n';
1917                                         advance_line ();
1918                                 } else if (x == '\n') {
1919                                         advance_line ();
1920                                 } else {
1921                                         col++;
1922                                 }
1923                         } else if (x >= UnicodeLS && x <= UnicodePS) {
1924                                 advance_line ();
1925                         } else {
1926                                 col++;
1927                         }
1928
1929                         return x;
1930                 }
1931
1932                 void advance_line ()
1933                 {
1934                         line++;
1935                         ref_line++;
1936                         previous_col = col;
1937                         col = 0;
1938                 }
1939
1940                 int peek_char ()
1941                 {
1942                         if (putback_char == -1)
1943                                 putback_char = reader.Read ();
1944                         return putback_char;
1945                 }
1946
1947                 int peek_char2 ()
1948                 {
1949                         if (putback_char != -1)
1950                                 return putback_char;
1951                         return reader.Peek ();
1952                 }
1953
1954                 public void putback (int c)
1955                 {
1956                         if (putback_char != -1) {
1957                                 throw new InternalErrorException (string.Format ("Secondary putback [{0}] putting back [{1}] is not allowed", (char)putback_char, (char) c), Location);
1958                         }
1959
1960                         if (c == '\n' || col == 0 || (c >= UnicodeLS && c <= UnicodePS)) {
1961                                 // It won't happen though.
1962                                 line--;
1963                                 ref_line--;
1964                                 col = previous_col;
1965                         }
1966                         else
1967                                 col--;
1968                         putback_char = c;
1969                 }
1970
1971                 public bool advance ()
1972                 {
1973                         return peek_char () != -1 || CompleteOnEOF;
1974                 }
1975
1976                 public Object Value {
1977                         get {
1978                                 return val;
1979                         }
1980                 }
1981
1982                 public Object value ()
1983                 {
1984                         return val;
1985                 }
1986
1987                 public int token ()
1988                 {
1989                         current_token = xtoken ();
1990                         return current_token;
1991                 }
1992
1993                 int TokenizePreprocessorKeyword (out int c)
1994                 {
1995                         // skip over white space
1996                         do {
1997                                 c = get_char ();
1998                         } while (c == ' ' || c == '\t');
1999
2000
2001                         int pos = 0;
2002                         while (c != -1 && c >= 'a' && c <= 'z') {
2003                                 id_builder[pos++] = (char) c;
2004                                 c = get_char ();
2005                                 if (c == '\\') {
2006                                         int peek = peek_char ();
2007                                         if (peek == 'U' || peek == 'u') {
2008                                                 int surrogate;
2009                                                 c = EscapeUnicode (c, out surrogate);
2010                                                 if (surrogate != 0) {
2011                                                         if (is_identifier_part_character ((char) c)) {
2012                                                                 id_builder[pos++] = (char) c;
2013                                                         }
2014                                                         c = surrogate;
2015                                                 }
2016                                         }
2017                                 }
2018                         }
2019
2020                         return pos;
2021                 }
2022
2023                 PreprocessorDirective get_cmd_arg (out string arg)
2024                 {
2025                         int c;
2026
2027                         tokens_seen = false;
2028                         arg = "";
2029
2030                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorKeyword (out c));
2031
2032                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
2033                                 return cmd;
2034
2035                         // skip over white space
2036                         while (c == ' ' || c == '\t')
2037                                 c = get_char ();
2038
2039                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
2040                         int pos = 0;
2041
2042                         while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
2043                                 if (c == '\\' && has_identifier_argument >= 0) {
2044                                         if (has_identifier_argument != 0) {
2045                                                 has_identifier_argument = 1;
2046
2047                                                 int peek = peek_char ();
2048                                                 if (peek == 'U' || peek == 'u') {
2049                                                         int surrogate;
2050                                                         c = EscapeUnicode (c, out surrogate);
2051                                                         if (surrogate != 0) {
2052                                                                 if (is_identifier_part_character ((char) c)) {
2053                                                                         if (pos == value_builder.Length)
2054                                                                                 Array.Resize (ref value_builder, pos * 2);
2055
2056                                                                         value_builder[pos++] = (char) c;
2057                                                                 }
2058                                                                 c = surrogate;
2059                                                         }
2060                                                 }
2061                                         } else {
2062                                                 has_identifier_argument = -1;
2063                                         }
2064                                 } else if (c == '/' && peek_char () == '/') {
2065                                         //
2066                                         // Eat single-line comments
2067                                         //
2068                                         get_char ();
2069                                         ReadToEndOfLine ();
2070                                         break;
2071                                 }
2072
2073                                 if (pos == value_builder.Length)
2074                                         Array.Resize (ref value_builder, pos * 2);
2075
2076                                 value_builder[pos++] = (char) c;
2077                                 c = get_char ();
2078                         }
2079
2080                         if (pos != 0) {
2081                                 if (pos > MaxIdentifierLength)
2082                                         arg = new string (value_builder, 0, pos);
2083                                 else
2084                                         arg = InternIdentifier (value_builder, pos);
2085
2086                                 // Eat any trailing whitespaces
2087                                 arg = arg.Trim (simple_whitespaces);
2088                         }
2089
2090                         return cmd;
2091                 }
2092
2093                 //
2094                 // Handles the #line directive
2095                 //
2096                 bool PreProcessLine ()
2097                 {
2098                         Location loc = Location;
2099
2100                         int c;
2101
2102                         int length = TokenizePreprocessorKeyword (out c);
2103                         if (length == line_default.Length) {
2104                                 if (!IsTokenIdentifierEqual (line_default))
2105                                         return false;
2106
2107                                 current_source = source_file.SourceFile;
2108                                 if (!hidden_block_start.IsNull) {
2109                                         current_source.RegisterHiddenScope (hidden_block_start, loc);
2110                                         hidden_block_start = Location.Null;
2111                                 }
2112
2113                                 ref_line = line;
2114                                 return true;
2115                         }
2116
2117                         if (length == line_hidden.Length) {
2118                                 if (!IsTokenIdentifierEqual (line_hidden))
2119                                         return false;
2120
2121                                 if (hidden_block_start.IsNull)
2122                                         hidden_block_start = loc;
2123
2124                                 return true;
2125                         }
2126
2127                         if (length != 0 || c < '0' || c > '9') {
2128                                 //
2129                                 // Eat any remaining characters to continue parsing on next line
2130                                 //
2131                                 ReadToEndOfLine ();
2132                                 return false;
2133                         }
2134
2135                         int new_line = TokenizeNumber (c);
2136                         if (new_line < 1) {
2137                                 //
2138                                 // Eat any remaining characters to continue parsing on next line
2139                                 //
2140                                 ReadToEndOfLine ();
2141                                 return new_line != 0;
2142                         }
2143
2144                         c = get_char ();
2145                         if (c == ' ') {
2146                                 // skip over white space
2147                                 do {
2148                                         c = get_char ();
2149                                 } while (c == ' ' || c == '\t');
2150                         } else if (c == '"') {
2151                                 c = 0;
2152                         }
2153
2154                         if (c != '\n' && c != '/' && c != '"' && c != UnicodeLS && c != UnicodePS) {
2155                                 //
2156                                 // Eat any remaining characters to continue parsing on next line
2157                                 //
2158                                 ReadToEndOfLine ();
2159
2160                                 Report.Error (1578, loc, "Filename, single-line comment or end-of-line expected");
2161                                 return true;
2162                         }
2163
2164                         string new_file_name = null;
2165                         if (c == '"') {
2166                                 new_file_name = TokenizeFileName (ref c);
2167
2168                                 // skip over white space
2169                                 while (c == ' ' || c == '\t') {
2170                                         c = get_char ();
2171                                 }
2172                         }
2173
2174                         if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
2175
2176                         } else if (c == '/') {
2177                                 ReadSingleLineComment ();
2178                         } else {
2179                                 //
2180                                 // Eat any remaining characters to continue parsing on next line
2181                                 //
2182                                 ReadToEndOfLine ();
2183
2184                                 Error_EndLineExpected ();
2185                                 return true;
2186                         }
2187
2188                         if (new_file_name != null) {
2189                                 current_source = context.LookupFile (source_file, new_file_name);
2190                                 source_file.AddIncludeFile (current_source);
2191                         }
2192
2193                         if (!hidden_block_start.IsNull) {
2194                                 current_source.RegisterHiddenScope (hidden_block_start, loc);
2195                                 hidden_block_start = Location.Null;
2196                         }
2197
2198                         ref_line = new_line;
2199                         return true;
2200                 }
2201
2202                 //
2203                 // Handles #define and #undef
2204                 //
2205                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
2206                 {
2207                         if (ident.Length == 0 || ident == "true" || ident == "false"){
2208                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
2209                                 return;
2210                         }
2211
2212                         if (ident.IndexOfAny (simple_whitespaces) != -1){
2213                                 Error_EndLineExpected ();
2214                                 return;
2215                         }
2216
2217                         if (!is_identifier_start_character (ident [0]))
2218                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
2219
2220                         foreach (char c in ident.Substring (1)){
2221                                 if (!is_identifier_part_character (c)){
2222                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
2223                                         return;
2224                                 }
2225                         }
2226
2227                         if (!caller_is_taking)
2228                                 return;
2229
2230                         if (is_define) {
2231                                 //
2232                                 // #define ident
2233                                 //
2234                                 if (context.Settings.IsConditionalSymbolDefined (ident))
2235                                         return;
2236
2237                                 source_file.AddDefine (ident);
2238                         } else {
2239                                 //
2240                                 // #undef ident
2241                                 //
2242                                 source_file.AddUndefine (ident);
2243                         }
2244                 }
2245
2246                 byte read_hex (out bool error)
2247                 {
2248                         int total;
2249                         int c = get_char ();
2250
2251                         if ((c >= '0') && (c <= '9'))
2252                                 total = (int) c - (int) '0';
2253                         else if ((c >= 'A') && (c <= 'F'))
2254                                 total = (int) c - (int) 'A' + 10;
2255                         else if ((c >= 'a') && (c <= 'f'))
2256                                 total = (int) c - (int) 'a' + 10;
2257                         else {
2258                                 error = true;
2259                                 return 0;
2260                         }
2261
2262                         total *= 16;
2263                         c = get_char ();
2264
2265                         if ((c >= '0') && (c <= '9'))
2266                                 total += (int) c - (int) '0';
2267                         else if ((c >= 'A') && (c <= 'F'))
2268                                 total += (int) c - (int) 'A' + 10;
2269                         else if ((c >= 'a') && (c <= 'f'))
2270                                 total += (int) c - (int) 'a' + 10;
2271                         else {
2272                                 error = true;
2273                                 return 0;
2274                         }
2275
2276                         error = false;
2277                         return (byte) total;
2278                 }
2279
2280                 //
2281                 // Parses #pragma checksum
2282                 //
2283                 bool ParsePragmaChecksum ()
2284                 {
2285                         //
2286                         // The syntax is ` "foo.txt" "{guid}" "hash"'
2287                         //
2288                         // guid is predefined hash algorithm guid {406ea660-64cf-4c82-b6f0-42d48172a799} for md5
2289                         //
2290                         int c = get_char ();
2291
2292                         if (c != '"')
2293                                 return false;
2294
2295                         string file_name = TokenizeFileName (ref c);
2296
2297                         // TODO: Any white-spaces count
2298                         if (c != ' ')
2299                                 return false;
2300
2301                         SourceFile file = context.LookupFile (source_file, file_name);
2302
2303                         if (get_char () != '"' || get_char () != '{')
2304                                 return false;
2305
2306                         bool error;
2307                         byte[] guid_bytes = new byte [16];
2308                         int i = 0;
2309
2310                         for (; i < 4; i++) {
2311                                 guid_bytes [i] = read_hex (out error);
2312                                 if (error)
2313                                         return false;
2314                         }
2315
2316                         if (get_char () != '-')
2317                                 return false;
2318
2319                         for (; i < 10; i++) {
2320                                 guid_bytes [i] = read_hex (out error);
2321                                 if (error)
2322                                         return false;
2323
2324                                 guid_bytes [i++] = read_hex (out error);
2325                                 if (error)
2326                                         return false;
2327
2328                                 if (get_char () != '-')
2329                                         return false;
2330                         }
2331
2332                         for (; i < 16; i++) {
2333                                 guid_bytes [i] = read_hex (out error);
2334                                 if (error)
2335                                         return false;
2336                         }
2337
2338                         if (get_char () != '}' || get_char () != '"')
2339                                 return false;
2340
2341                         // TODO: Any white-spaces count
2342                         c = get_char ();
2343                         if (c != ' ')
2344                                 return false;
2345
2346                         if (get_char () != '"')
2347                                 return false;
2348
2349                         // Any length of checksum
2350                         List<byte> checksum_bytes = new List<byte> (16);
2351
2352                         var checksum_location = Location;
2353                         c = peek_char ();
2354                         while (c != '"' && c != -1) {
2355                                 checksum_bytes.Add (read_hex (out error));
2356                                 if (error)
2357                                         return false;
2358
2359                                 c = peek_char ();
2360                         }
2361
2362                         if (c == '/') {
2363                                 ReadSingleLineComment ();
2364                         } else if (get_char () != '"') {
2365                                 return false;
2366                         }
2367
2368                         if (context.Settings.GenerateDebugInfo) {
2369                                 var chsum = checksum_bytes.ToArray ();
2370
2371                                 if (file.HasChecksum) {
2372                                         if (!ArrayComparer.IsEqual (file.Checksum, chsum)) {
2373                                                 // TODO: Report.SymbolRelatedToPreviousError
2374                                                 Report.Warning (1697, 1, checksum_location, "Different checksum values specified for file `{0}'", file.Name);
2375                                         }
2376                                 }
2377
2378                                 file.SetChecksum (guid_bytes, chsum);
2379                                 current_source.AutoGenerated = true;
2380                         }
2381
2382                         return true;
2383                 }
2384
2385                 bool IsTokenIdentifierEqual (char[] identifier)
2386                 {
2387                         for (int i = 0; i < identifier.Length; ++i) {
2388                                 if (identifier[i] != id_builder[i])
2389                                         return false;
2390                         }
2391
2392                         return true;
2393                 }
2394
2395                 int TokenizeNumber (int value)
2396                 {
2397                         number_pos = 0;
2398
2399                         decimal_digits (value);
2400                         uint ui = (uint) (number_builder[0] - '0');
2401
2402                         try {
2403                                 for (int i = 1; i < number_pos; i++) {
2404                                         ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2405                                 }
2406
2407                                 return (int) ui;
2408                         } catch (OverflowException) {
2409                                 Error_NumericConstantTooLong ();
2410                                 return -1;
2411                         }
2412                 }
2413
2414                 string TokenizeFileName (ref int c)
2415                 {
2416                         var string_builder = new StringBuilder ();
2417                         while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
2418                                 c = get_char ();
2419                                 if (c == '"') {
2420                                         c = get_char ();
2421                                         break;
2422                                 }
2423
2424                                 string_builder.Append ((char) c);
2425                         }
2426
2427                         if (string_builder.Length == 0) {
2428                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
2429                         }
2430
2431
2432                         return string_builder.ToString ();
2433                 }
2434
2435                 int TokenizePragmaWarningIdentifier (ref int c, ref bool identifier)
2436                 {
2437
2438                         if ((c >= '0' && c <= '9') || is_identifier_start_character (c)) {
2439                                 int number;
2440
2441                                 if (c >= '0' && c <= '9') {
2442                                         number_pos = 0;
2443                                         number = TokenizeNumber (c);
2444
2445                                         c = get_char ();
2446
2447                                         if (c != ' ' && c != '\t' && c != ',' && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS) {
2448                                                 return ReadPragmaWarningComment (c);
2449                                         }
2450                                 } else {
2451                                         //
2452                                         // LAMESPEC v6: No spec what identifier really is in this context, it seems keywords are allowed too
2453                                         //
2454                                         int pos = 0;
2455                                         number = -1;
2456                                         id_builder [pos++] = (char)c;
2457                                         while (c < MaxIdentifierLength) {
2458                                                 c = reader.Read ();
2459                                                 id_builder [pos] = (char)c;
2460
2461                                                 if (c >= '0' && c <= '9') {
2462                                                         if (pos == 6 && id_builder [0] == 'C' && id_builder [1] == 'S') {
2463                                                                 // Recognize CSXXXX as C# XXXX warning
2464                                                                 number = 0;
2465                                                                 int pow = 1000;
2466                                                                 for (int i = 0; i < 4; ++i) {
2467                                                                         var ch = id_builder [i + 2];
2468                                                                         if (ch < '0' || ch > '9') {
2469                                                                                 number = -1;
2470                                                                                 break;
2471                                                                         }
2472
2473                                                                         number += (ch - '0') * pow;
2474                                                                         pow /= 10;
2475                                                                 }
2476                                                         }
2477                                                 } else if ((c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && c != '_') {
2478                                                         break;
2479                                                 }
2480
2481                                                 ++pos;
2482                                         }
2483
2484                                         if (number < 0) {
2485                                                 identifier = true;
2486                                                 number = pos;
2487                                         }
2488                                 }
2489
2490                                 // skip over white space
2491                                 while (c == ' ' || c == '\t')
2492                                         c = get_char ();
2493
2494                                 if (c == ',') {
2495                                         c = get_char ();
2496                                 }
2497
2498                                 // skip over white space
2499                                 while (c == ' ' || c == '\t')
2500                                         c = get_char ();
2501
2502                                 return number;
2503                         }
2504
2505                         return ReadPragmaWarningComment (c);
2506                 }
2507
2508                 int ReadPragmaWarningComment (int c)
2509                 {
2510                         if (c == '/') {
2511                                 ReadSingleLineComment ();
2512                         } else {
2513                                 Report.Warning (1692, 1, Location, "Invalid number");
2514
2515                                 // Read everything till the end of the line or file
2516                                 ReadToEndOfLine ();
2517                         }
2518
2519                         return -1;
2520                 }
2521
2522                 void ReadToEndOfLine ()
2523                 {
2524                         int c;
2525                         do {
2526                                 c = get_char ();
2527                         } while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS);
2528                 }
2529
2530                 void ReadSingleLineComment ()
2531                 {
2532                         if (peek_char () != '/')
2533                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2534
2535                         // Read everything till the end of the line or file
2536                         ReadToEndOfLine ();
2537                 }
2538
2539                 /// <summary>
2540                 /// Handles #pragma directive
2541                 /// </summary>
2542                 void ParsePragmaDirective ()
2543                 {
2544                         int c;
2545                         int length = TokenizePreprocessorKeyword (out c);
2546                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2547                                 length = TokenizePreprocessorKeyword (out c);
2548
2549                                 //
2550                                 // #pragma warning disable
2551                                 // #pragma warning restore
2552                                 //
2553                                 if (length == pragma_warning_disable.Length) {
2554                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2555                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2556                                                 // skip over white space
2557                                                 while (c == ' ' || c == '\t')
2558                                                         c = get_char ();
2559
2560                                                 var loc = Location;
2561
2562                                                 if (c == '\n' || c == '/' || c == UnicodeLS || c == UnicodePS) {
2563                                                         if (c == '/')
2564                                                                 ReadSingleLineComment ();
2565
2566                                                         //
2567                                                         // Disable/Restore all warnings
2568                                                         //
2569                                                         if (disable) {
2570                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2571                                                         } else {
2572                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2573                                                         }
2574                                                 } else {
2575                                                         //
2576                                                         // Disable/Restore a warning or group of warnings
2577                                                         //
2578                                                         int code;
2579                                                         do {
2580                                                                 bool identifier = false;
2581                                                                 code = TokenizePragmaWarningIdentifier (ref c, ref identifier);
2582                                                                 if (code > 0) {
2583                                                                         if (identifier) {
2584                                                                                 // no-op, custom warnings cannot occur in mcs
2585                                                                         } else if (disable) {
2586                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, context.Report);
2587                                                                         } else {
2588                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, context);
2589                                                                         }
2590                                                                 }
2591                                                         } while (code >= 0 && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS);
2592                                                 }
2593
2594                                                 return;
2595                                         }
2596                                 }
2597
2598                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2599
2600                                 // Eat any remaining characters on the line
2601                                 ReadToEndOfLine ();
2602
2603                                 return;
2604                         }
2605
2606                         //
2607                         // #pragma checksum
2608                         //
2609                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2610                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2611                                         Report.Warning (1695, 1, Location,
2612                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2613                                 }
2614
2615                                 return;
2616                         }
2617
2618                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2619                 }
2620
2621                 bool eval_val (string s)
2622                 {
2623                         if (s == "true")
2624                                 return true;
2625                         if (s == "false")
2626                                 return false;
2627
2628                         return source_file.IsConditionalDefined (s);
2629                 }
2630
2631                 bool pp_primary (ref string s)
2632                 {
2633                         s = s.Trim ();
2634                         int len = s.Length;
2635
2636                         if (len > 0){
2637                                 char c = s [0];
2638
2639                                 if (c == '('){
2640                                         s = s.Substring (1);
2641                                         bool val = pp_expr (ref s, false);
2642                                         if (s.Length > 0 && s [0] == ')'){
2643                                                 s = s.Substring (1);
2644                                                 return val;
2645                                         }
2646                                         Error_InvalidDirective ();
2647                                         return false;
2648                                 }
2649
2650                                 if (is_identifier_start_character (c)){
2651                                         int j = 1;
2652
2653                                         while (j < len){
2654                                                 c = s [j];
2655
2656                                                 if (is_identifier_part_character (c)){
2657                                                         j++;
2658                                                         continue;
2659                                                 }
2660                                                 bool v = eval_val (s.Substring (0, j));
2661                                                 s = s.Substring (j);
2662                                                 return v;
2663                                         }
2664                                         bool vv = eval_val (s);
2665                                         s = "";
2666                                         return vv;
2667                                 }
2668                         }
2669                         Error_InvalidDirective ();
2670                         return false;
2671                 }
2672
2673                 bool pp_unary (ref string s)
2674                 {
2675                         s = s.Trim ();
2676                         int len = s.Length;
2677
2678                         if (len > 0){
2679                                 if (s [0] == '!'){
2680                                         if (len > 1 && s [1] == '='){
2681                                                 Error_InvalidDirective ();
2682                                                 return false;
2683                                         }
2684                                         s = s.Substring (1);
2685                                         return ! pp_primary (ref s);
2686                                 } else
2687                                         return pp_primary (ref s);
2688                         } else {
2689                                 Error_InvalidDirective ();
2690                                 return false;
2691                         }
2692                 }
2693
2694                 bool pp_eq (ref string s)
2695                 {
2696                         bool va = pp_unary (ref s);
2697
2698                         s = s.Trim ();
2699                         int len = s.Length;
2700                         if (len > 0){
2701                                 if (s [0] == '='){
2702                                         if (len > 2 && s [1] == '='){
2703                                                 s = s.Substring (2);
2704                                                 return va == pp_unary (ref s);
2705                                         } else {
2706                                                 Error_InvalidDirective ();
2707                                                 return false;
2708                                         }
2709                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2710                                         s = s.Substring (2);
2711
2712                                         return va != pp_unary (ref s);
2713
2714                                 }
2715                         }
2716
2717                         return va;
2718
2719                 }
2720
2721                 bool pp_and (ref string s)
2722                 {
2723                         bool va = pp_eq (ref s);
2724
2725                         s = s.Trim ();
2726                         int len = s.Length;
2727                         if (len > 0){
2728                                 if (s [0] == '&'){
2729                                         if (len > 2 && s [1] == '&'){
2730                                                 s = s.Substring (2);
2731                                                 return (va & pp_and (ref s));
2732                                         } else {
2733                                                 Error_InvalidDirective ();
2734                                                 return false;
2735                                         }
2736                                 }
2737                         }
2738                         return va;
2739                 }
2740
2741                 //
2742                 // Evaluates an expression for `#if' or `#elif'
2743                 //
2744                 bool pp_expr (ref string s, bool isTerm)
2745                 {
2746                         bool va = pp_and (ref s);
2747                         s = s.Trim ();
2748                         int len = s.Length;
2749                         if (len > 0){
2750                                 char c = s [0];
2751
2752                                 if (c == '|'){
2753                                         if (len > 2 && s [1] == '|'){
2754                                                 s = s.Substring (2);
2755                                                 return va | pp_expr (ref s, isTerm);
2756                                         } else {
2757                                                 Error_InvalidDirective ();
2758                                                 return false;
2759                                         }
2760                                 }
2761                                 if (isTerm) {
2762                                         Error_EndLineExpected ();
2763                                         return false;
2764                                 }
2765                         }
2766
2767                         return va;
2768                 }
2769
2770                 bool eval (string s)
2771                 {
2772                         bool v = pp_expr (ref s, true);
2773                         s = s.Trim ();
2774                         if (s.Length != 0){
2775                                 return false;
2776                         }
2777
2778                         return v;
2779                 }
2780
2781                 void Error_NumericConstantTooLong ()
2782                 {
2783                         Report.Error (1021, Location, "Integral constant is too large");
2784                 }
2785
2786                 void Error_InvalidDirective ()
2787                 {
2788                         Report.Error (1517, Location, "Invalid preprocessor directive");
2789                 }
2790
2791                 void Error_UnexpectedDirective (string extra)
2792                 {
2793                         Report.Error (
2794                                 1028, Location,
2795                                 "Unexpected processor directive ({0})", extra);
2796                 }
2797
2798                 void Error_TokensSeen ()
2799                 {
2800                         Report.Error (1032, Location,
2801                                 "Cannot define or undefine preprocessor symbols after first token in file");
2802                 }
2803
2804                 void Eror_WrongPreprocessorLocation ()
2805                 {
2806                         Report.Error (1040, Location,
2807                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2808                 }
2809
2810                 void Error_EndLineExpected ()
2811                 {
2812                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2813                 }
2814
2815                 //
2816                 // Raises a warning when tokenizer found documentation comment
2817                 // on unexpected place
2818                 //
2819                 void WarningMisplacedComment (Location loc)
2820                 {
2821                         if (doc_state != XmlCommentState.Error) {
2822                                 doc_state = XmlCommentState.Error;
2823                                 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element");
2824                         }
2825                 }
2826
2827                 //
2828                 // if true, then the code continues processing the code
2829                 // if false, the code stays in a loop until another directive is
2830                 // reached.
2831                 // When caller_is_taking is false we ignore all directives except the ones
2832                 // which can help us to identify where the #if block ends
2833                 bool ParsePreprocessingDirective (bool caller_is_taking)
2834                 {
2835                         string arg;
2836                         bool region_directive = false;
2837
2838                         var directive = get_cmd_arg (out arg);
2839
2840                         //
2841                         // The first group of pre-processing instructions is always processed
2842                         //
2843                         switch (directive) {
2844                         case PreprocessorDirective.Region:
2845                                 region_directive = true;
2846                                 arg = "true";
2847                                 goto case PreprocessorDirective.If;
2848
2849                         case PreprocessorDirective.Endregion:
2850                                 if (ifstack == null || ifstack.Count == 0){
2851                                         Error_UnexpectedDirective ("no #region for this #endregion");
2852                                         return true;
2853                                 }
2854                                 int pop = ifstack.Pop ();
2855
2856                                 if ((pop & REGION) == 0)
2857                                         Report.Error (1027, Location, "Expected `#endif' directive");
2858
2859                                 return caller_is_taking;
2860
2861                         case PreprocessorDirective.If:
2862                                 if (ifstack == null)
2863                                         ifstack = new Stack<int> (2);
2864
2865                                 int flags = region_directive ? REGION : 0;
2866                                 if (ifstack.Count == 0){
2867                                         flags |= PARENT_TAKING;
2868                                 } else {
2869                                         int state = ifstack.Peek ();
2870                                         if ((state & TAKING) != 0) {
2871                                                 flags |= PARENT_TAKING;
2872                                         }
2873                                 }
2874
2875                                 if (eval (arg) && caller_is_taking) {
2876                                         ifstack.Push (flags | TAKING);
2877                                         return true;
2878                                 }
2879                                 ifstack.Push (flags);
2880                                 return false;
2881
2882                         case PreprocessorDirective.Endif:
2883                                 if (ifstack == null || ifstack.Count == 0){
2884                                         Error_UnexpectedDirective ("no #if for this #endif");
2885                                         return true;
2886                                 } else {
2887                                         pop = ifstack.Pop ();
2888
2889                                         if ((pop & REGION) != 0)
2890                                                 Report.Error (1038, Location, "#endregion directive expected");
2891
2892                                         if (arg.Length != 0) {
2893                                                 Error_EndLineExpected ();
2894                                         }
2895
2896                                         if (ifstack.Count == 0)
2897                                                 return true;
2898
2899                                         int state = ifstack.Peek ();
2900                                         return (state & TAKING) != 0;
2901                                 }
2902
2903                         case PreprocessorDirective.Elif:
2904                                 if (ifstack == null || ifstack.Count == 0){
2905                                         Error_UnexpectedDirective ("no #if for this #elif");
2906                                         return true;
2907                                 } else {
2908                                         int state = ifstack.Pop ();
2909
2910                                         if ((state & REGION) != 0) {
2911                                                 Report.Error (1038, Location, "#endregion directive expected");
2912                                                 return true;
2913                                         }
2914
2915                                         if ((state & ELSE_SEEN) != 0){
2916                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2917                                                 return true;
2918                                         }
2919
2920                                         if ((state & TAKING) != 0) {
2921                                                 ifstack.Push (0);
2922                                                 return false;
2923                                         }
2924
2925                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2926                                                 ifstack.Push (state | TAKING);
2927                                                 return true;
2928                                         }
2929
2930                                         ifstack.Push (state);
2931                                         return false;
2932                                 }
2933
2934                         case PreprocessorDirective.Else:
2935                                 if (ifstack == null || ifstack.Count == 0){
2936                                         Error_UnexpectedDirective ("no #if for this #else");
2937                                         return true;
2938                                 } else {
2939                                         int state = ifstack.Peek ();
2940
2941                                         if ((state & REGION) != 0) {
2942                                                 Report.Error (1038, Location, "#endregion directive expected");
2943                                                 return true;
2944                                         }
2945
2946                                         if ((state & ELSE_SEEN) != 0){
2947                                                 Error_UnexpectedDirective ("#else within #else");
2948                                                 return true;
2949                                         }
2950
2951                                         ifstack.Pop ();
2952
2953                                         if (arg.Length != 0) {
2954                                                 Error_EndLineExpected ();
2955                                                 return true;
2956                                         }
2957
2958                                         bool ret = false;
2959                                         if ((state & PARENT_TAKING) != 0) {
2960                                                 ret = (state & TAKING) == 0;
2961
2962                                                 if (ret)
2963                                                         state |= TAKING;
2964                                                 else
2965                                                         state &= ~TAKING;
2966                                         }
2967
2968                                         ifstack.Push (state | ELSE_SEEN);
2969
2970                                         return ret;
2971                                 }
2972                         case PreprocessorDirective.Define:
2973                                 if (any_token_seen){
2974                                         if (caller_is_taking)
2975                                                 Error_TokensSeen ();
2976                                         return caller_is_taking;
2977                                 }
2978                                 PreProcessDefinition (true, arg, caller_is_taking);
2979                                 return caller_is_taking;
2980
2981                         case PreprocessorDirective.Undef:
2982                                 if (any_token_seen){
2983                                         if (caller_is_taking)
2984                                                 Error_TokensSeen ();
2985                                         return caller_is_taking;
2986                                 }
2987                                 PreProcessDefinition (false, arg, caller_is_taking);
2988                                 return caller_is_taking;
2989
2990                         case PreprocessorDirective.Invalid:
2991                                 Report.Error (1024, Location, "Wrong preprocessor directive");
2992                                 return true;
2993                         }
2994
2995                         //
2996                         // These are only processed if we are in a `taking' block
2997                         //
2998                         if (!caller_is_taking)
2999                                 return false;
3000
3001                         switch (directive){
3002                         case PreprocessorDirective.Error:
3003                                 Report.Error (1029, Location, "#error: '{0}'", arg);
3004                                 return true;
3005
3006                         case PreprocessorDirective.Warning:
3007                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
3008                                 return true;
3009
3010                         case PreprocessorDirective.Pragma:
3011                                 if (context.Settings.Version == LanguageVersion.ISO_1) {
3012                                         Report.FeatureIsNotAvailable (context, Location, "#pragma");
3013                                 }
3014
3015                                 ParsePragmaDirective ();
3016                                 return true;
3017
3018                         case PreprocessorDirective.Line:
3019                                 Location loc = Location;
3020                                 if (!PreProcessLine ())
3021                                         Report.Error (1576, loc, "The line number specified for #line directive is missing or invalid");
3022
3023                                 return caller_is_taking;
3024                         }
3025
3026                         throw new NotImplementedException (directive.ToString ());
3027                 }
3028
3029                 private int consume_string (bool quoted)
3030                 {
3031                         int c;
3032                         int pos = 0;
3033                         Location start_location = Location;
3034                         if (quoted)
3035                                 start_location = start_location - 1;
3036
3037 #if FULL_AST
3038                         int reader_pos = reader.Position;
3039 #endif
3040
3041                         while (true){
3042                                 // Cannot use get_char because of \r in quoted strings
3043                                 if (putback_char != -1) {
3044                                         c = putback_char;
3045                                         putback_char = -1;
3046                                 } else {
3047                                         c = reader.Read ();
3048                                 }
3049
3050                                 if (c == '"') {
3051                                         ++col;
3052
3053                                         if (quoted && peek_char () == '"') {
3054                                                 if (pos == value_builder.Length)
3055                                                         Array.Resize (ref value_builder, pos * 2);
3056
3057                                                 value_builder[pos++] = (char) c;
3058                                                 get_char ();
3059                                                 continue;
3060                                         }
3061
3062                                         string s;
3063                                         if (pos == 0)
3064                                                 s = string.Empty;
3065                                         else if (pos <= 4)
3066                                                 s = InternIdentifier (value_builder, pos);
3067                                         else
3068                                                 s = new string (value_builder, 0, pos);
3069
3070                                         ILiteralConstant res = new StringLiteral (context.BuiltinTypes, s, start_location);
3071                                         val = res;
3072 #if FULL_AST
3073                                         res.ParsedValue = quoted ?
3074                                                 reader.ReadChars (reader_pos - 2, reader.Position - 1) :
3075                                                 reader.ReadChars (reader_pos - 1, reader.Position);
3076 #endif
3077
3078                                         return Token.LITERAL;
3079                                 }
3080
3081                                 if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
3082                                         if (!quoted) {
3083                                                 Report.Error (1010, Location, "Newline in constant");
3084
3085                                                 advance_line ();
3086
3087                                                 // Don't add \r to string literal
3088                                                 if (pos > 1 && value_builder [pos - 1] == '\r')
3089                                                         --pos;
3090
3091                                                 val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location);
3092                                                 return Token.LITERAL;
3093                                         }
3094
3095                                         advance_line ();
3096                                 } else if (c == '\\' && !quoted) {
3097                                         ++col;
3098                                         int surrogate;
3099                                         c = escape (c, out surrogate);
3100                                         if (c == -1)
3101                                                 return Token.ERROR;
3102                                         if (surrogate != 0) {
3103                                                 if (pos == value_builder.Length)
3104                                                         Array.Resize (ref value_builder, pos * 2);
3105
3106                                                 value_builder[pos++] = (char) c;
3107                                                 c = surrogate;
3108                                         }
3109                                 } else if (c == -1) {
3110                                         Report.Error (1039, Location, "Unterminated string literal");
3111                                         return Token.EOF;
3112                                 } else {
3113                                         ++col;
3114                                 }
3115
3116                                 if (pos == value_builder.Length)
3117                                         Array.Resize (ref value_builder, pos * 2);
3118
3119                                 value_builder[pos++] = (char) c;
3120                         }
3121                 }
3122
3123                 private int consume_identifier (int s)
3124                 {
3125                         int res = consume_identifier (s, false);
3126
3127                         if (doc_state == XmlCommentState.Allowed)
3128                                 doc_state = XmlCommentState.NotAllowed;
3129
3130                         return res;
3131                 }
3132
3133                 int consume_identifier (int c, bool quoted)
3134                 {
3135                         //
3136                         // This method is very performance sensitive. It accounts
3137                         // for approximately 25% of all parser time
3138                         //
3139
3140                         int pos = 0;
3141                         int column = col;
3142                         if (quoted)
3143                                 --column;
3144
3145                         if (c == '\\') {
3146                                 int surrogate;
3147                                 c = escape (c, out surrogate);
3148                                 if (surrogate != 0) {
3149                                         id_builder [pos++] = (char) c;
3150                                         c = surrogate;
3151                                 }
3152                         }
3153
3154                         id_builder [pos++] = (char) c;
3155
3156                         try {
3157                                 while (true) {
3158                                         c = reader.Read ();
3159
3160                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
3161                                                 id_builder [pos++] = (char) c;
3162                                                 continue;
3163                                         }
3164
3165                                         if (c < 0x80) {
3166                                                 if (c == '\\') {
3167                                                         int surrogate;
3168                                                         c = escape (c, out surrogate);
3169                                                         if (is_identifier_part_character ((char) c))
3170                                                                 id_builder[pos++] = (char) c;
3171
3172                                                         if (surrogate != 0) {
3173                                                                 c = surrogate;
3174                                                         }
3175
3176                                                         continue;
3177                                                 }
3178                                         } else if (is_identifier_part_character_slow_part ((char) c)) {
3179                                                 id_builder [pos++] = (char) c;
3180                                                 continue;
3181                                         }
3182
3183                                         putback_char = c;
3184                                         break;
3185                                 }
3186                         } catch (IndexOutOfRangeException) {
3187                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
3188                                 --pos;
3189                                 col += pos;
3190                         }
3191
3192                         col += pos - 1;
3193
3194                         //
3195                         // Optimization: avoids doing the keyword lookup
3196                         // on uppercase letters
3197                         //
3198                         if (id_builder [0] >= '_' && !quoted) {
3199                                 int keyword = GetKeyword (id_builder, pos);
3200                                 if (keyword != -1) {
3201                                         val = ltb.Create (keyword == Token.AWAIT ? "await" : null, current_source, ref_line, column);
3202                                         return keyword;
3203                                 }
3204                         }
3205
3206                         string s = InternIdentifier (id_builder, pos);
3207                         val = ltb.Create (s, current_source, ref_line, column);
3208                         if (quoted && parsing_attribute_section)
3209                                 AddEscapedIdentifier (((LocatedToken) val).Location);
3210
3211                         return Token.IDENTIFIER;
3212                 }
3213
3214                 string InternIdentifier (char[] charBuffer, int length)
3215                 {
3216                         //
3217                         // Keep identifiers in an array of hashtables to avoid needless
3218                         // allocations
3219                         //
3220                         var identifiers_group = identifiers[length];
3221                         string s;
3222                         if (identifiers_group != null) {
3223                                 if (identifiers_group.TryGetValue (charBuffer, out s)) {
3224                                         return s;
3225                                 }
3226                         } else {
3227                                 // TODO: this should be number of files dependant
3228                                 // corlib compilation peaks at 1000 and System.Core at 150
3229                                 int capacity = length > 20 ? 10 : 100;
3230                                 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length));
3231                                 identifiers[length] = identifiers_group;
3232                         }
3233
3234                         char[] chars = new char[length];
3235                         Array.Copy (charBuffer, chars, length);
3236
3237                         s = new string (charBuffer, 0, length);
3238                         identifiers_group.Add (chars, s);
3239                         return s;
3240                 }
3241
3242                 public int xtoken ()
3243                 {
3244                         int d, c;
3245
3246                         // Whether we have seen comments on the current line
3247                         bool comments_seen = false;
3248                         while ((c = get_char ()) != -1) {
3249                                 switch (c) {
3250                                 case '\t':
3251                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
3252                                         continue;
3253
3254                                 case ' ':
3255                                 case '\f':
3256                                 case '\v':
3257                                 case 0xa0:
3258                                 case 0:
3259                                 case 0xFEFF:    // Ignore BOM anywhere in the file
3260                                         continue;
3261
3262 /*                              This is required for compatibility with .NET
3263                                 case 0xEF:
3264                                         if (peek_char () == 0xBB) {
3265                                                 PushPosition ();
3266                                                 get_char ();
3267                                                 if (get_char () == 0xBF)
3268                                                         continue;
3269                                                 PopPosition ();
3270                                         }
3271                                         break;
3272 */
3273                                 case '\\':
3274                                         tokens_seen = true;
3275                                         return consume_identifier (c);
3276
3277                                 case '{':
3278                                         val = ltb.Create (current_source, ref_line, col);
3279                                         return Token.OPEN_BRACE;
3280                                 case '}':
3281                                         val = ltb.Create (current_source, ref_line, col);
3282                                         return Token.CLOSE_BRACE;
3283                                 case '[':
3284                                         // To block doccomment inside attribute declaration.
3285                                         if (doc_state == XmlCommentState.Allowed)
3286                                                 doc_state = XmlCommentState.NotAllowed;
3287
3288                                         val = ltb.Create (current_source, ref_line, col);
3289
3290                                         if (parsing_block == 0 || lambda_arguments_parsing)
3291                                                 return Token.OPEN_BRACKET;
3292
3293                                         int next = peek_char ();
3294                                         switch (next) {
3295                                         case ']':
3296                                         case ',':
3297                                                 return Token.OPEN_BRACKET;
3298
3299                                         case ' ':
3300                                         case '\f':
3301                                         case '\v':
3302                                         case '\r':
3303                                         case '\n':
3304                                         case UnicodeLS:
3305                                         case UnicodePS:
3306                                         case '/':
3307                                                 next = peek_token ();
3308                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
3309                                                         return Token.OPEN_BRACKET;
3310
3311                                                 return Token.OPEN_BRACKET_EXPR;
3312                                         default:
3313                                                 return Token.OPEN_BRACKET_EXPR;
3314                                         }
3315                                 case ']':
3316                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3317                                         return Token.CLOSE_BRACKET;
3318                                 case '(':
3319                                         val = ltb.Create (current_source, ref_line, col);
3320                                         //
3321                                         // An expression versions of parens can appear in block context only
3322                                         //
3323                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
3324
3325                                                 //
3326                                                 // Optmize most common case where we know that parens
3327                                                 // is not special
3328                                                 //
3329                                                 switch (current_token) {
3330                                                 case Token.IDENTIFIER:
3331                                                 case Token.IF:
3332                                                 case Token.FOR:
3333                                                 case Token.FOREACH:
3334                                                 case Token.TYPEOF:
3335                                                 case Token.WHILE:
3336                                                 case Token.SWITCH:
3337                                                 case Token.USING:
3338                                                 case Token.DEFAULT:
3339                                                 case Token.DELEGATE:
3340                                                 case Token.OP_GENERICS_GT:
3341                                                         return Token.OPEN_PARENS;
3342                                                 }
3343
3344                                                 // Optimize using peek
3345                                                 int xx = peek_char ();
3346                                                 switch (xx) {
3347                                                 case '(':
3348                                                 case '\'':
3349                                                 case '"':
3350                                                 case '0':
3351                                                 case '1':
3352                                                         return Token.OPEN_PARENS;
3353                                                 }
3354
3355                                                 lambda_arguments_parsing = true;
3356                                                 PushPosition ();
3357                                                 d = TokenizeOpenParens ();
3358                                                 PopPosition ();
3359                                                 lambda_arguments_parsing = false;
3360                                                 return d;
3361                                         }
3362
3363                                         return Token.OPEN_PARENS;
3364                                 case ')':
3365                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3366                                         return Token.CLOSE_PARENS;
3367                                 case ',':
3368                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3369                                         return Token.COMMA;
3370                                 case ';':
3371                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3372                                         return Token.SEMICOLON;
3373                                 case '~':
3374                                         val = ltb.Create (current_source, ref_line, col);
3375                                         return Token.TILDE;
3376                                 case '?':
3377                                         val = ltb.Create (current_source, ref_line, col);
3378                                         return TokenizePossibleNullableType ();
3379                                 case '<':
3380                                         val = ltb.Create (current_source, ref_line, col);
3381                                         if (parsing_generic_less_than++ > 0)
3382                                                 return Token.OP_GENERICS_LT;
3383
3384                                         return TokenizeLessThan ();
3385
3386                                 case '>':
3387                                         val = ltb.Create (current_source, ref_line, col);
3388                                         d = peek_char ();
3389
3390                                         if (d == '='){
3391                                                 get_char ();
3392                                                 return Token.OP_GE;
3393                                         }
3394
3395                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
3396                                                 parsing_generic_less_than--;
3397                                                 return Token.OP_GENERICS_GT;
3398                                         }
3399
3400                                         if (d == '>') {
3401                                                 get_char ();
3402                                                 d = peek_char ();
3403
3404                                                 if (d == '=') {
3405                                                         get_char ();
3406                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
3407                                                 }
3408                                                 return Token.OP_SHIFT_RIGHT;
3409                                         }
3410
3411                                         return Token.OP_GT;
3412
3413                                 case '+':
3414                                         val = ltb.Create (current_source, ref_line, col);
3415                                         d = peek_char ();
3416                                         if (d == '+') {
3417                                                 d = Token.OP_INC;
3418                                         } else if (d == '=') {
3419                                                 d = Token.OP_ADD_ASSIGN;
3420                                         } else {
3421                                                 return Token.PLUS;
3422                                         }
3423                                         get_char ();
3424                                         return d;
3425
3426                                 case '-':
3427                                         val = ltb.Create (current_source, ref_line, col);
3428                                         d = peek_char ();
3429                                         if (d == '-') {
3430                                                 d = Token.OP_DEC;
3431                                         } else if (d == '=')
3432                                                 d = Token.OP_SUB_ASSIGN;
3433                                         else if (d == '>')
3434                                                 d = Token.OP_PTR;
3435                                         else {
3436                                                 return Token.MINUS;
3437                                         }
3438                                         get_char ();
3439                                         return d;
3440
3441                                 case '!':
3442                                         val = ltb.Create (current_source, ref_line, col);
3443                                         if (peek_char () == '='){
3444                                                 get_char ();
3445                                                 return Token.OP_NE;
3446                                         }
3447                                         return Token.BANG;
3448
3449                                 case '=':
3450                                         val = ltb.Create (current_source, ref_line, col);
3451                                         d = peek_char ();
3452                                         if (d == '='){
3453                                                 get_char ();
3454                                                 return Token.OP_EQ;
3455                                         }
3456                                         if (d == '>'){
3457                                                 get_char ();
3458                                                 return Token.ARROW;
3459                                         }
3460
3461                                         return Token.ASSIGN;
3462
3463                                 case '&':
3464                                         val = ltb.Create (current_source, ref_line, col);
3465                                         d = peek_char ();
3466                                         if (d == '&'){
3467                                                 get_char ();
3468                                                 return Token.OP_AND;
3469                                         }
3470                                         if (d == '='){
3471                                                 get_char ();
3472                                                 return Token.OP_AND_ASSIGN;
3473                                         }
3474                                         return Token.BITWISE_AND;
3475
3476                                 case '|':
3477                                         val = ltb.Create (current_source, ref_line, col);
3478                                         d = peek_char ();
3479                                         if (d == '|'){
3480                                                 get_char ();
3481                                                 return Token.OP_OR;
3482                                         }
3483                                         if (d == '='){
3484                                                 get_char ();
3485                                                 return Token.OP_OR_ASSIGN;
3486                                         }
3487                                         return Token.BITWISE_OR;
3488
3489                                 case '*':
3490                                         val = ltb.Create (current_source, ref_line, col);
3491                                         if (peek_char () == '='){
3492                                                 get_char ();
3493                                                 return Token.OP_MULT_ASSIGN;
3494                                         }
3495                                         return Token.STAR;
3496
3497                                 case '/':
3498                                         d = peek_char ();
3499                                         if (d == '='){
3500                                                 val = ltb.Create (current_source, ref_line, col);
3501                                                 get_char ();
3502                                                 return Token.OP_DIV_ASSIGN;
3503                                         }
3504
3505                                         // Handle double-slash comments.
3506                                         if (d == '/'){
3507                                                 get_char ();
3508                                                 if (doc_processing) {
3509                                                         if (peek_char () == '/') {
3510                                                                 get_char ();
3511                                                                 // Don't allow ////.
3512                                                                 if ((d = peek_char ()) != '/') {
3513                                                                         if (doc_state == XmlCommentState.Allowed)
3514                                                                                 handle_one_line_xml_comment ();
3515                                                                         else if (doc_state == XmlCommentState.NotAllowed)
3516                                                                                 WarningMisplacedComment (Location - 3);
3517                                                                 }
3518                                                         } else {
3519                                                                 if (xml_comment_buffer.Length > 0)
3520                                                                         doc_state = XmlCommentState.NotAllowed;
3521                                                         }
3522                                                 }
3523
3524                                                 ReadToEndOfLine ();
3525
3526                                                 any_token_seen |= tokens_seen;
3527                                                 tokens_seen = false;
3528                                                 comments_seen = false;
3529                                                 continue;
3530                                         } else if (d == '*'){
3531                                                 get_char ();
3532                                                 bool docAppend = false;
3533                                                 if (doc_processing && peek_char () == '*') {
3534                                                         get_char ();
3535                                                         // But when it is /**/, just do nothing.
3536                                                         if (peek_char () == '/') {
3537                                                                 get_char ();
3538                                                                 continue;
3539                                                         }
3540                                                         if (doc_state == XmlCommentState.Allowed)
3541                                                                 docAppend = true;
3542                                                         else if (doc_state == XmlCommentState.NotAllowed) {
3543                                                                 WarningMisplacedComment (Location - 2);
3544                                                         }
3545                                                 }
3546
3547                                                 int current_comment_start = 0;
3548                                                 if (docAppend) {
3549                                                         current_comment_start = xml_comment_buffer.Length;
3550                                                         xml_comment_buffer.Append (Environment.NewLine);
3551                                                 }
3552
3553                                                 while ((d = get_char ()) != -1){
3554                                                         if (d == '*' && peek_char () == '/'){
3555                                                                 get_char ();
3556                                                                 comments_seen = true;
3557                                                                 break;
3558                                                         }
3559                                                         if (docAppend)
3560                                                                 xml_comment_buffer.Append ((char) d);
3561
3562                                                         if (d == '\n' || d == UnicodeLS || d == UnicodePS){
3563                                                                 any_token_seen |= tokens_seen;
3564                                                                 tokens_seen = false;
3565                                                                 //
3566                                                                 // Reset 'comments_seen' just to be consistent.
3567                                                                 // It doesn't matter either way, here.
3568                                                                 //
3569                                                                 comments_seen = false;
3570                                                         }
3571                                                 }
3572                                                 if (!comments_seen)
3573                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3574
3575                                                 if (docAppend)
3576                                                         update_formatted_doc_comment (current_comment_start);
3577                                                 continue;
3578                                         }
3579                                         val = ltb.Create (current_source, ref_line, col);
3580                                         return Token.DIV;
3581
3582                                 case '%':
3583                                         val = ltb.Create (current_source, ref_line, col);
3584                                         if (peek_char () == '='){
3585                                                 get_char ();
3586                                                 return Token.OP_MOD_ASSIGN;
3587                                         }
3588                                         return Token.PERCENT;
3589
3590                                 case '^':
3591                                         val = ltb.Create (current_source, ref_line, col);
3592                                         if (peek_char () == '='){
3593                                                 get_char ();
3594                                                 return Token.OP_XOR_ASSIGN;
3595                                         }
3596                                         return Token.CARRET;
3597
3598                                 case ':':
3599                                         val = ltb.Create (current_source, ref_line, col);
3600                                         if (peek_char () == ':') {
3601                                                 get_char ();
3602                                                 return Token.DOUBLE_COLON;
3603                                         }
3604                                         return Token.COLON;
3605
3606                                 case '0': case '1': case '2': case '3': case '4':
3607                                 case '5': case '6': case '7': case '8': case '9':
3608                                         tokens_seen = true;
3609                                         return is_number (c, false);
3610
3611                                 case '\n': // white space
3612                                 case UnicodeLS:
3613                                 case UnicodePS:
3614                                         any_token_seen |= tokens_seen;
3615                                         tokens_seen = false;
3616                                         comments_seen = false;
3617                                         continue;
3618
3619                                 case '.':
3620                                         tokens_seen = true;
3621                                         d = peek_char ();
3622                                         if (d >= '0' && d <= '9')
3623                                                 return is_number (c, true);
3624
3625                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3626                                         return Token.DOT;
3627
3628                                 case '#':
3629                                         if (tokens_seen || comments_seen) {
3630                                                 Eror_WrongPreprocessorLocation ();
3631                                                 return Token.ERROR;
3632                                         }
3633
3634                                         if (ParsePreprocessingDirective (true))
3635                                                 continue;
3636
3637                                         bool directive_expected = false;
3638                                         while ((c = get_char ()) != -1) {
3639                                                 if (col == 1) {
3640                                                         directive_expected = true;
3641                                                 } else if (!directive_expected) {
3642                                                         // TODO: Implement comment support for disabled code and uncomment this code
3643 //                                                      if (c == '#') {
3644 //                                                              Eror_WrongPreprocessorLocation ();
3645 //                                                              return Token.ERROR;
3646 //                                                      }
3647                                                         continue;
3648                                                 }
3649
3650                                                 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' || c == UnicodeLS || c == UnicodePS)
3651                                                         continue;
3652
3653                                                 if (c == '#') {
3654                                                         if (ParsePreprocessingDirective (false))
3655                                                                 break;
3656                                                 }
3657                                                 directive_expected = false;
3658                                         }
3659
3660                                         if (c != -1) {
3661                                                 tokens_seen = false;
3662                                                 continue;
3663                                         }
3664
3665                                         return Token.EOF;
3666
3667                                 case '"':
3668                                         return consume_string (false);
3669
3670                                 case '\'':
3671                                         return TokenizeBackslash ();
3672
3673                                 case '@':
3674                                         c = get_char ();
3675                                         if (c == '"') {
3676                                                 tokens_seen = true;
3677                                                 return consume_string (true);
3678                                         }
3679
3680                                         if (is_identifier_start_character (c)){
3681                                                 return consume_identifier (c, true);
3682                                         }
3683
3684                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3685                                         return Token.ERROR;
3686
3687                                 case EvalStatementParserCharacter:
3688                                         return Token.EVAL_STATEMENT_PARSER;
3689                                 case EvalCompilationUnitParserCharacter:
3690                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3691                                 case EvalUsingDeclarationsParserCharacter:
3692                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3693                                 case DocumentationXref:
3694                                         return Token.DOC_SEE;
3695                                 }
3696
3697                                 if (is_identifier_start_character (c)) {
3698                                         tokens_seen = true;
3699                                         return consume_identifier (c);
3700                                 }
3701
3702                                 if (char.IsWhiteSpace ((char) c))
3703                                         continue;
3704
3705                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3706                         }
3707
3708                         if (CompleteOnEOF){
3709                                 if (generated)
3710                                         return Token.COMPLETE_COMPLETION;
3711
3712                                 generated = true;
3713                                 return Token.GENERATE_COMPLETION;
3714                         }
3715
3716
3717                         return Token.EOF;
3718                 }
3719
3720                 int TokenizeBackslash ()
3721                 {
3722 #if FULL_AST
3723                         int read_start = reader.Position;
3724 #endif
3725                         Location start_location = Location;
3726                         int c = get_char ();
3727                         tokens_seen = true;
3728                         if (c == '\'') {
3729                                 val = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3730                                 Report.Error (1011, start_location, "Empty character literal");
3731                                 return Token.LITERAL;
3732                         }
3733
3734                         if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
3735                                 Report.Error (1010, start_location, "Newline in constant");
3736                                 return Token.ERROR;
3737                         }
3738
3739                         int d;
3740                         c = escape (c, out d);
3741                         if (c == -1)
3742                                 return Token.ERROR;
3743                         if (d != 0)
3744                                 throw new NotImplementedException ();
3745
3746                         ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3747                         val = res;
3748                         c = get_char ();
3749
3750                         if (c != '\'') {
3751                                 Report.Error (1012, start_location, "Too many characters in character literal");
3752
3753                                 // Try to recover, read until newline or next "'"
3754                                 while ((c = get_char ()) != -1) {
3755                                         if (c == '\n' || c == '\'' || c == UnicodeLS || c == UnicodePS)
3756                                                 break;
3757                                 }
3758                         }
3759
3760 #if FULL_AST
3761                         res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position);
3762 #endif
3763
3764                         return Token.LITERAL;
3765                 }
3766
3767                 int TokenizeLessThan ()
3768                 {
3769                         int d;
3770
3771                         // Save current position and parse next token.
3772                         PushPosition ();
3773                         int generic_dimension = 0;
3774                         if (parse_less_than (ref generic_dimension)) {
3775                                 if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) {
3776                                         d = Token.OP_GENERICS_LT_DECL;
3777                                 } else {
3778                                         if (generic_dimension > 0) {
3779                                                 val = generic_dimension;
3780                                                 DiscardPosition ();
3781                                                 return Token.GENERIC_DIMENSION;
3782                                         }
3783
3784                                         d = Token.OP_GENERICS_LT;
3785                                 }
3786                                 PopPosition ();
3787                                 return d;
3788                         }
3789
3790                         PopPosition ();
3791                         parsing_generic_less_than = 0;
3792
3793                         d = peek_char ();
3794                         if (d == '<') {
3795                                 get_char ();
3796                                 d = peek_char ();
3797
3798                                 if (d == '=') {
3799                                         get_char ();
3800                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3801                                 }
3802                                 return Token.OP_SHIFT_LEFT;
3803                         }
3804
3805                         if (d == '=') {
3806                                 get_char ();
3807                                 return Token.OP_LE;
3808                         }
3809                         return Token.OP_LT;
3810                 }
3811
3812                 //
3813                 // Handles one line xml comment
3814                 //
3815                 private void handle_one_line_xml_comment ()
3816                 {
3817                         int c;
3818                         while ((c = peek_char ()) == ' ')
3819                                 get_char (); // skip heading whitespaces.
3820                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3821                                 xml_comment_buffer.Append ((char) get_char ());
3822                         }
3823                         if (c == '\r' || c == '\n')
3824                                 xml_comment_buffer.Append (Environment.NewLine);
3825                 }
3826
3827                 //
3828                 // Remove heading "*" in Javadoc-like xml documentation.
3829                 //
3830                 private void update_formatted_doc_comment (int current_comment_start)
3831                 {
3832                         int length = xml_comment_buffer.Length - current_comment_start;
3833                         string [] lines = xml_comment_buffer.ToString (
3834                                 current_comment_start,
3835                                 length).Replace ("\r", "").Split ('\n');
3836
3837                         // The first line starts with /**, thus it is not target
3838                         // for the format check.
3839                         for (int i = 1; i < lines.Length; i++) {
3840                                 string s = lines [i];
3841                                 int idx = s.IndexOf ('*');
3842                                 string head = null;
3843                                 if (idx < 0) {
3844                                         if (i < lines.Length - 1)
3845                                                 return;
3846                                         head = s;
3847                                 } else
3848                                         head = s.Substring (0, idx);
3849                                 foreach (char c in head)
3850                                         if (c != ' ')
3851                                                 return;
3852                                 lines [i] = s.Substring (idx + 1);
3853                         }
3854                         xml_comment_buffer.Remove (current_comment_start, length);
3855                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3856                 }
3857
3858                 //
3859                 // Checks if there was incorrect doc comments and raise
3860                 // warnings.
3861                 //
3862                 public void check_incorrect_doc_comment ()
3863                 {
3864                         if (xml_comment_buffer.Length > 0)
3865                                 WarningMisplacedComment (Location);
3866                 }
3867
3868                 //
3869                 // Consumes the saved xml comment lines (if any)
3870                 // as for current target member or type.
3871                 //
3872                 public string consume_doc_comment ()
3873                 {
3874                         if (xml_comment_buffer.Length > 0) {
3875                                 string ret = xml_comment_buffer.ToString ();
3876                                 reset_doc_comment ();
3877                                 return ret;
3878                         }
3879                         return null;
3880                 }
3881
3882                 void reset_doc_comment ()
3883                 {
3884                         xml_comment_buffer.Length = 0;
3885                 }
3886
3887                 public void cleanup ()
3888                 {
3889                         if (ifstack != null && ifstack.Count >= 1) {
3890                                 int state = ifstack.Pop ();
3891                                 if ((state & REGION) != 0)
3892                                         Report.Error (1038, Location, "#endregion directive expected");
3893                                 else
3894                                         Report.Error (1027, Location, "Expected `#endif' directive");
3895                         }
3896                 }
3897         }
3898
3899         //
3900         // Indicates whether it accepts XML documentation or not.
3901         //
3902         public enum XmlCommentState {
3903                 // comment is allowed in this state.
3904                 Allowed,
3905                 // comment is not allowed in this state.
3906                 NotAllowed,
3907                 // once comments appeared when it is NotAllowed, then the
3908                 // state is changed to it, until the state is changed to
3909                 // .Allowed.
3910                 Error
3911         }
3912 }
3913