mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 //
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20
  21 namespace Mono.CSharp
  22 {
  23         /// <summary>
  24         ///    Tokenizer for C# source code.
  25         /// </summary>
  26
  27         public class Tokenizer : yyParser.yyInput
  28         {
  29                 class KeywordEntry<T>
  30                 {
  31                         public readonly T Token;
  32                         public KeywordEntry<T> Next;
  33                         public readonly char[] Value;
  34
  35                         public KeywordEntry (string value, T token)
  36                         {
  37                                 this.Value = value.ToCharArray ();
  38                                 this.Token = token;
  39                         }
  40                 }
  41
  42                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  43                 {
  44                         readonly int length;
  45
  46                         public IdentifiersComparer (int length)
  47                         {
  48                                 this.length = length;
  49                         }
  50
  51                         public bool Equals (char[] x, char[] y)
  52                         {
  53                                 for (int i = 0; i < length; ++i)
  54                                         if (x [i] != y [i])
  55                                                 return false;
  56
  57                                 return true;
  58                         }
  59
  60                         public int GetHashCode (char[] obj)
  61                         {
  62                                 int h = 0;
  63                                 for (int i = 0; i < length; ++i)
  64                                         h = (h << 5) - h + obj [i];
  65
  66                                 return h;
  67                         }
  68                 }
  69
  70                 //
  71                 // This class has to be used in the parser only, it reuses token
  72                 // details after each parse
  73                 //
  74                 public class LocatedToken
  75                 {
  76                         int row, column;
  77                         string value;
  78
  79                         static LocatedToken[] buffer;
  80                         static int pos;
  81
  82                         private LocatedToken ()
  83                         {
  84                         }
  85
  86                         public static LocatedToken Create (int row, int column)
  87                         {
  88                                 return Create (null, row, column);
  89                         }
  90
  91                         public static LocatedToken Create (string value, int row, int column)
  92                         {
  93                                 //
  94                                 // TODO: I am not very happy about the logic but it's the best
  95                                 // what I could come up with for now.
  96                                 // Ideally we should be using just tiny buffer (256 elements) which
  97                                 // is enough to hold all details for currect stack and recycle elements
  98                                 // poped from the stack but there is a trick needed to recycle
  99                                 // them properly.
 100                                 //
 101                                 LocatedToken entry;
 102                                 if (pos >= buffer.Length) {
 103                                         entry = new LocatedToken ();
 104                                 } else {
 105                                         entry = buffer [pos];
 106                                         if (entry == null) {
 107                                                 entry = new LocatedToken ();
 108                                                 buffer [pos] = entry;
 109                                         }
 110
 111                                         ++pos;
 112                                 }
 113                                 entry.value = value;
 114                                 entry.row = row;
 115                                 entry.column = column;
 116                                 return entry;
 117                         }
 118
 119                         //
 120                         // Used for token not required by expression evaluator
 121                         //
 122                         [Conditional ("FULL_AST")]
 123                         public static void CreateOptional (int row, int col, ref object token)
 124                         {
 125                                 token = Create (row, col);
 126                         }
 127
 128                         public static void Initialize ()
 129                         {
 130                                 if (buffer == null)
 131                                         buffer = new LocatedToken [10000];
 132                                 pos = 0;
 133                         }
 134
 135                         public Location Location {
 136                                 get { return new Location (row, column); }
 137                         }
 138
 139                         public string Value {
 140                                 get { return value; }
 141                         }
 142                 }
 143
 144                 enum PreprocessorDirective
 145                 {
 146                         Invalid = 0,
 147
 148                         Region = 1,
 149                         Endregion = 2,
 150                         If = 3 | RequiresArgument,
 151                         Endif = 4,
 152                         Elif = 5 | RequiresArgument,
 153                         Else = 6,
 154                         Define = 7 | RequiresArgument,
 155                         Undef = 8 | RequiresArgument,
 156                         Error = 9,
 157                         Warning = 10,
 158                         Pragma = 11 | CustomArgumentsParsing,
 159                         Line = 12,
 160
 161                         CustomArgumentsParsing = 1 << 10,
 162                         RequiresArgument = 1 << 11
 163                 }
 164
 165                 SeekableStreamReader reader;
 166                 SourceFile ref_name;
 167                 CompilationUnit file_name;
 168                 CompilerContext context;
 169                 bool hidden = false;
 170                 int ref_line = 1;
 171                 int line = 1;
 172                 int col = 0;
 173                 int previous_col;
 174                 int current_token;
 175                 int tab_size;
 176                 bool handle_get_set = false;
 177                 bool handle_remove_add = false;
 178                 bool handle_where = false;
 179                 bool handle_typeof = false;
 180                 bool lambda_arguments_parsing;
 181                 Location current_comment_location = Location.Null;
 182                 List<Location> escaped_identifiers;
 183                 int parsing_generic_less_than;
 184
 185                 //
 186                 // Used mainly for parser optimizations. Some expressions for instance
 187                 // can appear only in block (including initializer, base initializer)
 188                 // scope only
 189                 //
 190                 public int parsing_block;
 191                 internal bool query_parsing;
 192
 193                 //
 194                 // When parsing type only, useful for ambiguous nullable types
 195                 //
 196                 public int parsing_type;
 197
 198                 //
 199                 // Set when parsing generic declaration (type or method header)
 200                 //
 201                 public bool parsing_generic_declaration;
 202
 203                 //
 204                 // The value indicates that we have not reach any declaration or
 205                 // namespace yet
 206                 //
 207                 public int parsing_declaration;
 208
 209                 //
 210                 // The special character to inject on streams to trigger the EXPRESSION_PARSE
 211                 // token to be returned.   It just happens to be a Unicode character that
 212                 // would never be part of a program (can not be an identifier).
 213                 //
 214                 // This character is only tested just before the tokenizer is about to report
 215                 // an error;   So on the regular operation mode, this addition will have no
 216                 // impact on the tokenizer's performance.
 217                 //
 218
 219                 public const int EvalStatementParserCharacter = 0x2190;   // Unicode Left Arrow
 220                 public const int EvalCompilationUnitParserCharacter = 0x2191;  // Unicode Arrow
 221                 public const int EvalUsingDeclarationsParserCharacter = 0x2192;  // Unicode Arrow
 222
 223                 //
 224                 // XML documentation buffer. The save point is used to divide
 225                 // comments on types and comments on members.
 226                 //
 227                 StringBuilder xml_comment_buffer;
 228
 229                 //
 230                 // See comment on XmlCommentState enumeration.
 231                 //
 232                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 233
 234                 //
 235                 // Whether tokens have been seen on this line
 236                 //
 237                 bool tokens_seen = false;
 238
 239                 //
 240                 // Set to true once the GENERATE_COMPLETION token has bee
 241                 // returned.   This helps produce one GENERATE_COMPLETION,
 242                 // as many COMPLETE_COMPLETION as necessary to complete the
 243                 // AST tree and one final EOF.
 244                 //
 245                 bool generated;
 246
 247                 //
 248                 // Whether a token has been seen on the file
 249                 // This is needed because `define' is not allowed to be used
 250                 // after a token has been seen.
 251                 //
 252                 bool any_token_seen = false;
 253
 254                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 255
 256                 public bool PropertyParsing {
 257                         get { return handle_get_set; }
 258                         set { handle_get_set = value; }
 259                 }
 260
 261                 public bool EventParsing {
 262                         get { return handle_remove_add; }
 263                         set { handle_remove_add = value; }
 264                 }
 265
 266                 public bool ConstraintsParsing {
 267                         get { return handle_where; }
 268                         set { handle_where = value; }
 269                 }
 270
 271                 public bool TypeOfParsing {
 272                         get { return handle_typeof; }
 273                         set { handle_typeof = value; }
 274                 }
 275
 276                 public int TabSize {
 277                         get { return tab_size; }
 278                         set { tab_size = value; }
 279                 }
 280
 281                 public XmlCommentState doc_state {
 282                         get { return xml_doc_state; }
 283                         set {
 284                                 if (value == XmlCommentState.Allowed) {
 285                                         check_incorrect_doc_comment ();
 286                                         reset_doc_comment ();
 287                                 }
 288                                 xml_doc_state = value;
 289                         }
 290                 }
 291
 292                 //
 293                 // This is used to trigger completion generation on the parser
 294                 public bool CompleteOnEOF;
 295
 296                 void AddEscapedIdentifier (Location loc)
 297                 {
 298                         if (escaped_identifiers == null)
 299                                 escaped_identifiers = new List<Location> ();
 300
 301                         escaped_identifiers.Add (loc);
 302                 }
 303
 304                 public bool IsEscapedIdentifier (MemberName name)
 305                 {
 306                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 307                 }
 308
 309                 //
 310                 // Class variables
 311                 //
 312                 static KeywordEntry<int>[][] keywords;
 313                 static KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 314                 static Dictionary<string, object> keyword_strings;              // TODO: HashSet
 315                 static NumberStyles styles;
 316                 static NumberFormatInfo csharp_format_info;
 317
 318                 // Pragma arguments
 319                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 320                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 321                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 322                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 323
 324                 //
 325                 // Values for the associated token returned
 326                 //
 327                 internal int putback_char;      // Used by repl only
 328                 object val;
 329
 330                 //
 331                 // Pre-processor
 332                 //
 333                 const int TAKING        = 1;
 334                 const int ELSE_SEEN     = 4;
 335                 const int PARENT_TAKING = 8;
 336                 const int REGION        = 16;
 337
 338                 //
 339                 // pre-processor if stack state:
 340                 //
 341                 Stack<int> ifstack;
 342
 343                 static System.Text.StringBuilder string_builder;
 344
 345                 const int max_id_size = 512;
 346                 static char [] id_builder = new char [max_id_size];
 347
 348                 public static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 349
 350                 const int max_number_size = 512;
 351                 static char [] number_builder = new char [max_number_size];
 352                 static int number_pos;
 353
 354                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
 355
 356                 public int Line {
 357                         get {
 358                                 return ref_line;
 359                         }
 360                 }
 361
 362                 //
 363                 // This is used when the tokenizer needs to save
 364                 // the current position as it needs to do some parsing
 365                 // on its own to deamiguate a token in behalf of the
 366                 // parser.
 367                 //
 368                 Stack<Position> position_stack = new Stack<Position> (2);
 369
 370                 class Position {
 371                         public int position;
 372                         public int line;
 373                         public int ref_line;
 374                         public int col;
 375                         public bool hidden;
 376                         public int putback_char;
 377                         public int previous_col;
 378                         public Stack<int> ifstack;
 379                         public int parsing_generic_less_than;
 380                         public int current_token;
 381                         public object val;
 382
 383                         public Position (Tokenizer t)
 384                         {
 385                                 position = t.reader.Position;
 386                                 line = t.line;
 387                                 ref_line = t.ref_line;
 388                                 col = t.col;
 389                                 hidden = t.hidden;
 390                                 putback_char = t.putback_char;
 391                                 previous_col = t.previous_col;
 392                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 393                                         // There is no simple way to clone Stack<T> all
 394                                         // methods reverse the order
 395                                         var clone = t.ifstack.ToArray ();
 396                                         Array.Reverse (clone);
 397                                         ifstack = new Stack<int> (clone);
 398                                 }
 399                                 parsing_generic_less_than = t.parsing_generic_less_than;
 400                                 current_token = t.current_token;
 401                                 val = t.val;
 402                         }
 403                 }
 404
 405                 public void PushPosition ()
 406                 {
 407                         position_stack.Push (new Position (this));
 408                 }
 409
 410                 public void PopPosition ()
 411                 {
 412                         Position p = position_stack.Pop ();
 413
 414                         reader.Position = p.position;
 415                         ref_line = p.ref_line;
 416                         line = p.line;
 417                         col = p.col;
 418                         hidden = p.hidden;
 419                         putback_char = p.putback_char;
 420                         previous_col = p.previous_col;
 421                         ifstack = p.ifstack;
 422                         parsing_generic_less_than = p.parsing_generic_less_than;
 423                         current_token = p.current_token;
 424                         val = p.val;
 425                 }
 426
 427                 // Do not reset the position, ignore it.
 428                 public void DiscardPosition ()
 429                 {
 430                         position_stack.Pop ();
 431                 }
 432
 433                 static void AddKeyword (string kw, int token)
 434                 {
 435                         keyword_strings.Add (kw, null);
 436
 437                         AddKeyword (keywords, kw, token);
 438                 }
 439
 440                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 441                 {
 442                         AddKeyword (keywords_preprocessor, kw, directive);
 443                 }
 444
 445                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 446                 {
 447                         int length = kw.Length;
 448                         if (keywords[length] == null) {
 449                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 450                         }
 451
 452                         int char_index = kw[0] - '_';
 453                         var kwe = keywords[length][char_index];
 454                         if (kwe == null) {
 455                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 456                                 return;
 457                         }
 458
 459                         while (kwe.Next != null) {
 460                                 kwe = kwe.Next;
 461                         }
 462
 463                         kwe.Next = new KeywordEntry<T> (kw, token);
 464                 }
 465
 466                 static void InitTokens ()
 467                 {
 468                         keyword_strings = new Dictionary<string, object> ();
 469
 470                         // 11 is the length of the longest keyword for now
 471                         keywords = new KeywordEntry<int> [11] [];
 472
 473                         AddKeyword ("__arglist", Token.ARGLIST);
 474                         AddKeyword ("abstract", Token.ABSTRACT);
 475                         AddKeyword ("as", Token.AS);
 476                         AddKeyword ("add", Token.ADD);
 477                         AddKeyword ("base", Token.BASE);
 478                         AddKeyword ("bool", Token.BOOL);
 479                         AddKeyword ("break", Token.BREAK);
 480                         AddKeyword ("byte", Token.BYTE);
 481                         AddKeyword ("case", Token.CASE);
 482                         AddKeyword ("catch", Token.CATCH);
 483                         AddKeyword ("char", Token.CHAR);
 484                         AddKeyword ("checked", Token.CHECKED);
 485                         AddKeyword ("class", Token.CLASS);
 486                         AddKeyword ("const", Token.CONST);
 487                         AddKeyword ("continue", Token.CONTINUE);
 488                         AddKeyword ("decimal", Token.DECIMAL);
 489                         AddKeyword ("default", Token.DEFAULT);
 490                         AddKeyword ("delegate", Token.DELEGATE);
 491                         AddKeyword ("do", Token.DO);
 492                         AddKeyword ("double", Token.DOUBLE);
 493                         AddKeyword ("else", Token.ELSE);
 494                         AddKeyword ("enum", Token.ENUM);
 495                         AddKeyword ("event", Token.EVENT);
 496                         AddKeyword ("explicit", Token.EXPLICIT);
 497                         AddKeyword ("extern", Token.EXTERN);
 498                         AddKeyword ("false", Token.FALSE);
 499                         AddKeyword ("finally", Token.FINALLY);
 500                         AddKeyword ("fixed", Token.FIXED);
 501                         AddKeyword ("float", Token.FLOAT);
 502                         AddKeyword ("for", Token.FOR);
 503                         AddKeyword ("foreach", Token.FOREACH);
 504                         AddKeyword ("goto", Token.GOTO);
 505                         AddKeyword ("get", Token.GET);
 506                         AddKeyword ("if", Token.IF);
 507                         AddKeyword ("implicit", Token.IMPLICIT);
 508                         AddKeyword ("in", Token.IN);
 509                         AddKeyword ("int", Token.INT);
 510                         AddKeyword ("interface", Token.INTERFACE);
 511                         AddKeyword ("internal", Token.INTERNAL);
 512                         AddKeyword ("is", Token.IS);
 513                         AddKeyword ("lock", Token.LOCK);
 514                         AddKeyword ("long", Token.LONG);
 515                         AddKeyword ("namespace", Token.NAMESPACE);
 516                         AddKeyword ("new", Token.NEW);
 517                         AddKeyword ("null", Token.NULL);
 518                         AddKeyword ("object", Token.OBJECT);
 519                         AddKeyword ("operator", Token.OPERATOR);
 520                         AddKeyword ("out", Token.OUT);
 521                         AddKeyword ("override", Token.OVERRIDE);
 522                         AddKeyword ("params", Token.PARAMS);
 523                         AddKeyword ("private", Token.PRIVATE);
 524                         AddKeyword ("protected", Token.PROTECTED);
 525                         AddKeyword ("public", Token.PUBLIC);
 526                         AddKeyword ("readonly", Token.READONLY);
 527                         AddKeyword ("ref", Token.REF);
 528                         AddKeyword ("remove", Token.REMOVE);
 529                         AddKeyword ("return", Token.RETURN);
 530                         AddKeyword ("sbyte", Token.SBYTE);
 531                         AddKeyword ("sealed", Token.SEALED);
 532                         AddKeyword ("set", Token.SET);
 533                         AddKeyword ("short", Token.SHORT);
 534                         AddKeyword ("sizeof", Token.SIZEOF);
 535                         AddKeyword ("stackalloc", Token.STACKALLOC);
 536                         AddKeyword ("static", Token.STATIC);
 537                         AddKeyword ("string", Token.STRING);
 538                         AddKeyword ("struct", Token.STRUCT);
 539                         AddKeyword ("switch", Token.SWITCH);
 540                         AddKeyword ("this", Token.THIS);
 541                         AddKeyword ("throw", Token.THROW);
 542                         AddKeyword ("true", Token.TRUE);
 543                         AddKeyword ("try", Token.TRY);
 544                         AddKeyword ("typeof", Token.TYPEOF);
 545                         AddKeyword ("uint", Token.UINT);
 546                         AddKeyword ("ulong", Token.ULONG);
 547                         AddKeyword ("unchecked", Token.UNCHECKED);
 548                         AddKeyword ("unsafe", Token.UNSAFE);
 549                         AddKeyword ("ushort", Token.USHORT);
 550                         AddKeyword ("using", Token.USING);
 551                         AddKeyword ("virtual", Token.VIRTUAL);
 552                         AddKeyword ("void", Token.VOID);
 553                         AddKeyword ("volatile", Token.VOLATILE);
 554                         AddKeyword ("while", Token.WHILE);
 555                         AddKeyword ("partial", Token.PARTIAL);
 556                         AddKeyword ("where", Token.WHERE);
 557                         AddKeyword ("async", Token.ASYNC);
 558
 559                         // LINQ keywords
 560                         AddKeyword ("from", Token.FROM);
 561                         AddKeyword ("join", Token.JOIN);
 562                         AddKeyword ("on", Token.ON);
 563                         AddKeyword ("equals", Token.EQUALS);
 564                         AddKeyword ("select", Token.SELECT);
 565                         AddKeyword ("group", Token.GROUP);
 566                         AddKeyword ("by", Token.BY);
 567                         AddKeyword ("let", Token.LET);
 568                         AddKeyword ("orderby", Token.ORDERBY);
 569                         AddKeyword ("ascending", Token.ASCENDING);
 570                         AddKeyword ("descending", Token.DESCENDING);
 571                         AddKeyword ("into", Token.INTO);
 572
 573                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 574
 575                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 576                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 577                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 578                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 579                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 580                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 581                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 582                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 583                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 584                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 585                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 586                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 587                 }
 588
 589                 //
 590                 // Class initializer
 591                 //
 592                 static Tokenizer ()
 593                 {
 594                         InitTokens ();
 595                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 596                         styles = NumberStyles.Float;
 597
 598                         string_builder = new System.Text.StringBuilder ();
 599                 }
 600
 601                 int GetKeyword (char[] id, int id_len)
 602                 {
 603                         //
 604                         // Keywords are stored in an array of arrays grouped by their
 605                         // length and then by the first character
 606                         //
 607                         if (id_len >= keywords.Length || keywords [id_len] == null)
 608                                 return -1;
 609
 610                         int first_index = id [0] - '_';
 611                         if (first_index > 'z' - '_')
 612                                 return -1;
 613
 614                         var kwe = keywords [id_len] [first_index];
 615                         if (kwe == null)
 616                                 return -1;
 617
 618                         int res;
 619                         do {
 620                                 res = kwe.Token;
 621                                 for (int i = 1; i < id_len; ++i) {
 622                                         if (id [i] != kwe.Value [i]) {
 623                                                 res = 0;
 624                                                 kwe = kwe.Next;
 625                                                 break;
 626                                         }
 627                                 }
 628                         } while (res == 0 && kwe != null);
 629
 630                         if (res == 0)
 631                                 return -1;
 632
 633                         int next_token;
 634                         switch (res) {
 635                         case Token.GET:
 636                         case Token.SET:
 637                                 if (!handle_get_set)
 638                                         res = -1;
 639                                 break;
 640                         case Token.REMOVE:
 641                         case Token.ADD:
 642                                 if (!handle_remove_add)
 643                                         res = -1;
 644                                 break;
 645                         case Token.EXTERN:
 646                                 if (parsing_declaration == 0)
 647                                         res = Token.EXTERN_ALIAS;
 648                                 break;
 649                         case Token.DEFAULT:
 650                                 if (peek_token () == Token.COLON) {
 651                                         token ();
 652                                         res = Token.DEFAULT_COLON;
 653                                 }
 654                                 break;
 655                         case Token.WHERE:
 656                                 if (!handle_where && !query_parsing)
 657                                         res = -1;
 658                                 break;
 659                         case Token.FROM:
 660                                 //
 661                                 // A query expression is any expression that starts with `from identifier'
 662                                 // followed by any token except ; , =
 663                                 //
 664                                 if (!query_parsing) {
 665                                         if (lambda_arguments_parsing) {
 666                                                 res = -1;
 667                                                 break;
 668                                         }
 669
 670                                         PushPosition ();
 671                                         // HACK: to disable generics micro-parser, because PushPosition does not
 672                                         // store identifiers array
 673                                         parsing_generic_less_than = 1;
 674                                         switch (xtoken ()) {
 675                                         case Token.IDENTIFIER:
 676                                         case Token.INT:
 677                                         case Token.BOOL:
 678                                         case Token.BYTE:
 679                                         case Token.CHAR:
 680                                         case Token.DECIMAL:
 681                                         case Token.FLOAT:
 682                                         case Token.LONG:
 683                                         case Token.OBJECT:
 684                                         case Token.STRING:
 685                                         case Token.UINT:
 686                                         case Token.ULONG:
 687                                                 next_token = xtoken ();
 688                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 689                                                         goto default;
 690
 691                                                 res = Token.FROM_FIRST;
 692                                                 query_parsing = true;
 693                                                 if (RootContext.Version <= LanguageVersion.ISO_2)
 694                                                         Report.FeatureIsNotAvailable (Location, "query expressions");
 695                                                 break;
 696                                         case Token.VOID:
 697                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 698                                                 break;
 699                                         default:
 700                                                 PopPosition ();
 701                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 702                                                 // which has been overwritten before we grabbed the identifier
 703                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 704                                                 return -1;
 705                                         }
 706                                         PopPosition ();
 707                                 }
 708                                 break;
 709                         case Token.JOIN:
 710                         case Token.ON:
 711                         case Token.EQUALS:
 712                         case Token.SELECT:
 713                         case Token.GROUP:
 714                         case Token.BY:
 715                         case Token.LET:
 716                         case Token.ORDERBY:
 717                         case Token.ASCENDING:
 718                         case Token.DESCENDING:
 719                         case Token.INTO:
 720                                 if (!query_parsing)
 721                                         res = -1;
 722                                 break;
 723
 724                         case Token.USING:
 725                         case Token.NAMESPACE:
 726                                 // TODO: some explanation needed
 727                                 check_incorrect_doc_comment ();
 728                                 break;
 729
 730                         case Token.PARTIAL:
 731                                 if (parsing_block > 0) {
 732                                         res = -1;
 733                                         break;
 734                                 }
 735
 736                                 // Save current position and parse next token.
 737                                 PushPosition ();
 738
 739                                 next_token = token ();
 740                                 bool ok = (next_token == Token.CLASS) ||
 741                                         (next_token == Token.STRUCT) ||
 742                                         (next_token == Token.INTERFACE) ||
 743                                         (next_token == Token.VOID);
 744
 745                                 PopPosition ();
 746
 747                                 if (ok) {
 748                                         if (next_token == Token.VOID) {
 749                                                 if (RootContext.Version == LanguageVersion.ISO_1 ||
 750                                                     RootContext.Version == LanguageVersion.ISO_2)
 751                                                         Report.FeatureIsNotAvailable (Location, "partial methods");
 752                                         } else if (RootContext.Version == LanguageVersion.ISO_1)
 753                                                 Report.FeatureIsNotAvailable (Location, "partial types");
 754
 755                                         return res;
 756                                 }
 757
 758                                 if (next_token < Token.LAST_KEYWORD) {
 759                                         Report.Error (267, Location,
 760                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 761                                         return token ();
 762                                 }
 763
 764                                 res = -1;
 765                                 break;
 766
 767                         case Token.ASYNC:
 768                                 if (parsing_block > 0 || RootContext.Version != LanguageVersion.Future) {
 769                                         res = -1;
 770                                         break;
 771                                 }
 772                                 break;
 773                         }
 774
 775                         return res;
 776                 }
 777
 778                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 779                 {
 780                         //
 781                         // Keywords are stored in an array of arrays grouped by their
 782                         // length and then by the first character
 783                         //
 784                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 785                                 return PreprocessorDirective.Invalid;
 786
 787                         int first_index = id[0] - '_';
 788                         if (first_index > 'z' - '_')
 789                                 return PreprocessorDirective.Invalid;
 790
 791                         var kwe = keywords_preprocessor[id_len][first_index];
 792                         if (kwe == null)
 793                                 return PreprocessorDirective.Invalid;
 794
 795                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 796                         do {
 797                                 res = kwe.Token;
 798                                 for (int i = 1; i < id_len; ++i) {
 799                                         if (id[i] != kwe.Value[i]) {
 800                                                 res = 0;
 801                                                 kwe = kwe.Next;
 802                                                 break;
 803                                         }
 804                                 }
 805                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 806
 807                         return res;
 808                 }
 809
 810                 public Location Location {
 811                         get {
 812                                 return new Location (ref_line, hidden ? -1 : col);
 813                         }
 814                 }
 815
 816                 public Tokenizer (SeekableStreamReader input, CompilationUnit file, CompilerContext ctx)
 817                 {
 818                         this.ref_name = file;
 819                         this.file_name = file;
 820                         this.context = ctx;
 821                         reader = input;
 822
 823                         putback_char = -1;
 824
 825                         xml_comment_buffer = new StringBuilder ();
 826
 827                         if (Environment.OSVersion.Platform == PlatformID.Win32NT)
 828                                 tab_size = 4;
 829                         else
 830                                 tab_size = 8;
 831
 832                         //
 833                         // FIXME: This could be `Location.Push' but we have to
 834                         // find out why the MS compiler allows this
 835                         //
 836                         Mono.CSharp.Location.Push (file, file);
 837                 }
 838
 839                 static bool is_identifier_start_character (int c)
 840                 {
 841                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 842                 }
 843
 844                 static bool is_identifier_part_character (char c)
 845                 {
 846                         if (c >= 'a' && c <= 'z')
 847                                 return true;
 848
 849                         if (c >= 'A' && c <= 'Z')
 850                                 return true;
 851
 852                         if (c == '_' || (c >= '0' && c <= '9'))
 853                                 return true;
 854
 855                         if (c < 0x80)
 856                                 return false;
 857
 858                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 859                 }
 860
 861                 public static bool IsKeyword (string s)
 862                 {
 863                         return keyword_strings.ContainsKey (s);
 864                 }
 865
 866                 //
 867                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 868                 //
 869                 int TokenizeOpenParens ()
 870                 {
 871                         int ptoken;
 872                         current_token = -1;
 873
 874                         int bracket_level = 0;
 875                         bool is_type = false;
 876                         bool can_be_type = false;
 877
 878                         while (true) {
 879                                 ptoken = current_token;
 880                                 token ();
 881
 882                                 switch (current_token) {
 883                                 case Token.CLOSE_PARENS:
 884                                         token ();
 885
 886                                         //
 887                                         // Expression inside parens is lambda, (int i) =>
 888                                         //
 889                                         if (current_token == Token.ARROW)
 890                                                 return Token.OPEN_PARENS_LAMBDA;
 891
 892                                         //
 893                                         // Expression inside parens is single type, (int[])
 894                                         //
 895                                         if (is_type)
 896                                                 return Token.OPEN_PARENS_CAST;
 897
 898                                         //
 899                                         // Expression is possible cast, look at next token, (T)null
 900                                         //
 901                                         if (can_be_type) {
 902                                                 switch (current_token) {
 903                                                 case Token.OPEN_PARENS:
 904                                                 case Token.BANG:
 905                                                 case Token.TILDE:
 906                                                 case Token.IDENTIFIER:
 907                                                 case Token.LITERAL:
 908                                                 case Token.BASE:
 909                                                 case Token.CHECKED:
 910                                                 case Token.DELEGATE:
 911                                                 case Token.FALSE:
 912                                                 case Token.FIXED:
 913                                                 case Token.NEW:
 914                                                 case Token.NULL:
 915                                                 case Token.SIZEOF:
 916                                                 case Token.THIS:
 917                                                 case Token.THROW:
 918                                                 case Token.TRUE:
 919                                                 case Token.TYPEOF:
 920                                                 case Token.UNCHECKED:
 921                                                 case Token.UNSAFE:
 922                                                 case Token.DEFAULT:
 923
 924                                                 //
 925                                                 // These can be part of a member access
 926                                                 //
 927                                                 case Token.INT:
 928                                                 case Token.UINT:
 929                                                 case Token.SHORT:
 930                                                 case Token.USHORT:
 931                                                 case Token.LONG:
 932                                                 case Token.ULONG:
 933                                                 case Token.DOUBLE:
 934                                                 case Token.FLOAT:
 935                                                 case Token.CHAR:
 936                                                 case Token.BYTE:
 937                                                 case Token.DECIMAL:
 938                                                 case Token.BOOL:
 939                                                         return Token.OPEN_PARENS_CAST;
 940                                                 }
 941                                         }
 942                                         return Token.OPEN_PARENS;
 943
 944                                 case Token.DOT:
 945                                 case Token.DOUBLE_COLON:
 946                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
 947                                                 goto default;
 948
 949                                         continue;
 950
 951                                 case Token.IDENTIFIER:
 952                                         switch (ptoken) {
 953                                         case Token.DOT:
 954                                                 if (bracket_level == 0) {
 955                                                         is_type = false;
 956                                                         can_be_type = true;
 957                                                 }
 958
 959                                                 continue;
 960                                         case Token.OP_GENERICS_LT:
 961                                         case Token.COMMA:
 962                                         case Token.DOUBLE_COLON:
 963                                         case -1:
 964                                                 if (bracket_level == 0)
 965                                                         can_be_type = true;
 966                                                 continue;
 967                                         default:
 968                                                 can_be_type = is_type = false;
 969                                                 continue;
 970                                         }
 971
 972                                 case Token.OBJECT:
 973                                 case Token.STRING:
 974                                 case Token.BOOL:
 975                                 case Token.DECIMAL:
 976                                 case Token.FLOAT:
 977                                 case Token.DOUBLE:
 978                                 case Token.SBYTE:
 979                                 case Token.BYTE:
 980                                 case Token.SHORT:
 981                                 case Token.USHORT:
 982                                 case Token.INT:
 983                                 case Token.UINT:
 984                                 case Token.LONG:
 985                                 case Token.ULONG:
 986                                 case Token.CHAR:
 987                                 case Token.VOID:
 988                                         if (bracket_level == 0)
 989                                                 is_type = true;
 990                                         continue;
 991
 992                                 case Token.COMMA:
 993                                         if (bracket_level == 0) {
 994                                                 bracket_level = 100;
 995                                                 can_be_type = is_type = false;
 996                                         }
 997                                         continue;
 998
 999                                 case Token.OP_GENERICS_LT:
1000                                 case Token.OPEN_BRACKET:
1001                                         if (bracket_level++ == 0)
1002                                                 is_type = true;
1003                                         continue;
1004
1005                                 case Token.OP_GENERICS_GT:
1006                                 case Token.CLOSE_BRACKET:
1007                                         --bracket_level;
1008                                         continue;
1009
1010                                 case Token.INTERR_NULLABLE:
1011                                 case Token.STAR:
1012                                         if (bracket_level == 0)
1013                                                 is_type = true;
1014                                         continue;
1015
1016                                 case Token.REF:
1017                                 case Token.OUT:
1018                                         can_be_type = is_type = false;
1019                                         continue;
1020
1021                                 default:
1022                                         return Token.OPEN_PARENS;
1023                                 }
1024                         }
1025                 }
1026
1027                 public static bool IsValidIdentifier (string s)
1028                 {
1029                         if (s == null || s.Length == 0)
1030                                 return false;
1031
1032                         if (!is_identifier_start_character (s [0]))
1033                                 return false;
1034
1035                         for (int i = 1; i < s.Length; i ++)
1036                                 if (! is_identifier_part_character (s [i]))
1037                                         return false;
1038
1039                         return true;
1040                 }
1041
1042                 bool parse_less_than ()
1043                 {
1044                 start:
1045                         int the_token = token ();
1046                         if (the_token == Token.OPEN_BRACKET) {
1047                                 do {
1048                                         the_token = token ();
1049                                 } while (the_token != Token.CLOSE_BRACKET);
1050                                 the_token = token ();
1051                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1052                                 the_token = token ();
1053                         }
1054                         switch (the_token) {
1055                         case Token.IDENTIFIER:
1056                         case Token.OBJECT:
1057                         case Token.STRING:
1058                         case Token.BOOL:
1059                         case Token.DECIMAL:
1060                         case Token.FLOAT:
1061                         case Token.DOUBLE:
1062                         case Token.SBYTE:
1063                         case Token.BYTE:
1064                         case Token.SHORT:
1065                         case Token.USHORT:
1066                         case Token.INT:
1067                         case Token.UINT:
1068                         case Token.LONG:
1069                         case Token.ULONG:
1070                         case Token.CHAR:
1071                         case Token.VOID:
1072                                 break;
1073                         case Token.OP_GENERICS_GT:
1074                                 return true;
1075
1076                         default:
1077                                 return false;
1078                         }
1079                 again:
1080                         the_token = token ();
1081
1082                         if (the_token == Token.OP_GENERICS_GT)
1083                                 return true;
1084                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1085                                 goto start;
1086                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1087                                 goto again;
1088                         else if (the_token == Token.OP_GENERICS_LT) {
1089                                 if (!parse_less_than ())
1090                                         return false;
1091                                 goto again;
1092                         } else if (the_token == Token.OPEN_BRACKET) {
1093                         rank_specifiers:
1094                                 the_token = token ();
1095                                 if (the_token == Token.CLOSE_BRACKET)
1096                                         goto again;
1097                                 else if (the_token == Token.COMMA)
1098                                         goto rank_specifiers;
1099                                 return false;
1100                         }
1101
1102                         return false;
1103                 }
1104
1105                 bool parse_generic_dimension (out int dimension)
1106                 {
1107                         dimension = 1;
1108
1109                 again:
1110                         int the_token = token ();
1111                         if (the_token == Token.OP_GENERICS_GT)
1112                                 return true;
1113                         else if (the_token == Token.COMMA) {
1114                                 dimension++;
1115                                 goto again;
1116                         }
1117
1118                         return false;
1119                 }
1120
1121                 public int peek_token ()
1122                 {
1123                         int the_token;
1124
1125                         PushPosition ();
1126                         the_token = token ();
1127                         PopPosition ();
1128
1129                         return the_token;
1130                 }
1131
1132                 //
1133                 // Tonizes `?' using custom disambiguous rules to return one
1134                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1135                 //
1136                 // Tricky expression look like:
1137                 //
1138                 // Foo ? a = x ? b : c;
1139                 //
1140                 int TokenizePossibleNullableType ()
1141                 {
1142                         if (parsing_block == 0 || parsing_type > 0)
1143                                 return Token.INTERR_NULLABLE;
1144
1145                         int d = peek_char ();
1146                         if (d == '?') {
1147                                 get_char ();
1148                                 return Token.OP_COALESCING;
1149                         }
1150
1151                         switch (current_token) {
1152                         case Token.CLOSE_PARENS:
1153                         case Token.TRUE:
1154                         case Token.FALSE:
1155                         case Token.NULL:
1156                         case Token.LITERAL:
1157                                 return Token.INTERR;
1158                         }
1159
1160                         if (d != ' ') {
1161                                 if (d == ',' || d == ';' || d == '>')
1162                                         return Token.INTERR_NULLABLE;
1163                                 if (d == '*' || (d >= '0' && d <= '9'))
1164                                         return Token.INTERR;
1165                         }
1166
1167                         PushPosition ();
1168                         current_token = Token.NONE;
1169                         int next_token;
1170                         switch (xtoken ()) {
1171                         case Token.LITERAL:
1172                         case Token.TRUE:
1173                         case Token.FALSE:
1174                         case Token.NULL:
1175                         case Token.THIS:
1176                         case Token.NEW:
1177                                 next_token = Token.INTERR;
1178                                 break;
1179
1180                         case Token.SEMICOLON:
1181                         case Token.COMMA:
1182                         case Token.CLOSE_PARENS:
1183                         case Token.OPEN_BRACKET:
1184                         case Token.OP_GENERICS_GT:
1185                                 next_token = Token.INTERR_NULLABLE;
1186                                 break;
1187
1188                         default:
1189                                 next_token = -1;
1190                                 break;
1191                         }
1192
1193                         if (next_token == -1) {
1194                                 switch (xtoken ()) {
1195                                 case Token.COMMA:
1196                                 case Token.SEMICOLON:
1197                                 case Token.OPEN_BRACE:
1198                                 case Token.CLOSE_PARENS:
1199                                 case Token.IN:
1200                                         next_token = Token.INTERR_NULLABLE;
1201                                         break;
1202
1203                                 case Token.COLON:
1204                                         next_token = Token.INTERR;
1205                                         break;
1206
1207                                 default:
1208                                         int ntoken;
1209                                         int interrs = 1;
1210                                         int colons = 0;
1211                                         //
1212                                         // All shorcuts failed, do it hard way
1213                                         //
1214                                         while ((ntoken = xtoken ()) != Token.EOF) {
1215                                                 if (ntoken == Token.SEMICOLON)
1216                                                         break;
1217
1218                                                 if (ntoken == Token.COLON) {
1219                                                         if (++colons == interrs)
1220                                                                 break;
1221                                                         continue;
1222                                                 }
1223
1224                                                 if (ntoken == Token.INTERR) {
1225                                                         ++interrs;
1226                                                         continue;
1227                                                 }
1228                                         }
1229
1230                                         next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1231                                         break;
1232                                 }
1233                         }
1234
1235                         PopPosition ();
1236                         return next_token;
1237                 }
1238
1239                 bool decimal_digits (int c)
1240                 {
1241                         int d;
1242                         bool seen_digits = false;
1243
1244                         if (c != -1){
1245                                 if (number_pos == max_number_size)
1246                                         Error_NumericConstantTooLong ();
1247                                 number_builder [number_pos++] = (char) c;
1248                         }
1249
1250                         //
1251                         // We use peek_char2, because decimal_digits needs to do a
1252                         // 2-character look-ahead (5.ToString for example).
1253                         //
1254                         while ((d = peek_char2 ()) != -1){
1255                                 if (d >= '0' && d <= '9'){
1256                                         if (number_pos == max_number_size)
1257                                                 Error_NumericConstantTooLong ();
1258                                         number_builder [number_pos++] = (char) d;
1259                                         get_char ();
1260                                         seen_digits = true;
1261                                 } else
1262                                         break;
1263                         }
1264
1265                         return seen_digits;
1266                 }
1267
1268                 static bool is_hex (int e)
1269                 {
1270                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1271                 }
1272
1273                 static TypeCode real_type_suffix (int c)
1274                 {
1275                         switch (c){
1276                         case 'F': case 'f':
1277                                 return TypeCode.Single;
1278                         case 'D': case 'd':
1279                                 return TypeCode.Double;
1280                         case 'M': case 'm':
1281                                 return TypeCode.Decimal;
1282                         default:
1283                                 return TypeCode.Empty;
1284                         }
1285                 }
1286
1287                 int integer_type_suffix (ulong ul, int c)
1288                 {
1289                         bool is_unsigned = false;
1290                         bool is_long = false;
1291
1292                         if (c != -1){
1293                                 bool scanning = true;
1294                                 do {
1295                                         switch (c){
1296                                         case 'U': case 'u':
1297                                                 if (is_unsigned)
1298                                                         scanning = false;
1299                                                 is_unsigned = true;
1300                                                 get_char ();
1301                                                 break;
1302
1303                                         case 'l':
1304                                                 if (!is_unsigned){
1305                                                         //
1306                                                         // if we have not seen anything in between
1307                                                         // report this error
1308                                                         //
1309                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1310                                                 }
1311
1312                                                 goto case 'L';
1313
1314                                         case 'L':
1315                                                 if (is_long)
1316                                                         scanning = false;
1317                                                 is_long = true;
1318                                                 get_char ();
1319                                                 break;
1320
1321                                         default:
1322                                                 scanning = false;
1323                                                 break;
1324                                         }
1325                                         c = peek_char ();
1326                                 } while (scanning);
1327                         }
1328
1329                         if (is_long && is_unsigned){
1330                                 val = new ULongLiteral (ul, Location);
1331                                 return Token.LITERAL;
1332                         }
1333
1334                         if (is_unsigned){
1335                                 // uint if possible, or ulong else.
1336
1337                                 if ((ul & 0xffffffff00000000) == 0)
1338                                         val = new UIntLiteral ((uint) ul, Location);
1339                                 else
1340                                         val = new ULongLiteral (ul, Location);
1341                         } else if (is_long){
1342                                 // long if possible, ulong otherwise
1343                                 if ((ul & 0x8000000000000000) != 0)
1344                                         val = new ULongLiteral (ul, Location);
1345                                 else
1346                                         val = new LongLiteral ((long) ul, Location);
1347                         } else {
1348                                 // int, uint, long or ulong in that order
1349                                 if ((ul & 0xffffffff00000000) == 0){
1350                                         uint ui = (uint) ul;
1351
1352                                         if ((ui & 0x80000000) != 0)
1353                                                 val = new UIntLiteral (ui, Location);
1354                                         else
1355                                                 val = new IntLiteral ((int) ui, Location);
1356                                 } else {
1357                                         if ((ul & 0x8000000000000000) != 0)
1358                                                 val = new ULongLiteral (ul, Location);
1359                                         else
1360                                                 val = new LongLiteral ((long) ul, Location);
1361                                 }
1362                         }
1363                         return Token.LITERAL;
1364                 }
1365
1366                 //
1367                 // given `c' as the next char in the input decide whether
1368                 // we need to convert to a special type, and then choose
1369                 // the best representation for the integer
1370                 //
1371                 int adjust_int (int c)
1372                 {
1373                         try {
1374                                 if (number_pos > 9){
1375                                         ulong ul = (uint) (number_builder [0] - '0');
1376
1377                                         for (int i = 1; i < number_pos; i++){
1378                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1379                                         }
1380                                         return integer_type_suffix (ul, c);
1381                                 } else {
1382                                         uint ui = (uint) (number_builder [0] - '0');
1383
1384                                         for (int i = 1; i < number_pos; i++){
1385                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1386                                         }
1387                                         return integer_type_suffix (ui, c);
1388                                 }
1389                         } catch (OverflowException) {
1390                                 Error_NumericConstantTooLong ();
1391                                 val = new IntLiteral (0, Location);
1392                                 return Token.LITERAL;
1393                         }
1394                         catch (FormatException) {
1395                                 Report.Error (1013, Location, "Invalid number");
1396                                 val = new IntLiteral (0, Location);
1397                                 return Token.LITERAL;
1398                         }
1399                 }
1400
1401                 int adjust_real (TypeCode t)
1402                 {
1403                         string s = new String (number_builder, 0, number_pos);
1404                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1405
1406                         switch (t){
1407                         case TypeCode.Decimal:
1408                                 try {
1409                                         val = new DecimalLiteral (decimal.Parse (s, styles, csharp_format_info), Location);
1410                                 } catch (OverflowException) {
1411                                         val = new DecimalLiteral (0, Location);
1412                                         Report.Error (594, Location, error_details, "decimal");
1413                                 }
1414                                 break;
1415                         case TypeCode.Single:
1416                                 try {
1417                                         val = new FloatLiteral (float.Parse (s, styles, csharp_format_info), Location);
1418                                 } catch (OverflowException) {
1419                                         val = new FloatLiteral (0, Location);
1420                                         Report.Error (594, Location, error_details, "float");
1421                                 }
1422                                 break;
1423                         default:
1424                                 try {
1425                                         val = new DoubleLiteral (double.Parse (s, styles, csharp_format_info), Location);
1426                                 } catch (OverflowException) {
1427                                         val = new DoubleLiteral (0, Location);
1428                                         Report.Error (594, Location, error_details, "double");
1429                                 }
1430                                 break;
1431                         }
1432
1433                         return Token.LITERAL;
1434                 }
1435
1436                 int handle_hex ()
1437                 {
1438                         int d;
1439                         ulong ul;
1440
1441                         get_char ();
1442                         while ((d = peek_char ()) != -1){
1443                                 if (is_hex (d)){
1444                                         number_builder [number_pos++] = (char) d;
1445                                         get_char ();
1446                                 } else
1447                                         break;
1448                         }
1449
1450                         string s = new String (number_builder, 0, number_pos);
1451                         try {
1452                                 if (number_pos <= 8)
1453                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1454                                 else
1455                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1456                         } catch (OverflowException){
1457                                 Error_NumericConstantTooLong ();
1458                                 val = new IntLiteral (0, Location);
1459                                 return Token.LITERAL;
1460                         }
1461                         catch (FormatException) {
1462                                 Report.Error (1013, Location, "Invalid number");
1463                                 val = new IntLiteral (0, Location);
1464                                 return Token.LITERAL;
1465                         }
1466
1467                         return integer_type_suffix (ul, peek_char ());
1468                 }
1469
1470                 //
1471                 // Invoked if we know we have .digits or digits
1472                 //
1473                 int is_number (int c)
1474                 {
1475                         bool is_real = false;
1476
1477                         number_pos = 0;
1478
1479                         if (c >= '0' && c <= '9'){
1480                                 if (c == '0'){
1481                                         int peek = peek_char ();
1482
1483                                         if (peek == 'x' || peek == 'X')
1484                                                 return handle_hex ();
1485                                 }
1486                                 decimal_digits (c);
1487                                 c = get_char ();
1488                         }
1489
1490                         //
1491                         // We need to handle the case of
1492                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1493                         //
1494                         if (c == '.'){
1495                                 if (decimal_digits ('.')){
1496                                         is_real = true;
1497                                         c = get_char ();
1498                                 } else {
1499                                         putback ('.');
1500                                         number_pos--;
1501                                         return adjust_int (-1);
1502                                 }
1503                         }
1504
1505                         if (c == 'e' || c == 'E'){
1506                                 is_real = true;
1507                                 if (number_pos == max_number_size)
1508                                         Error_NumericConstantTooLong ();
1509                                 number_builder [number_pos++] = 'e';
1510                                 c = get_char ();
1511
1512                                 if (c == '+'){
1513                                         if (number_pos == max_number_size)
1514                                                 Error_NumericConstantTooLong ();
1515                                         number_builder [number_pos++] = '+';
1516                                         c = -1;
1517                                 } else if (c == '-') {
1518                                         if (number_pos == max_number_size)
1519                                                 Error_NumericConstantTooLong ();
1520                                         number_builder [number_pos++] = '-';
1521                                         c = -1;
1522                                 } else {
1523                                         if (number_pos == max_number_size)
1524                                                 Error_NumericConstantTooLong ();
1525                                         number_builder [number_pos++] = '+';
1526                                 }
1527
1528                                 decimal_digits (c);
1529                                 c = get_char ();
1530                         }
1531
1532                         var type = real_type_suffix (c);
1533                         if (type == TypeCode.Empty && !is_real){
1534                                 putback (c);
1535                                 return adjust_int (c);
1536                         }
1537
1538                         is_real = true;
1539
1540                         if (type == TypeCode.Empty){
1541                                 putback (c);
1542                         }
1543
1544                         if (is_real)
1545                                 return adjust_real (type);
1546
1547                         throw new Exception ("Is Number should never reach this point");
1548                 }
1549
1550                 //
1551                 // Accepts exactly count (4 or 8) hex, no more no less
1552                 //
1553                 int getHex (int count, out int surrogate, out bool error)
1554                 {
1555                         int i;
1556                         int total = 0;
1557                         int c;
1558                         int top = count != -1 ? count : 4;
1559
1560                         get_char ();
1561                         error = false;
1562                         surrogate = 0;
1563                         for (i = 0; i < top; i++){
1564                                 c = get_char ();
1565
1566                                 if (c >= '0' && c <= '9')
1567                                         c = (int) c - (int) '0';
1568                                 else if (c >= 'A' && c <= 'F')
1569                                         c = (int) c - (int) 'A' + 10;
1570                                 else if (c >= 'a' && c <= 'f')
1571                                         c = (int) c - (int) 'a' + 10;
1572                                 else {
1573                                         error = true;
1574                                         return 0;
1575                                 }
1576
1577                                 total = (total * 16) + c;
1578                                 if (count == -1){
1579                                         int p = peek_char ();
1580                                         if (p == -1)
1581                                                 break;
1582                                         if (!is_hex ((char)p))
1583                                                 break;
1584                                 }
1585                         }
1586
1587                         if (top == 8) {
1588                                 if (total > 0x0010FFFF) {
1589                                         error = true;
1590                                         return 0;
1591                                 }
1592
1593                                 if (total >= 0x00010000) {
1594                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1595                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1596                                 }
1597                         }
1598
1599                         return total;
1600                 }
1601
1602                 int escape (int c, out int surrogate)
1603                 {
1604                         bool error;
1605                         int d;
1606                         int v;
1607
1608                         d = peek_char ();
1609                         if (c != '\\') {
1610                                 surrogate = 0;
1611                                 return c;
1612                         }
1613
1614                         switch (d){
1615                         case 'a':
1616                                 v = '\a'; break;
1617                         case 'b':
1618                                 v = '\b'; break;
1619                         case 'n':
1620                                 v = '\n'; break;
1621                         case 't':
1622                                 v = '\t'; break;
1623                         case 'v':
1624                                 v = '\v'; break;
1625                         case 'r':
1626                                 v = '\r'; break;
1627                         case '\\':
1628                                 v = '\\'; break;
1629                         case 'f':
1630                                 v = '\f'; break;
1631                         case '0':
1632                                 v = 0; break;
1633                         case '"':
1634                                 v = '"'; break;
1635                         case '\'':
1636                                 v = '\''; break;
1637                         case 'x':
1638                                 v = getHex (-1, out surrogate, out error);
1639                                 if (error)
1640                                         goto default;
1641                                 return v;
1642                         case 'u':
1643                         case 'U':
1644                                 return EscapeUnicode (d, out surrogate);
1645                         default:
1646                                 surrogate = 0;
1647                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1648                                 return d;
1649                         }
1650
1651                         get_char ();
1652                         surrogate = 0;
1653                         return v;
1654                 }
1655
1656                 int EscapeUnicode (int ch, out int surrogate)
1657                 {
1658                         bool error;
1659                         if (ch == 'U') {
1660                                 ch = getHex (8, out surrogate, out error);
1661                         } else {
1662                                 ch = getHex (4, out surrogate, out error);
1663                         }
1664
1665                         if (error)
1666                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1667
1668                         return ch;
1669                 }
1670
1671                 int get_char ()
1672                 {
1673                         int x;
1674                         if (putback_char != -1) {
1675                                 x = putback_char;
1676                                 putback_char = -1;
1677                         } else
1678                                 x = reader.Read ();
1679                         if (x == '\n') {
1680                                 advance_line ();
1681                         } else {
1682                                 col++;
1683                         }
1684                         return x;
1685                 }
1686
1687                 void advance_line ()
1688                 {
1689                         line++;
1690                         ref_line++;
1691                         previous_col = col;
1692                         col = 0;
1693                 }
1694
1695                 int peek_char ()
1696                 {
1697                         if (putback_char == -1)
1698                                 putback_char = reader.Read ();
1699                         return putback_char;
1700                 }
1701
1702                 int peek_char2 ()
1703                 {
1704                         if (putback_char != -1)
1705                                 return putback_char;
1706                         return reader.Peek ();
1707                 }
1708
1709                 void putback (int c)
1710                 {
1711                         if (putback_char != -1){
1712                                 Console.WriteLine ("Col: " + col);
1713                                 Console.WriteLine ("Row: " + line);
1714                                 Console.WriteLine ("Name: " + ref_name.Name);
1715                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1716                                 throw new Exception ("This should not happen putback on putback");
1717                         }
1718                         if (c == '\n' || col == 0) {
1719                                 // It won't happen though.
1720                                 line--;
1721                                 ref_line--;
1722                                 col = previous_col;
1723                         }
1724                         else
1725                                 col--;
1726                         putback_char = c;
1727                 }
1728
1729                 public bool advance ()
1730                 {
1731                         return peek_char () != -1 || CompleteOnEOF;
1732                 }
1733
1734                 public Object Value {
1735                         get {
1736                                 return val;
1737                         }
1738                 }
1739
1740                 public Object value ()
1741                 {
1742                         return val;
1743                 }
1744
1745                 public int token ()
1746                 {
1747                         current_token = xtoken ();
1748                         return current_token;
1749                 }
1750
1751                 int TokenizePreprocessorIdentifier (out int c)
1752                 {
1753                         // skip over white space
1754                         do {
1755                                 c = get_char ();
1756                         } while (c == '\r' || c == ' ' || c == '\t');
1757
1758
1759                         int pos = 0;
1760                         while (c != -1 && c >= 'a' && c <= 'z') {
1761                                 id_builder[pos++] = (char) c;
1762                                 c = get_char ();
1763                                 if (c == '\\') {
1764                                         int peek = peek_char ();
1765                                         if (peek == 'U' || peek == 'u') {
1766                                                 int surrogate;
1767                                                 c = EscapeUnicode (c, out surrogate);
1768                                                 if (surrogate != 0) {
1769                                                         if (is_identifier_part_character ((char) c)) {
1770                                                                 id_builder[pos++] = (char) c;
1771                                                         }
1772                                                         c = surrogate;
1773                                                 }
1774                                         }
1775                                 }
1776                         }
1777
1778                         return pos;
1779                 }
1780
1781                 PreprocessorDirective get_cmd_arg (out string arg)
1782                 {
1783                         int c;
1784
1785                         tokens_seen = false;
1786                         arg = "";
1787
1788                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorIdentifier (out c));
1789
1790                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
1791                                 return cmd;
1792
1793                         // skip over white space
1794                         while (c == '\r' || c == ' ' || c == '\t')
1795                                 c = get_char ();
1796
1797                         static_cmd_arg.Length = 0;
1798                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
1799
1800                         while (c != -1 && c != '\n' && c != '\r') {
1801                                 if (c == '\\' && has_identifier_argument >= 0) {
1802                                         if (has_identifier_argument != 0) {
1803                                                 has_identifier_argument = 1;
1804
1805                                                 int peek = peek_char ();
1806                                                 if (peek == 'U' || peek == 'u') {
1807                                                         int surrogate;
1808                                                         c = EscapeUnicode (c, out surrogate);
1809                                                         if (surrogate != 0) {
1810                                                                 if (is_identifier_part_character ((char) c))
1811                                                                         static_cmd_arg.Append ((char) c);
1812                                                                 c = surrogate;
1813                                                         }
1814                                                 }
1815                                         } else {
1816                                                 has_identifier_argument = -1;
1817                                         }
1818                                 }
1819                                 static_cmd_arg.Append ((char) c);
1820                                 c = get_char ();
1821                         }
1822
1823                         if (static_cmd_arg.Length != 0) {
1824                                 arg = static_cmd_arg.ToString ();
1825
1826                                 // Eat any trailing whitespaces and single-line comments
1827                                 if (arg.IndexOf ("//") != -1) {
1828                                         arg = arg.Substring (0, arg.IndexOf ("//"));
1829                                 }
1830
1831                                 arg = arg.Trim (simple_whitespaces);
1832                         }
1833
1834                         return cmd;
1835                 }
1836
1837                 //
1838                 // Handles the #line directive
1839                 //
1840                 bool PreProcessLine (string arg)
1841                 {
1842                         if (arg.Length == 0)
1843                                 return false;
1844
1845                         if (arg == "default"){
1846                                 ref_line = line;
1847                                 ref_name = file_name;
1848                                 hidden = false;
1849                                 Location.Push (file_name, ref_name);
1850                                 return true;
1851                         } else if (arg == "hidden"){
1852                                 hidden = true;
1853                                 return true;
1854                         }
1855
1856                         try {
1857                                 int pos;
1858
1859                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1860                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1861                                         pos++;
1862
1863                                         char [] quotes = { '\"' };
1864
1865                                         string name = arg.Substring (pos). Trim (quotes);
1866                                         ref_name = Location.LookupFile (file_name, name);
1867                                         file_name.AddFile (ref_name);
1868                                         hidden = false;
1869                                         Location.Push (file_name, ref_name);
1870                                 } else {
1871                                         ref_line = System.Int32.Parse (arg);
1872                                         hidden = false;
1873                                 }
1874                         } catch {
1875                                 return false;
1876                         }
1877
1878                         return true;
1879                 }
1880
1881                 //
1882                 // Handles #define and #undef
1883                 //
1884                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
1885                 {
1886                         if (ident.Length == 0 || ident == "true" || ident == "false"){
1887                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
1888                                 return;
1889                         }
1890
1891                         if (ident.IndexOfAny (simple_whitespaces) != -1){
1892                                 Error_EndLineExpected ();
1893                                 return;
1894                         }
1895
1896                         if (!is_identifier_start_character (ident [0]))
1897                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
1898
1899                         foreach (char c in ident.Substring (1)){
1900                                 if (!is_identifier_part_character (c)){
1901                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
1902                                         return;
1903                                 }
1904                         }
1905
1906                         if (!caller_is_taking)
1907                                 return;
1908
1909                         if (is_define) {
1910                                 //
1911                                 // #define ident
1912                                 //
1913                                 if (RootContext.IsConditionalDefined (ident))
1914                                         return;
1915
1916                                 file_name.AddDefine (ident);
1917                         } else {
1918                                 //
1919                                 // #undef ident
1920                                 //
1921                                 file_name.AddUndefine (ident);
1922                         }
1923                 }
1924
1925                 byte read_hex (out bool error)
1926                 {
1927                         int total;
1928                         int c = get_char ();
1929
1930                         if ((c >= '0') && (c <= '9'))
1931                                 total = (int) c - (int) '0';
1932                         else if ((c >= 'A') && (c <= 'F'))
1933                                 total = (int) c - (int) 'A' + 10;
1934                         else if ((c >= 'a') && (c <= 'f'))
1935                                 total = (int) c - (int) 'a' + 10;
1936                         else {
1937                                 error = true;
1938                                 return 0;
1939                         }
1940
1941                         total *= 16;
1942                         c = get_char ();
1943
1944                         if ((c >= '0') && (c <= '9'))
1945                                 total += (int) c - (int) '0';
1946                         else if ((c >= 'A') && (c <= 'F'))
1947                                 total += (int) c - (int) 'A' + 10;
1948                         else if ((c >= 'a') && (c <= 'f'))
1949                                 total += (int) c - (int) 'a' + 10;
1950                         else {
1951                                 error = true;
1952                                 return 0;
1953                         }
1954
1955                         error = false;
1956                         return (byte) total;
1957                 }
1958
1959                 //
1960                 // Parses #pragma checksum
1961                 //
1962                 bool ParsePragmaChecksum ()
1963                 {
1964                         //
1965                         // The syntax is ` "foo.txt" "{guid}" "hash"'
1966                         //
1967                         int c = get_char ();
1968
1969                         if (c != '"')
1970                                 return false;
1971
1972                         string_builder.Length = 0;
1973                         while (c != -1 && c != '\n') {
1974                                 c = get_char ();
1975                                 if (c == '"') {
1976                                         c = get_char ();
1977                                         break;
1978                                 }
1979
1980                                 string_builder.Append ((char) c);
1981                         }
1982
1983                         if (string_builder.Length == 0) {
1984                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
1985                         }
1986
1987                         // TODO: Any white-spaces count
1988                         if (c != ' ')
1989                                 return false;
1990
1991                         SourceFile file = Location.LookupFile (file_name, string_builder.ToString ());
1992
1993                         if (get_char () != '"' || get_char () != '{')
1994                                 return false;
1995
1996                         bool error;
1997                         byte[] guid_bytes = new byte [16];
1998                         int i = 0;
1999
2000                         for (; i < 4; i++) {
2001                                 guid_bytes [i] = read_hex (out error);
2002                                 if (error)
2003                                         return false;
2004                         }
2005
2006                         if (get_char () != '-')
2007                                 return false;
2008
2009                         for (; i < 10; i++) {
2010                                 guid_bytes [i] = read_hex (out error);
2011                                 if (error)
2012                                         return false;
2013
2014                                 guid_bytes [i++] = read_hex (out error);
2015                                 if (error)
2016                                         return false;
2017
2018                                 if (get_char () != '-')
2019                                         return false;
2020                         }
2021
2022                         for (; i < 16; i++) {
2023                                 guid_bytes [i] = read_hex (out error);
2024                                 if (error)
2025                                         return false;
2026                         }
2027
2028                         if (get_char () != '}' || get_char () != '"')
2029                                 return false;
2030
2031                         // TODO: Any white-spaces count
2032                         c = get_char ();
2033                         if (c != ' ')
2034                                 return false;
2035
2036                         if (get_char () != '"')
2037                                 return false;
2038
2039                         // Any length of checksum
2040                         List<byte> checksum_bytes = new List<byte> (16);
2041
2042                         c = peek_char ();
2043                         while (c != '"' && c != -1) {
2044                                 checksum_bytes.Add (read_hex (out error));
2045                                 if (error)
2046                                         return false;
2047
2048                                 c = peek_char ();
2049                         }
2050
2051                         if (c == '/') {
2052                                 ReadSingleLineComment ();
2053                         } else if (get_char () != '"') {
2054                                 return false;
2055                         }
2056
2057                         file.SetChecksum (guid_bytes, checksum_bytes.ToArray ());
2058                         ref_name.AutoGenerated = true;
2059                         return true;
2060                 }
2061
2062                 bool IsTokenIdentifierEqual (char[] identifier)
2063                 {
2064                         for (int i = 0; i < identifier.Length; ++i) {
2065                                 if (identifier[i] != id_builder[i])
2066                                         return false;
2067                         }
2068
2069                         return true;
2070                 }
2071
2072                 int TokenizePragmaNumber (ref int c)
2073                 {
2074                         number_pos = 0;
2075
2076                         int number;
2077
2078                         if (c >= '0' && c <= '9') {
2079                                 decimal_digits (c);
2080                                 uint ui = (uint) (number_builder[0] - '0');
2081
2082                                 try {
2083                                         for (int i = 1; i < number_pos; i++) {
2084                                                 ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2085                                         }
2086
2087                                         number = (int) ui;
2088                                 } catch (OverflowException) {
2089                                         Error_NumericConstantTooLong ();
2090                                         number = -1;
2091                                 }
2092
2093
2094                                 c = get_char ();
2095
2096                                 // skip over white space
2097                                 while (c == '\r' || c == ' ' || c == '\t')
2098                                         c = get_char ();
2099
2100                                 if (c == ',') {
2101                                         c = get_char ();
2102                                 }
2103
2104                                 // skip over white space
2105                                 while (c == '\r' || c == ' ' || c == '\t')
2106                                         c = get_char ();
2107                         } else {
2108                                 number = -1;
2109                                 if (c == '/') {
2110                                         ReadSingleLineComment ();
2111                                 } else {
2112                                         Report.Warning (1692, 1, Location, "Invalid number");
2113
2114                                         // Read everything till the end of the line or file
2115                                         do {
2116                                                 c = get_char ();
2117                                         } while (c != -1 && c != '\n');
2118                                 }
2119                         }
2120
2121                         return number;
2122                 }
2123
2124                 void ReadSingleLineComment ()
2125                 {
2126                         if (peek_char () != '/')
2127                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2128
2129                         // Read everything till the end of the line or file
2130                         int c;
2131                         do {
2132                                 c = get_char ();
2133                         } while (c != -1 && c != '\n');
2134                 }
2135
2136                 /// <summary>
2137                 /// Handles #pragma directive
2138                 /// </summary>
2139                 void ParsePragmaDirective (string arg)
2140                 {
2141                         int c;
2142                         int length = TokenizePreprocessorIdentifier (out c);
2143                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2144                                 length = TokenizePreprocessorIdentifier (out c);
2145
2146                                 //
2147                                 // #pragma warning disable
2148                                 // #pragma warning restore
2149                                 //
2150                                 if (length == pragma_warning_disable.Length) {
2151                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2152                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2153                                                 // skip over white space
2154                                                 while (c == '\r' || c == ' ' || c == '\t')
2155                                                         c = get_char ();
2156
2157                                                 var loc = Location;
2158
2159                                                 if (c == '\n' || c == '/') {
2160                                                         if (c == '/')
2161                                                                 ReadSingleLineComment ();
2162
2163                                                         //
2164                                                         // Disable/Restore all warnings
2165                                                         //
2166                                                         if (disable) {
2167                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2168                                                         } else {
2169                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2170                                                         }
2171                                                 } else {
2172                                                         //
2173                                                         // Disable/Restore a warning or group of warnings
2174                                                         //
2175                                                         int code;
2176                                                         do {
2177                                                                 code = TokenizePragmaNumber (ref c);
2178                                                                 if (code > 0) {
2179                                                                         if (disable) {
2180                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, Report);
2181                                                                         } else {
2182                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, Report);
2183                                                                         }
2184                                                                 }
2185                                                         } while (code >= 0 && c != '\n' && c != -1);
2186                                                 }
2187
2188                                                 return;
2189                                         }
2190                                 }
2191
2192                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2193                                 return;
2194                         }
2195
2196                         //
2197                         // #pragma checksum
2198                         //
2199                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2200                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2201                                         Report.Warning (1695, 1, Location,
2202                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2203                                 }
2204
2205                                 return;
2206                         }
2207
2208                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2209                 }
2210
2211                 bool eval_val (string s)
2212                 {
2213                         if (s == "true")
2214                                 return true;
2215                         if (s == "false")
2216                                 return false;
2217
2218                         return file_name.IsConditionalDefined (s);
2219                 }
2220
2221                 bool pp_primary (ref string s)
2222                 {
2223                         s = s.Trim ();
2224                         int len = s.Length;
2225
2226                         if (len > 0){
2227                                 char c = s [0];
2228
2229                                 if (c == '('){
2230                                         s = s.Substring (1);
2231                                         bool val = pp_expr (ref s, false);
2232                                         if (s.Length > 0 && s [0] == ')'){
2233                                                 s = s.Substring (1);
2234                                                 return val;
2235                                         }
2236                                         Error_InvalidDirective ();
2237                                         return false;
2238                                 }
2239
2240                                 if (is_identifier_start_character (c)){
2241                                         int j = 1;
2242
2243                                         while (j < len){
2244                                                 c = s [j];
2245
2246                                                 if (is_identifier_part_character (c)){
2247                                                         j++;
2248                                                         continue;
2249                                                 }
2250                                                 bool v = eval_val (s.Substring (0, j));
2251                                                 s = s.Substring (j);
2252                                                 return v;
2253                                         }
2254                                         bool vv = eval_val (s);
2255                                         s = "";
2256                                         return vv;
2257                                 }
2258                         }
2259                         Error_InvalidDirective ();
2260                         return false;
2261                 }
2262
2263                 bool pp_unary (ref string s)
2264                 {
2265                         s = s.Trim ();
2266                         int len = s.Length;
2267
2268                         if (len > 0){
2269                                 if (s [0] == '!'){
2270                                         if (len > 1 && s [1] == '='){
2271                                                 Error_InvalidDirective ();
2272                                                 return false;
2273                                         }
2274                                         s = s.Substring (1);
2275                                         return ! pp_primary (ref s);
2276                                 } else
2277                                         return pp_primary (ref s);
2278                         } else {
2279                                 Error_InvalidDirective ();
2280                                 return false;
2281                         }
2282                 }
2283
2284                 bool pp_eq (ref string s)
2285                 {
2286                         bool va = pp_unary (ref s);
2287
2288                         s = s.Trim ();
2289                         int len = s.Length;
2290                         if (len > 0){
2291                                 if (s [0] == '='){
2292                                         if (len > 2 && s [1] == '='){
2293                                                 s = s.Substring (2);
2294                                                 return va == pp_unary (ref s);
2295                                         } else {
2296                                                 Error_InvalidDirective ();
2297                                                 return false;
2298                                         }
2299                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2300                                         s = s.Substring (2);
2301
2302                                         return va != pp_unary (ref s);
2303
2304                                 }
2305                         }
2306
2307                         return va;
2308
2309                 }
2310
2311                 bool pp_and (ref string s)
2312                 {
2313                         bool va = pp_eq (ref s);
2314
2315                         s = s.Trim ();
2316                         int len = s.Length;
2317                         if (len > 0){
2318                                 if (s [0] == '&'){
2319                                         if (len > 2 && s [1] == '&'){
2320                                                 s = s.Substring (2);
2321                                                 return (va & pp_and (ref s));
2322                                         } else {
2323                                                 Error_InvalidDirective ();
2324                                                 return false;
2325                                         }
2326                                 }
2327                         }
2328                         return va;
2329                 }
2330
2331                 //
2332                 // Evaluates an expression for `#if' or `#elif'
2333                 //
2334                 bool pp_expr (ref string s, bool isTerm)
2335                 {
2336                         bool va = pp_and (ref s);
2337                         s = s.Trim ();
2338                         int len = s.Length;
2339                         if (len > 0){
2340                                 char c = s [0];
2341
2342                                 if (c == '|'){
2343                                         if (len > 2 && s [1] == '|'){
2344                                                 s = s.Substring (2);
2345                                                 return va | pp_expr (ref s, isTerm);
2346                                         } else {
2347                                                 Error_InvalidDirective ();
2348                                                 return false;
2349                                         }
2350                                 }
2351                                 if (isTerm) {
2352                                         Error_EndLineExpected ();
2353                                         return false;
2354                                 }
2355                         }
2356
2357                         return va;
2358                 }
2359
2360                 bool eval (string s)
2361                 {
2362                         bool v = pp_expr (ref s, true);
2363                         s = s.Trim ();
2364                         if (s.Length != 0){
2365                                 return false;
2366                         }
2367
2368                         return v;
2369                 }
2370
2371                 void Error_NumericConstantTooLong ()
2372                 {
2373                         Report.Error (1021, Location, "Integral constant is too large");
2374                 }
2375
2376                 void Error_InvalidDirective ()
2377                 {
2378                         Report.Error (1517, Location, "Invalid preprocessor directive");
2379                 }
2380
2381                 void Error_UnexpectedDirective (string extra)
2382                 {
2383                         Report.Error (
2384                                 1028, Location,
2385                                 "Unexpected processor directive ({0})", extra);
2386                 }
2387
2388                 void Error_TokensSeen ()
2389                 {
2390                         Report.Error (1032, Location,
2391                                 "Cannot define or undefine preprocessor symbols after first token in file");
2392                 }
2393
2394                 void Eror_WrongPreprocessorLocation ()
2395                 {
2396                         Report.Error (1040, Location,
2397                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2398                 }
2399
2400                 void Error_EndLineExpected ()
2401                 {
2402                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2403                 }
2404
2405                 //
2406                 // if true, then the code continues processing the code
2407                 // if false, the code stays in a loop until another directive is
2408                 // reached.
2409                 // When caller_is_taking is false we ignore all directives except the ones
2410                 // which can help us to identify where the #if block ends
2411                 bool ParsePreprocessingDirective (bool caller_is_taking)
2412                 {
2413                         string arg;
2414                         bool region_directive = false;
2415
2416                         var directive = get_cmd_arg (out arg);
2417
2418                         //
2419                         // The first group of pre-processing instructions is always processed
2420                         //
2421                         switch (directive) {
2422                         case PreprocessorDirective.Region:
2423                                 region_directive = true;
2424                                 arg = "true";
2425                                 goto case PreprocessorDirective.If;
2426
2427                         case PreprocessorDirective.Endregion:
2428                                 if (ifstack == null || ifstack.Count == 0){
2429                                         Error_UnexpectedDirective ("no #region for this #endregion");
2430                                         return true;
2431                                 }
2432                                 int pop = ifstack.Pop ();
2433
2434                                 if ((pop & REGION) == 0)
2435                                         Report.Error (1027, Location, "Expected `#endif' directive");
2436
2437                                 return caller_is_taking;
2438
2439                         case PreprocessorDirective.If:
2440                                 if (ifstack == null)
2441                                         ifstack = new Stack<int> (2);
2442
2443                                 int flags = region_directive ? REGION : 0;
2444                                 if (ifstack.Count == 0){
2445                                         flags |= PARENT_TAKING;
2446                                 } else {
2447                                         int state = ifstack.Peek ();
2448                                         if ((state & TAKING) != 0) {
2449                                                 flags |= PARENT_TAKING;
2450                                         }
2451                                 }
2452
2453                                 if (eval (arg) && caller_is_taking) {
2454                                         ifstack.Push (flags | TAKING);
2455                                         return true;
2456                                 }
2457                                 ifstack.Push (flags);
2458                                 return false;
2459
2460                         case PreprocessorDirective.Endif:
2461                                 if (ifstack == null || ifstack.Count == 0){
2462                                         Error_UnexpectedDirective ("no #if for this #endif");
2463                                         return true;
2464                                 } else {
2465                                         pop = ifstack.Pop ();
2466
2467                                         if ((pop & REGION) != 0)
2468                                                 Report.Error (1038, Location, "#endregion directive expected");
2469
2470                                         if (arg.Length != 0) {
2471                                                 Error_EndLineExpected ();
2472                                         }
2473
2474                                         if (ifstack.Count == 0)
2475                                                 return true;
2476
2477                                         int state = ifstack.Peek ();
2478                                         return (state & TAKING) != 0;
2479                                 }
2480
2481                         case PreprocessorDirective.Elif:
2482                                 if (ifstack == null || ifstack.Count == 0){
2483                                         Error_UnexpectedDirective ("no #if for this #elif");
2484                                         return true;
2485                                 } else {
2486                                         int state = ifstack.Pop ();
2487
2488                                         if ((state & REGION) != 0) {
2489                                                 Report.Error (1038, Location, "#endregion directive expected");
2490                                                 return true;
2491                                         }
2492
2493                                         if ((state & ELSE_SEEN) != 0){
2494                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2495                                                 return true;
2496                                         }
2497
2498                                         if ((state & TAKING) != 0) {
2499                                                 ifstack.Push (0);
2500                                                 return false;
2501                                         }
2502
2503                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2504                                                 ifstack.Push (state | TAKING);
2505                                                 return true;
2506                                         }
2507
2508                                         ifstack.Push (state);
2509                                         return false;
2510                                 }
2511
2512                         case PreprocessorDirective.Else:
2513                                 if (ifstack == null || ifstack.Count == 0){
2514                                         Error_UnexpectedDirective ("no #if for this #else");
2515                                         return true;
2516                                 } else {
2517                                         int state = ifstack.Peek ();
2518
2519                                         if ((state & REGION) != 0) {
2520                                                 Report.Error (1038, Location, "#endregion directive expected");
2521                                                 return true;
2522                                         }
2523
2524                                         if ((state & ELSE_SEEN) != 0){
2525                                                 Error_UnexpectedDirective ("#else within #else");
2526                                                 return true;
2527                                         }
2528
2529                                         ifstack.Pop ();
2530
2531                                         if (arg.Length != 0) {
2532                                                 Error_EndLineExpected ();
2533                                                 return true;
2534                                         }
2535
2536                                         bool ret = false;
2537                                         if ((state & PARENT_TAKING) != 0) {
2538                                                 ret = (state & TAKING) == 0;
2539
2540                                                 if (ret)
2541                                                         state |= TAKING;
2542                                                 else
2543                                                         state &= ~TAKING;
2544                                         }
2545
2546                                         ifstack.Push (state | ELSE_SEEN);
2547
2548                                         return ret;
2549                                 }
2550                         case PreprocessorDirective.Define:
2551                                 if (any_token_seen){
2552                                         Error_TokensSeen ();
2553                                         return caller_is_taking;
2554                                 }
2555                                 PreProcessDefinition (true, arg, caller_is_taking);
2556                                 return caller_is_taking;
2557
2558                         case PreprocessorDirective.Undef:
2559                                 if (any_token_seen){
2560                                         Error_TokensSeen ();
2561                                         return caller_is_taking;
2562                                 }
2563                                 PreProcessDefinition (false, arg, caller_is_taking);
2564                                 return caller_is_taking;
2565
2566                         case PreprocessorDirective.Invalid:
2567                                 Report.Error (1024, Location, "Wrong preprocessor directive");
2568                                 return true;
2569                         }
2570
2571                         //
2572                         // These are only processed if we are in a `taking' block
2573                         //
2574                         if (!caller_is_taking)
2575                                 return false;
2576
2577                         switch (directive){
2578                         case PreprocessorDirective.Error:
2579                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2580                                 return true;
2581
2582                         case PreprocessorDirective.Warning:
2583                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2584                                 return true;
2585
2586                         case PreprocessorDirective.Pragma:
2587                                 if (RootContext.Version == LanguageVersion.ISO_1) {
2588                                         Report.FeatureIsNotAvailable (Location, "#pragma");
2589                                 }
2590
2591                                 ParsePragmaDirective (arg);
2592                                 return true;
2593
2594                         case PreprocessorDirective.Line:
2595                                 if (!PreProcessLine (arg))
2596                                         Report.Error (
2597                                                 1576, Location,
2598                                                 "The line number specified for #line directive is missing or invalid");
2599                                 return caller_is_taking;
2600                         }
2601
2602                         throw new NotImplementedException (directive.ToString ());
2603                 }
2604
2605                 private int consume_string (bool quoted)
2606                 {
2607                         int c;
2608                         string_builder.Length = 0;
2609
2610                         while (true){
2611                                 c = get_char ();
2612                                 if (c == '"') {
2613                                         if (quoted && peek_char () == '"') {
2614                                                 string_builder.Append ((char) c);
2615                                                 get_char ();
2616                                                 continue;
2617                                         }
2618
2619                                         val = new StringLiteral (string_builder.ToString (), Location);
2620                                         return Token.LITERAL;
2621                                 }
2622
2623                                 if (c == '\n') {
2624                                         if (!quoted)
2625                                                 Report.Error (1010, Location, "Newline in constant");
2626                                 } else if (c == '\\' && !quoted) {
2627                                         int surrogate;
2628                                         c = escape (c, out surrogate);
2629                                         if (c == -1)
2630                                                 return Token.ERROR;
2631                                         if (surrogate != 0) {
2632                                                 string_builder.Append ((char) c);
2633                                                 c = surrogate;
2634                                         }
2635                                 } else if (c == -1) {
2636                                         Report.Error (1039, Location, "Unterminated string literal");
2637                                         return Token.EOF;
2638                                 }
2639
2640                                 string_builder.Append ((char) c);
2641                         }
2642                 }
2643
2644                 private int consume_identifier (int s)
2645                 {
2646                         int res = consume_identifier (s, false);
2647
2648                         if (doc_state == XmlCommentState.Allowed)
2649                                 doc_state = XmlCommentState.NotAllowed;
2650
2651                         return res;
2652                 }
2653
2654                 int consume_identifier (int c, bool quoted)
2655                 {
2656                         //
2657                         // This method is very performance sensitive. It accounts
2658                         // for approximately 25% of all parser time
2659                         //
2660
2661                         int pos = 0;
2662                         int column = col;
2663
2664                         if (c == '\\') {
2665                                 int surrogate;
2666                                 c = escape (c, out surrogate);
2667                                 if (surrogate != 0) {
2668                                         id_builder [pos++] = (char) c;
2669                                         c = surrogate;
2670                                 }
2671                         }
2672
2673                         id_builder [pos++] = (char) c;
2674
2675                         try {
2676                                 while (true) {
2677                                         c = reader.Read ();
2678
2679                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
2680                                                 id_builder [pos++] = (char) c;
2681                                                 continue;
2682                                         }
2683
2684                                         if (c < 0x80) {
2685                                                 if (c == '\\') {
2686                                                         int surrogate;
2687                                                         c = escape (c, out surrogate);
2688                                                         if (surrogate != 0) {
2689                                                                 if (is_identifier_part_character ((char) c))
2690                                                                         id_builder[pos++] = (char) c;
2691                                                                 c = surrogate;
2692                                                         }
2693
2694                                                         continue;
2695                                                 }
2696                                         } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
2697                                                 id_builder [pos++] = (char) c;
2698                                                 continue;
2699                                         }
2700
2701                                         putback_char = c;
2702                                         break;
2703                                 }
2704                         } catch (IndexOutOfRangeException) {
2705                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2706                                 --pos;
2707                                 col += pos;
2708                         }
2709
2710                         col += pos - 1;
2711
2712                         //
2713                         // Optimization: avoids doing the keyword lookup
2714                         // on uppercase letters
2715                         //
2716                         if (id_builder [0] >= '_' && !quoted) {
2717                                 int keyword = GetKeyword (id_builder, pos);
2718                                 if (keyword != -1) {
2719                                         val = LocatedToken.Create (null, ref_line, column);
2720                                         return keyword;
2721                                 }
2722                         }
2723
2724                         //
2725                         // Keep identifiers in an array of hashtables to avoid needless
2726                         // allocations
2727                         //
2728                         var identifiers_group = identifiers [pos];
2729                         string s;
2730                         if (identifiers_group != null) {
2731                                 if (identifiers_group.TryGetValue (id_builder, out s)) {
2732                                         val = LocatedToken.Create (s, ref_line, column);
2733                                         if (quoted)
2734                                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2735                                         return Token.IDENTIFIER;
2736                                 }
2737                         } else {
2738                                 // TODO: this should be number of files dependant
2739                                 // corlib compilation peaks at 1000 and System.Core at 150
2740                                 int capacity = pos > 20 ? 10 : 100;
2741                                 identifiers_group = new Dictionary<char[],string> (capacity, new IdentifiersComparer (pos));
2742                                 identifiers [pos] = identifiers_group;
2743                         }
2744
2745                         char [] chars = new char [pos];
2746                         Array.Copy (id_builder, chars, pos);
2747
2748                         s = new string (id_builder, 0, pos);
2749                         identifiers_group.Add (chars, s);
2750
2751                         val = LocatedToken.Create (s, ref_line, column);
2752                         if (quoted)
2753                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2754
2755                         return Token.IDENTIFIER;
2756                 }
2757
2758                 public int xtoken ()
2759                 {
2760                         int d, c;
2761
2762                         // Whether we have seen comments on the current line
2763                         bool comments_seen = false;
2764                         while ((c = get_char ()) != -1) {
2765                                 switch (c) {
2766                                 case '\t':
2767                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
2768                                         continue;
2769
2770                                 case ' ':
2771                                 case '\f':
2772                                 case '\v':
2773                                 case 0xa0:
2774                                 case 0:
2775                                 case 0xFEFF:    // Ignore BOM anywhere in the file
2776                                         continue;
2777
2778 /*                              This is required for compatibility with .NET
2779                                 case 0xEF:
2780                                         if (peek_char () == 0xBB) {
2781                                                 PushPosition ();
2782                                                 get_char ();
2783                                                 if (get_char () == 0xBF)
2784                                                         continue;
2785                                                 PopPosition ();
2786                                         }
2787                                         break;
2788 */
2789                                 case '\r':
2790                                         if (peek_char () != '\n')
2791                                                 advance_line ();
2792                                         else
2793                                                 get_char ();
2794
2795                                         any_token_seen |= tokens_seen;
2796                                         tokens_seen = false;
2797                                         comments_seen = false;
2798                                         continue;
2799
2800                                 case '\\':
2801                                         tokens_seen = true;
2802                                         return consume_identifier (c);
2803
2804                                 case '{':
2805                                         val = LocatedToken.Create (ref_line, col);
2806                                         return Token.OPEN_BRACE;
2807                                 case '}':
2808                                         val = LocatedToken.Create (ref_line, col);
2809                                         return Token.CLOSE_BRACE;
2810                                 case '[':
2811                                         // To block doccomment inside attribute declaration.
2812                                         if (doc_state == XmlCommentState.Allowed)
2813                                                 doc_state = XmlCommentState.NotAllowed;
2814
2815                                         val = LocatedToken.Create (ref_line, col);
2816
2817                                         if (parsing_block == 0 || lambda_arguments_parsing)
2818                                                 return Token.OPEN_BRACKET;
2819
2820                                         int next = peek_char ();
2821                                         switch (next) {
2822                                         case ']':
2823                                         case ',':
2824                                                 return Token.OPEN_BRACKET;
2825
2826                                         case ' ':
2827                                         case '\f':
2828                                         case '\v':
2829                                         case '\r':
2830                                         case '\n':
2831                                         case '/':
2832                                                 next = peek_token ();
2833                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
2834                                                         return Token.OPEN_BRACKET;
2835
2836                                                 return Token.OPEN_BRACKET_EXPR;
2837                                         default:
2838                                                 return Token.OPEN_BRACKET_EXPR;
2839                                         }
2840                                 case ']':
2841                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2842                                         return Token.CLOSE_BRACKET;
2843                                 case '(':
2844                                         val = LocatedToken.Create (ref_line, col);
2845                                         //
2846                                         // An expression versions of parens can appear in block context only
2847                                         //
2848                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
2849
2850                                                 //
2851                                                 // Optmize most common case where we know that parens
2852                                                 // is not special
2853                                                 //
2854                                                 switch (current_token) {
2855                                                 case Token.IDENTIFIER:
2856                                                 case Token.IF:
2857                                                 case Token.FOR:
2858                                                 case Token.FOREACH:
2859                                                 case Token.TYPEOF:
2860                                                 case Token.WHILE:
2861                                                 case Token.USING:
2862                                                 case Token.DEFAULT:
2863                                                 case Token.DELEGATE:
2864                                                 case Token.OP_GENERICS_GT:
2865                                                         return Token.OPEN_PARENS;
2866                                                 }
2867
2868                                                 // Optimize using peek
2869                                                 int xx = peek_char ();
2870                                                 switch (xx) {
2871                                                 case '(':
2872                                                 case '\'':
2873                                                 case '"':
2874                                                 case '0':
2875                                                 case '1':
2876                                                         return Token.OPEN_PARENS;
2877                                                 }
2878
2879                                                 lambda_arguments_parsing = true;
2880                                                 PushPosition ();
2881                                                 d = TokenizeOpenParens ();
2882                                                 PopPosition ();
2883                                                 lambda_arguments_parsing = false;
2884                                                 return d;
2885                                         }
2886
2887                                         return Token.OPEN_PARENS;
2888                                 case ')':
2889                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2890                                         return Token.CLOSE_PARENS;
2891                                 case ',':
2892                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2893                                         return Token.COMMA;
2894                                 case ';':
2895                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2896                                         return Token.SEMICOLON;
2897                                 case '~':
2898                                         val = LocatedToken.Create (ref_line, col);
2899                                         return Token.TILDE;
2900                                 case '?':
2901                                         val = LocatedToken.Create (ref_line, col);
2902                                         return TokenizePossibleNullableType ();
2903                                 case '<':
2904                                         val = LocatedToken.Create (ref_line, col);
2905                                         if (parsing_generic_less_than++ > 0)
2906                                                 return Token.OP_GENERICS_LT;
2907
2908                                         return TokenizeLessThan ();
2909
2910                                 case '>':
2911                                         val = LocatedToken.Create (ref_line, col);
2912                                         d = peek_char ();
2913
2914                                         if (d == '='){
2915                                                 get_char ();
2916                                                 return Token.OP_GE;
2917                                         }
2918
2919                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
2920                                                 parsing_generic_less_than--;
2921                                                 return Token.OP_GENERICS_GT;
2922                                         }
2923
2924                                         if (d == '>') {
2925                                                 get_char ();
2926                                                 d = peek_char ();
2927
2928                                                 if (d == '=') {
2929                                                         get_char ();
2930                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
2931                                                 }
2932                                                 return Token.OP_SHIFT_RIGHT;
2933                                         }
2934
2935                                         return Token.OP_GT;
2936
2937                                 case '+':
2938                                         val = LocatedToken.Create (ref_line, col);
2939                                         d = peek_char ();
2940                                         if (d == '+') {
2941                                                 d = Token.OP_INC;
2942                                         } else if (d == '=') {
2943                                                 d = Token.OP_ADD_ASSIGN;
2944                                         } else {
2945                                                 return Token.PLUS;
2946                                         }
2947                                         get_char ();
2948                                         return d;
2949
2950                                 case '-':
2951                                         val = LocatedToken.Create (ref_line, col);
2952                                         d = peek_char ();
2953                                         if (d == '-') {
2954                                                 d = Token.OP_DEC;
2955                                         } else if (d == '=')
2956                                                 d = Token.OP_SUB_ASSIGN;
2957                                         else if (d == '>')
2958                                                 d = Token.OP_PTR;
2959                                         else {
2960                                                 return Token.MINUS;
2961                                         }
2962                                         get_char ();
2963                                         return d;
2964
2965                                 case '!':
2966                                         val = LocatedToken.Create (ref_line, col);
2967                                         if (peek_char () == '='){
2968                                                 get_char ();
2969                                                 return Token.OP_NE;
2970                                         }
2971                                         return Token.BANG;
2972
2973                                 case '=':
2974                                         val = LocatedToken.Create (ref_line, col);
2975                                         d = peek_char ();
2976                                         if (d == '='){
2977                                                 get_char ();
2978                                                 return Token.OP_EQ;
2979                                         }
2980                                         if (d == '>'){
2981                                                 get_char ();
2982                                                 return Token.ARROW;
2983                                         }
2984
2985                                         return Token.ASSIGN;
2986
2987                                 case '&':
2988                                         val = LocatedToken.Create (ref_line, col);
2989                                         d = peek_char ();
2990                                         if (d == '&'){
2991                                                 get_char ();
2992                                                 return Token.OP_AND;
2993                                         }
2994                                         if (d == '='){
2995                                                 get_char ();
2996                                                 return Token.OP_AND_ASSIGN;
2997                                         }
2998                                         return Token.BITWISE_AND;
2999
3000                                 case '|':
3001                                         val = LocatedToken.Create (ref_line, col);
3002                                         d = peek_char ();
3003                                         if (d == '|'){
3004                                                 get_char ();
3005                                                 return Token.OP_OR;
3006                                         }
3007                                         if (d == '='){
3008                                                 get_char ();
3009                                                 return Token.OP_OR_ASSIGN;
3010                                         }
3011                                         return Token.BITWISE_OR;
3012
3013                                 case '*':
3014                                         val = LocatedToken.Create (ref_line, col);
3015                                         if (peek_char () == '='){
3016                                                 get_char ();
3017                                                 return Token.OP_MULT_ASSIGN;
3018                                         }
3019                                         return Token.STAR;
3020
3021                                 case '/':
3022                                         d = peek_char ();
3023                                         if (d == '='){
3024                                                 val = LocatedToken.Create (ref_line, col);
3025                                                 get_char ();
3026                                                 return Token.OP_DIV_ASSIGN;
3027                                         }
3028
3029                                         // Handle double-slash comments.
3030                                         if (d == '/'){
3031                                                 get_char ();
3032                                                 if (RootContext.Documentation != null && peek_char () == '/') {
3033                                                         get_char ();
3034                                                         // Don't allow ////.
3035                                                         if ((d = peek_char ()) != '/') {
3036                                                                 update_comment_location ();
3037                                                                 if (doc_state == XmlCommentState.Allowed)
3038                                                                         handle_one_line_xml_comment ();
3039                                                                 else if (doc_state == XmlCommentState.NotAllowed)
3040                                                                         warn_incorrect_doc_comment ();
3041                                                         }
3042                                                 }
3043                                                 while ((d = get_char ()) != -1 && (d != '\n') && d != '\r');
3044
3045                                                 any_token_seen |= tokens_seen;
3046                                                 tokens_seen = false;
3047                                                 comments_seen = false;
3048                                                 continue;
3049                                         } else if (d == '*'){
3050                                                 get_char ();
3051                                                 bool docAppend = false;
3052                                                 if (RootContext.Documentation != null && peek_char () == '*') {
3053                                                         get_char ();
3054                                                         update_comment_location ();
3055                                                         // But when it is /**/, just do nothing.
3056                                                         if (peek_char () == '/') {
3057                                                                 get_char ();
3058                                                                 continue;
3059                                                         }
3060                                                         if (doc_state == XmlCommentState.Allowed)
3061                                                                 docAppend = true;
3062                                                         else if (doc_state == XmlCommentState.NotAllowed)
3063                                                                 warn_incorrect_doc_comment ();
3064                                                 }
3065
3066                                                 int current_comment_start = 0;
3067                                                 if (docAppend) {
3068                                                         current_comment_start = xml_comment_buffer.Length;
3069                                                         xml_comment_buffer.Append (Environment.NewLine);
3070                                                 }
3071
3072                                                 while ((d = get_char ()) != -1){
3073                                                         if (d == '*' && peek_char () == '/'){
3074                                                                 get_char ();
3075                                                                 comments_seen = true;
3076                                                                 break;
3077                                                         }
3078                                                         if (docAppend)
3079                                                                 xml_comment_buffer.Append ((char) d);
3080
3081                                                         if (d == '\n'){
3082                                                                 any_token_seen |= tokens_seen;
3083                                                                 tokens_seen = false;
3084                                                                 //
3085                                                                 // Reset 'comments_seen' just to be consistent.
3086                                                                 // It doesn't matter either way, here.
3087                                                                 //
3088                                                                 comments_seen = false;
3089                                                         }
3090                                                 }
3091                                                 if (!comments_seen)
3092                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3093
3094                                                 if (docAppend)
3095                                                         update_formatted_doc_comment (current_comment_start);
3096                                                 continue;
3097                                         }
3098                                         val = LocatedToken.Create (ref_line, col);
3099                                         return Token.DIV;
3100
3101                                 case '%':
3102                                         val = LocatedToken.Create (ref_line, col);
3103                                         if (peek_char () == '='){
3104                                                 get_char ();
3105                                                 return Token.OP_MOD_ASSIGN;
3106                                         }
3107                                         return Token.PERCENT;
3108
3109                                 case '^':
3110                                         val = LocatedToken.Create (ref_line, col);
3111                                         if (peek_char () == '='){
3112                                                 get_char ();
3113                                                 return Token.OP_XOR_ASSIGN;
3114                                         }
3115                                         return Token.CARRET;
3116
3117                                 case ':':
3118                                         val = LocatedToken.Create (ref_line, col);
3119                                         if (peek_char () == ':') {
3120                                                 get_char ();
3121                                                 return Token.DOUBLE_COLON;
3122                                         }
3123                                         return Token.COLON;
3124
3125                                 case '0': case '1': case '2': case '3': case '4':
3126                                 case '5': case '6': case '7': case '8': case '9':
3127                                         tokens_seen = true;
3128                                         return is_number (c);
3129
3130                                 case '\n': // white space
3131                                         any_token_seen |= tokens_seen;
3132                                         tokens_seen = false;
3133                                         comments_seen = false;
3134                                         continue;
3135
3136                                 case '.':
3137                                         tokens_seen = true;
3138                                         d = peek_char ();
3139                                         if (d >= '0' && d <= '9')
3140                                                 return is_number (c);
3141
3142                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3143                                         return Token.DOT;
3144
3145                                 case '#':
3146                                         if (tokens_seen || comments_seen) {
3147                                                 Eror_WrongPreprocessorLocation ();
3148                                                 return Token.ERROR;
3149                                         }
3150
3151                                         if (ParsePreprocessingDirective (true))
3152                                                 continue;
3153
3154                                         bool directive_expected = false;
3155                                         while ((c = get_char ()) != -1) {
3156                                                 if (col == 1) {
3157                                                         directive_expected = true;
3158                                                 } else if (!directive_expected) {
3159                                                         // TODO: Implement comment support for disabled code and uncomment this code
3160 //                                                      if (c == '#') {
3161 //                                                              Eror_WrongPreprocessorLocation ();
3162 //                                                              return Token.ERROR;
3163 //                                                      }
3164                                                         continue;
3165                                                 }
3166
3167                                                 if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
3168                                                         continue;
3169
3170                                                 if (c == '#') {
3171                                                         if (ParsePreprocessingDirective (false))
3172                                                                 break;
3173                                                 }
3174                                                 directive_expected = false;
3175                                         }
3176
3177                                         if (c != -1) {
3178                                                 tokens_seen = false;
3179                                                 continue;
3180                                         }
3181
3182                                         return Token.EOF;
3183
3184                                 case '"':
3185                                         return consume_string (false);
3186
3187                                 case '\'':
3188                                         return TokenizeBackslash ();
3189
3190                                 case '@':
3191                                         c = get_char ();
3192                                         if (c == '"') {
3193                                                 tokens_seen = true;
3194                                                 return consume_string (true);
3195                                         }
3196
3197                                         if (is_identifier_start_character (c)){
3198                                                 return consume_identifier (c, true);
3199                                         }
3200
3201                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3202                                         return Token.ERROR;
3203
3204                                 case EvalStatementParserCharacter:
3205                                         return Token.EVAL_STATEMENT_PARSER;
3206                                 case EvalCompilationUnitParserCharacter:
3207                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3208                                 case EvalUsingDeclarationsParserCharacter:
3209                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3210                                 }
3211
3212                                 if (is_identifier_start_character (c)) {
3213                                         tokens_seen = true;
3214                                         return consume_identifier (c);
3215                                 }
3216
3217                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3218                         }
3219
3220                         if (CompleteOnEOF){
3221                                 if (generated)
3222                                         return Token.COMPLETE_COMPLETION;
3223
3224                                 generated = true;
3225                                 return Token.GENERATE_COMPLETION;
3226                         }
3227
3228
3229                         return Token.EOF;
3230                 }
3231
3232                 int TokenizeBackslash ()
3233                 {
3234                         int c = get_char ();
3235                         tokens_seen = true;
3236                         if (c == '\'') {
3237                                 val = new CharLiteral ((char) c, Location);
3238                                 Report.Error (1011, Location, "Empty character literal");
3239                                 return Token.LITERAL;
3240                         }
3241
3242                         if (c == '\r' || c == '\n') {
3243                                 Report.Error (1010, Location, "Newline in constant");
3244                                 return Token.ERROR;
3245                         }
3246
3247                         int d;
3248                         c = escape (c, out d);
3249                         if (c == -1)
3250                                 return Token.ERROR;
3251                         if (d != 0)
3252                                 throw new NotImplementedException ();
3253
3254                         val = new CharLiteral ((char) c, Location);
3255                         c = get_char ();
3256
3257                         if (c != '\'') {
3258                                 Report.Error (1012, Location, "Too many characters in character literal");
3259
3260                                 // Try to recover, read until newline or next "'"
3261                                 while ((c = get_char ()) != -1) {
3262                                         if (c == '\n' || c == '\'')
3263                                                 break;
3264                                 }
3265                         }
3266
3267                         return Token.LITERAL;
3268                 }
3269
3270                 int TokenizeLessThan ()
3271                 {
3272                         int d;
3273                         if (handle_typeof) {
3274                                 PushPosition ();
3275                                 if (parse_generic_dimension (out d)) {
3276                                         val = d;
3277                                         DiscardPosition ();
3278                                         return Token.GENERIC_DIMENSION;
3279                                 }
3280                                 PopPosition ();
3281                         }
3282
3283                         // Save current position and parse next token.
3284                         PushPosition ();
3285                         if (parse_less_than ()) {
3286                                 if (parsing_generic_declaration && token () != Token.DOT) {
3287                                         d = Token.OP_GENERICS_LT_DECL;
3288                                 } else {
3289                                         d = Token.OP_GENERICS_LT;
3290                                 }
3291                                 PopPosition ();
3292                                 return d;
3293                         }
3294
3295                         PopPosition ();
3296                         parsing_generic_less_than = 0;
3297
3298                         d = peek_char ();
3299                         if (d == '<') {
3300                                 get_char ();
3301                                 d = peek_char ();
3302
3303                                 if (d == '=') {
3304                                         get_char ();
3305                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3306                                 }
3307                                 return Token.OP_SHIFT_LEFT;
3308                         }
3309
3310                         if (d == '=') {
3311                                 get_char ();
3312                                 return Token.OP_LE;
3313                         }
3314                         return Token.OP_LT;
3315                 }
3316
3317                 //
3318                 // Handles one line xml comment
3319                 //
3320                 private void handle_one_line_xml_comment ()
3321                 {
3322                         int c;
3323                         while ((c = peek_char ()) == ' ')
3324                                 get_char (); // skip heading whitespaces.
3325                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3326                                 xml_comment_buffer.Append ((char) get_char ());
3327                         }
3328                         if (c == '\r' || c == '\n')
3329                                 xml_comment_buffer.Append (Environment.NewLine);
3330                 }
3331
3332                 //
3333                 // Remove heading "*" in Javadoc-like xml documentation.
3334                 //
3335                 private void update_formatted_doc_comment (int current_comment_start)
3336                 {
3337                         int length = xml_comment_buffer.Length - current_comment_start;
3338                         string [] lines = xml_comment_buffer.ToString (
3339                                 current_comment_start,
3340                                 length).Replace ("\r", "").Split ('\n');
3341
3342                         // The first line starts with /**, thus it is not target
3343                         // for the format check.
3344                         for (int i = 1; i < lines.Length; i++) {
3345                                 string s = lines [i];
3346                                 int idx = s.IndexOf ('*');
3347                                 string head = null;
3348                                 if (idx < 0) {
3349                                         if (i < lines.Length - 1)
3350                                                 return;
3351                                         head = s;
3352                                 } else
3353                                         head = s.Substring (0, idx);
3354                                 foreach (char c in head)
3355                                         if (c != ' ')
3356                                                 return;
3357                                 lines [i] = s.Substring (idx + 1);
3358                         }
3359                         xml_comment_buffer.Remove (current_comment_start, length);
3360                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3361                 }
3362
3363                 //
3364                 // Updates current comment location.
3365                 //
3366                 private void update_comment_location ()
3367                 {
3368                         if (current_comment_location.IsNull) {
3369                                 // "-2" is for heading "//" or "/*"
3370                                 current_comment_location =
3371                                         new Location (ref_line, hidden ? -1 : col - 2);
3372                         }
3373                 }
3374
3375                 //
3376                 // Checks if there was incorrect doc comments and raise
3377                 // warnings.
3378                 //
3379                 public void check_incorrect_doc_comment ()
3380                 {
3381                         if (xml_comment_buffer.Length > 0)
3382                                 warn_incorrect_doc_comment ();
3383                 }
3384
3385                 //
3386                 // Raises a warning when tokenizer found incorrect doccomment
3387                 // markup.
3388                 //
3389                 private void warn_incorrect_doc_comment ()
3390                 {
3391                         if (doc_state != XmlCommentState.Error) {
3392                                 doc_state = XmlCommentState.Error;
3393                                 // in csc, it is 'XML comment is not placed on
3394                                 // a valid language element'. But that does not
3395                                 // make sense.
3396                                 Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
3397                         }
3398                 }
3399
3400                 //
3401                 // Consumes the saved xml comment lines (if any)
3402                 // as for current target member or type.
3403                 //
3404                 public string consume_doc_comment ()
3405                 {
3406                         if (xml_comment_buffer.Length > 0) {
3407                                 string ret = xml_comment_buffer.ToString ();
3408                                 reset_doc_comment ();
3409                                 return ret;
3410                         }
3411                         return null;
3412                 }
3413
3414                 Report Report {
3415                         get { return context.Report; }
3416                 }
3417
3418                 void reset_doc_comment ()
3419                 {
3420                         xml_comment_buffer.Length = 0;
3421                         current_comment_location = Location.Null;
3422                 }
3423
3424                 public void cleanup ()
3425                 {
3426                         if (ifstack != null && ifstack.Count >= 1) {
3427                                 int state = ifstack.Pop ();
3428                                 if ((state & REGION) != 0)
3429                                         Report.Error (1038, Location, "#endregion directive expected");
3430                                 else
3431                                         Report.Error (1027, Location, "Expected `#endif' directive");
3432                         }
3433                 }
3434         }
3435
3436         //
3437         // Indicates whether it accepts XML documentation or not.
3438         //
3439         public enum XmlCommentState {
3440                 // comment is allowed in this state.
3441                 Allowed,
3442                 // comment is not allowed in this state.
3443                 NotAllowed,
3444                 // once comments appeared when it is NotAllowed, then the
3445                 // state is changed to it, until the state is changed to
3446                 // .Allowed.
3447                 Error
3448         }
3449 }
3450