mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Licensed under the terms of the GNU GPL
   9 //
  10 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // (C) 2004 Novell, Inc
  12 //
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections;
  18 using System.IO;
  19 using System.Globalization;
  20 using System.Reflection;
  21
  22 namespace Mono.CSharp
  23 {
  24         /// <summary>
  25         ///    Tokenizer for C# source code.
  26         /// </summary>
  27
  28         public class Tokenizer : yyParser.yyInput
  29         {
  30                 SeekableStreamReader reader;
  31                 SourceFile ref_name;
  32                 SourceFile file_name;
  33                 int ref_line = 1;
  34                 int line = 1;
  35                 int col = 0;
  36                 int previous_col;
  37                 int current_token;
  38                 bool handle_get_set = false;
  39                 bool handle_remove_add = false;
  40                 bool handle_assembly = false;
  41                 bool handle_constraints = false;
  42                 bool handle_typeof = false;
  43                 bool query_parsing;
  44                 Location current_location;
  45                 Location current_comment_location = Location.Null;
  46                 ArrayList escapedIdentifiers = new ArrayList ();
  47
  48                 static bool IsLinqEnabled {
  49                         get {
  50                                 return RootContext.Version == LanguageVersion.LINQ;
  51                         }
  52                 }
  53
  54                 //
  55                 // XML documentation buffer. The save point is used to divide
  56                 // comments on types and comments on members.
  57                 //
  58                 StringBuilder xml_comment_buffer;
  59
  60                 //
  61                 // See comment on XmlCommentState enumeration.
  62                 //
  63                 XmlCommentState xmlDocState = XmlCommentState.Allowed;
  64
  65                 //
  66                 // Whether tokens have been seen on this line
  67                 //
  68                 bool tokens_seen = false;
  69
  70                 //
  71                 // Whether a token has been seen on the file
  72                 // This is needed because `define' is not allowed to be used
  73                 // after a token has been seen.
  74                 //
  75                 bool any_token_seen = false;
  76
  77                 static Hashtable tokenValues;
  78                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
  79
  80                 private static Hashtable TokenValueName
  81                 {
  82                         get {
  83                                 if (tokenValues == null)
  84                                         tokenValues = GetTokenValueNameHash ();
  85
  86                                 return tokenValues;
  87                         }
  88                 }
  89
  90                 private static Hashtable GetTokenValueNameHash ()
  91                 {
  92                         Type t = typeof (Token);
  93                         FieldInfo [] fields = t.GetFields ();
  94                         Hashtable hash = new Hashtable ();
  95                         foreach (FieldInfo field in fields) {
  96                                 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))
  97                                         hash.Add (field.GetValue (null), field.Name);
  98                         }
  99                         return hash;
 100                 }
 101
 102                 //
 103                 // Returns a verbose representation of the current location
 104                 //
 105                 public string location {
 106                         get {
 107                                 string det;
 108
 109                                 if (current_token == Token.ERROR)
 110                                         det = "detail: " + error_details;
 111                                 else
 112                                         det = "";
 113
 114                                 // return "Line:     "+line+" Col: "+col + "\n" +
 115                                 //       "VirtLine: "+ref_line +
 116                                 //       " Token: "+current_token + " " + det;
 117                                 string current_token_name = TokenValueName [current_token] as string;
 118                                 if (current_token_name == null)
 119                                         current_token_name = current_token.ToString ();
 120
 121                                 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,
 122                                                                                        ref_line,
 123                                                                                        col,
 124                                                                                        current_token_name,
 125                                                                                        det);
 126                         }
 127                 }
 128
 129                 public bool PropertyParsing {
 130                         get {
 131                                 return handle_get_set;
 132                         }
 133
 134                         set {
 135                                 handle_get_set = value;
 136                         }
 137                 }
 138
 139                 public bool AssemblyTargetParsing {
 140                         get {
 141                                 return handle_assembly;
 142                         }
 143
 144                         set {
 145                                 handle_assembly = value;
 146                         }
 147                 }
 148
 149                 public bool EventParsing {
 150                         get {
 151                                 return handle_remove_add;
 152                         }
 153
 154                         set {
 155                                 handle_remove_add = value;
 156                         }
 157                 }
 158
 159                 public bool ConstraintsParsing {
 160                         get {
 161                                 return handle_constraints;
 162                         }
 163
 164                         set {
 165                                 handle_constraints = value;
 166                         }
 167                 }
 168
 169                 public bool TypeOfParsing {
 170                         get {
 171                                 return handle_typeof;
 172                         }
 173
 174                         set {
 175                                 handle_typeof = value;
 176                         }
 177                 }
 178
 179                 public bool QueryParsing {
 180                         set {
 181                                 query_parsing = value;
 182                         }
 183                 }
 184
 185                 public XmlCommentState doc_state {
 186                         get { return xmlDocState; }
 187                         set {
 188                                 if (value == XmlCommentState.Allowed) {
 189                                         check_incorrect_doc_comment ();
 190                                         reset_doc_comment ();
 191                                 }
 192                                 xmlDocState = value;
 193                         }
 194                 }
 195
 196                 public bool IsEscapedIdentifier (Location loc)
 197                 {
 198                         foreach (LocatedToken lt in escapedIdentifiers)
 199                                 if (lt.Location.Equals (loc))
 200                                         return true;
 201                         return false;
 202                 }
 203
 204                 //
 205                 // Class variables
 206                 //
 207                 static CharArrayHashtable[] keywords;
 208                 static Hashtable keywordStrings;
 209                 static NumberStyles styles;
 210                 static NumberFormatInfo csharp_format_info;
 211
 212                 //
 213                 // Values for the associated token returned
 214                 //
 215                 int putback_char;
 216                 Object val;
 217
 218                 //
 219                 // Pre-processor
 220                 //
 221                 Hashtable defines;
 222
 223                 const int TAKING        = 1;
 224                 const int ELSE_SEEN     = 4;
 225                 const int PARENT_TAKING = 8;
 226                 const int REGION        = 16;
 227
 228                 //
 229                 // pre-processor if stack state:
 230                 //
 231                 Stack ifstack;
 232
 233                 static System.Text.StringBuilder string_builder;
 234
 235                 const int max_id_size = 512;
 236                 static char [] id_builder = new char [max_id_size];
 237
 238                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 239
 240                 const int max_number_size = 512;
 241                 static char [] number_builder = new char [max_number_size];
 242                 static int number_pos;
 243
 244                 //
 245                 // Details about the error encoutered by the tokenizer
 246                 //
 247                 string error_details;
 248
 249                 public string error {
 250                         get {
 251                                 return error_details;
 252                         }
 253                 }
 254
 255                 public int Line {
 256                         get {
 257                                 return ref_line;
 258                         }
 259                 }
 260
 261                 public int Col {
 262                         get {
 263                                 return col;
 264                         }
 265                 }
 266
 267                 //
 268                 // This is used when the tokenizer needs to save
 269                 // the current position as it needs to do some parsing
 270                 // on its own to deamiguate a token in behalf of the
 271                 // parser.
 272                 //
 273                 Stack position_stack = new Stack (2);
 274                 class Position {
 275                         public int position;
 276                         public int ref_line;
 277                         public int col;
 278                         public int putback_char;
 279                         public int previous_col;
 280                         public Stack ifstack;
 281                         public int parsing_generic_less_than;
 282                         public int current_token;
 283
 284                         public Position (Tokenizer t)
 285                         {
 286                                 position = t.reader.Position;
 287                                 ref_line = t.ref_line;
 288                                 col = t.col;
 289                                 putback_char = t.putback_char;
 290                                 previous_col = t.previous_col;
 291                                 if (t.ifstack != null && t.ifstack.Count != 0)
 292                                         ifstack = (Stack)t.ifstack.Clone ();
 293                                 parsing_generic_less_than = t.parsing_generic_less_than;
 294                                 current_token = t.current_token;
 295                         }
 296                 }
 297
 298                 public void PushPosition ()
 299                 {
 300                         position_stack.Push (new Position (this));
 301                 }
 302
 303                 public void PopPosition ()
 304                 {
 305                         Position p = (Position) position_stack.Pop ();
 306
 307                         reader.Position = p.position;
 308                         ref_line = p.ref_line;
 309                         col = p.col;
 310                         putback_char = p.putback_char;
 311                         previous_col = p.previous_col;
 312                         ifstack = p.ifstack;
 313                         parsing_generic_less_than = p.parsing_generic_less_than;
 314                         current_token = p.current_token;
 315                 }
 316
 317                 // Do not reset the position, ignore it.
 318                 public void DiscardPosition ()
 319                 {
 320                         position_stack.Pop ();
 321                 }
 322
 323                 static void AddKeyword (string kw, int token)
 324                 {
 325                         keywordStrings.Add (kw, kw);
 326                         if (keywords [kw.Length] == null) {
 327                                 keywords [kw.Length] = new CharArrayHashtable (kw.Length);
 328                         }
 329                         keywords [kw.Length] [kw.ToCharArray ()] = token;
 330                 }
 331
 332                 static void InitTokens ()
 333                 {
 334                         keywordStrings = new Hashtable ();
 335                         keywords = new CharArrayHashtable [64];
 336
 337                         AddKeyword ("__arglist", Token.ARGLIST);
 338                         AddKeyword ("abstract", Token.ABSTRACT);
 339                         AddKeyword ("as", Token.AS);
 340                         AddKeyword ("add", Token.ADD);
 341                         AddKeyword ("assembly", Token.ASSEMBLY);
 342                         AddKeyword ("base", Token.BASE);
 343                         AddKeyword ("bool", Token.BOOL);
 344                         AddKeyword ("break", Token.BREAK);
 345                         AddKeyword ("byte", Token.BYTE);
 346                         AddKeyword ("case", Token.CASE);
 347                         AddKeyword ("catch", Token.CATCH);
 348                         AddKeyword ("char", Token.CHAR);
 349                         AddKeyword ("checked", Token.CHECKED);
 350                         AddKeyword ("class", Token.CLASS);
 351                         AddKeyword ("const", Token.CONST);
 352                         AddKeyword ("continue", Token.CONTINUE);
 353                         AddKeyword ("decimal", Token.DECIMAL);
 354                         AddKeyword ("default", Token.DEFAULT);
 355                         AddKeyword ("delegate", Token.DELEGATE);
 356                         AddKeyword ("do", Token.DO);
 357                         AddKeyword ("double", Token.DOUBLE);
 358                         AddKeyword ("else", Token.ELSE);
 359                         AddKeyword ("enum", Token.ENUM);
 360                         AddKeyword ("event", Token.EVENT);
 361                         AddKeyword ("explicit", Token.EXPLICIT);
 362                         AddKeyword ("extern", Token.EXTERN);
 363                         AddKeyword ("false", Token.FALSE);
 364                         AddKeyword ("finally", Token.FINALLY);
 365                         AddKeyword ("fixed", Token.FIXED);
 366                         AddKeyword ("float", Token.FLOAT);
 367                         AddKeyword ("for", Token.FOR);
 368                         AddKeyword ("foreach", Token.FOREACH);
 369                         AddKeyword ("goto", Token.GOTO);
 370                         AddKeyword ("get", Token.GET);
 371                         AddKeyword ("if", Token.IF);
 372                         AddKeyword ("implicit", Token.IMPLICIT);
 373                         AddKeyword ("in", Token.IN);
 374                         AddKeyword ("int", Token.INT);
 375                         AddKeyword ("interface", Token.INTERFACE);
 376                         AddKeyword ("internal", Token.INTERNAL);
 377                         AddKeyword ("is", Token.IS);
 378                         AddKeyword ("lock", Token.LOCK);
 379                         AddKeyword ("long", Token.LONG);
 380                         AddKeyword ("namespace", Token.NAMESPACE);
 381                         AddKeyword ("new", Token.NEW);
 382                         AddKeyword ("null", Token.NULL);
 383                         AddKeyword ("object", Token.OBJECT);
 384                         AddKeyword ("operator", Token.OPERATOR);
 385                         AddKeyword ("out", Token.OUT);
 386                         AddKeyword ("override", Token.OVERRIDE);
 387                         AddKeyword ("params", Token.PARAMS);
 388                         AddKeyword ("private", Token.PRIVATE);
 389                         AddKeyword ("protected", Token.PROTECTED);
 390                         AddKeyword ("public", Token.PUBLIC);
 391                         AddKeyword ("readonly", Token.READONLY);
 392                         AddKeyword ("ref", Token.REF);
 393                         AddKeyword ("remove", Token.REMOVE);
 394                         AddKeyword ("return", Token.RETURN);
 395                         AddKeyword ("sbyte", Token.SBYTE);
 396                         AddKeyword ("sealed", Token.SEALED);
 397                         AddKeyword ("set", Token.SET);
 398                         AddKeyword ("short", Token.SHORT);
 399                         AddKeyword ("sizeof", Token.SIZEOF);
 400                         AddKeyword ("stackalloc", Token.STACKALLOC);
 401                         AddKeyword ("static", Token.STATIC);
 402                         AddKeyword ("string", Token.STRING);
 403                         AddKeyword ("struct", Token.STRUCT);
 404                         AddKeyword ("switch", Token.SWITCH);
 405                         AddKeyword ("this", Token.THIS);
 406                         AddKeyword ("throw", Token.THROW);
 407                         AddKeyword ("true", Token.TRUE);
 408                         AddKeyword ("try", Token.TRY);
 409                         AddKeyword ("typeof", Token.TYPEOF);
 410                         AddKeyword ("uint", Token.UINT);
 411                         AddKeyword ("ulong", Token.ULONG);
 412                         AddKeyword ("unchecked", Token.UNCHECKED);
 413                         AddKeyword ("unsafe", Token.UNSAFE);
 414                         AddKeyword ("ushort", Token.USHORT);
 415                         AddKeyword ("using", Token.USING);
 416                         AddKeyword ("virtual", Token.VIRTUAL);
 417                         AddKeyword ("void", Token.VOID);
 418                         AddKeyword ("volatile", Token.VOLATILE);
 419                         AddKeyword ("while", Token.WHILE);
 420                         AddKeyword ("partial", Token.PARTIAL);
 421 #if GMCS_SOURCE
 422                         AddKeyword ("where", Token.WHERE);
 423 #endif
 424                 }
 425
 426 #if GMCS_SOURCE
 427                 public static void InitializeLinqKeywords ()
 428                 {
 429                         AddKeyword ("from", Token.FROM);
 430                         AddKeyword ("join", Token.JOIN);
 431                         AddKeyword ("on", Token.ON);
 432                         AddKeyword ("equals", Token.EQUALS);
 433                         AddKeyword ("select", Token.SELECT);
 434                         AddKeyword ("group", Token.GROUP);
 435                         AddKeyword ("by", Token.BY);
 436                         AddKeyword ("let", Token.LET);
 437                         AddKeyword ("orderby", Token.ORDERBY);
 438                         AddKeyword ("ascending", Token.ASCENDING);
 439                         AddKeyword ("descending", Token.DESCENDING);
 440                         AddKeyword ("into", Token.INTO);
 441                 }
 442 #endif
 443
 444                 //
 445                 // Class initializer
 446                 //
 447                 static Tokenizer ()
 448                 {
 449                         Reset ();
 450                 }
 451
 452                 public static void Reset ()
 453                 {
 454                         InitTokens ();
 455                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 456                         styles = NumberStyles.Float;
 457
 458                         string_builder = new System.Text.StringBuilder ();
 459                 }
 460
 461                 int GetKeyword (char[] id, int id_len)
 462                 {
 463                         /*
 464                          * Keywords are stored in an array of hashtables grouped by their
 465                          * length.
 466                          */
 467
 468                         if ((id_len >= keywords.Length) || (keywords [id_len] == null))
 469                                 return -1;
 470                         object o = keywords [id_len] [id];
 471
 472                         if (o == null)
 473                                 return -1;
 474
 475                         int res = (int) o;
 476
 477                         if (handle_get_set == false && (res == Token.GET || res == Token.SET))
 478                                 return -1;
 479                         if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
 480                                 return -1;
 481                         if (handle_assembly == false && res == Token.ASSEMBLY)
 482                                 return -1;
 483 #if GMCS_SOURCE
 484                         if (IsLinqEnabled) {
 485                                 if (res == Token.FROM &&
 486                                         (current_token == Token.ASSIGN || current_token == Token.OPEN_BRACKET ||
 487                                          current_token == Token.RETURN || current_token == Token.IN)) {
 488                                         query_parsing = true;
 489                                         return res;
 490                                 }
 491
 492                                 if (!query_parsing && res > Token.QUERY_FIRST_TOKEN && res < Token.QUERY_LAST_TOKEN)
 493                                         return -1;
 494
 495                                 return res;
 496                         }
 497
 498                         if (!handle_constraints && res == Token.WHERE)
 499                                 return -1;
 500 #endif
 501                         return res;
 502
 503                 }
 504
 505                 public Location Location {
 506                         get { return current_location; }
 507                 }
 508
 509                 void define (string def)
 510                 {
 511                         if (!RootContext.AllDefines.Contains (def)){
 512                                 RootContext.AllDefines [def] = true;
 513                         }
 514                         if (defines.Contains (def))
 515                                 return;
 516                         defines [def] = true;
 517                 }
 518
 519                 public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)
 520                 {
 521                         this.ref_name = file;
 522                         this.file_name = file;
 523                         reader = input;
 524
 525                         putback_char = -1;
 526
 527                         if (defs != null){
 528                                 defines = new Hashtable ();
 529                                 foreach (string def in defs)
 530                                         define (def);
 531                         }
 532
 533                         xml_comment_buffer = new StringBuilder ();
 534
 535                         //
 536                         // FIXME: This could be `Location.Push' but we have to
 537                         // find out why the MS compiler allows this
 538                         //
 539                         Mono.CSharp.Location.Push (file);
 540                 }
 541
 542                 static bool is_identifier_start_character (char c)
 543                 {
 544                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter (c);
 545                 }
 546
 547                 static bool is_identifier_part_character (char c)
 548                 {
 549                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') ||
 550                                 Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 551                 }
 552
 553                 public static bool IsKeyword (string s)
 554                 {
 555                         return keywordStrings [s] != null;
 556                 }
 557
 558                 public static bool IsValidIdentifier (string s)
 559                 {
 560                         if (s == null || s.Length == 0)
 561                                 return false;
 562
 563                         if (!is_identifier_start_character (s [0]))
 564                                 return false;
 565
 566                         for (int i = 1; i < s.Length; i ++)
 567                                 if (! is_identifier_part_character (s [i]))
 568                                         return false;
 569
 570                         return true;
 571                 }
 572
 573                 bool parse_generic_dimension (out int dimension)
 574                 {
 575                         dimension = 1;
 576
 577                 again:
 578                         int the_token = token ();
 579                         if (the_token == Token.OP_GENERICS_GT)
 580                                 return true;
 581                         else if (the_token == Token.COMMA) {
 582                                 dimension++;
 583                                 goto again;
 584                         }
 585
 586                         return false;
 587                 }
 588
 589                 bool parse_less_than ()
 590                 {
 591                 start:
 592                         int the_token = token ();
 593                         if (the_token == Token.OPEN_BRACKET) {
 594                                 do {
 595                                         the_token = token ();
 596                                 } while (the_token != Token.CLOSE_BRACKET);
 597                                 the_token = token ();
 598                         }
 599                         switch (the_token) {
 600                         case Token.IDENTIFIER:
 601                         case Token.OBJECT:
 602                         case Token.STRING:
 603                         case Token.BOOL:
 604                         case Token.DECIMAL:
 605                         case Token.FLOAT:
 606                         case Token.DOUBLE:
 607                         case Token.SBYTE:
 608                         case Token.BYTE:
 609                         case Token.SHORT:
 610                         case Token.USHORT:
 611                         case Token.INT:
 612                         case Token.UINT:
 613                         case Token.LONG:
 614                         case Token.ULONG:
 615                         case Token.CHAR:
 616                         case Token.VOID:
 617                                 break;
 618
 619                         default:
 620                                 return false;
 621                         }
 622                 again:
 623                         the_token = token ();
 624
 625                         if (the_token == Token.OP_GENERICS_GT)
 626                                 return true;
 627                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
 628                                 goto start;
 629                         else if (the_token == Token.INTERR || the_token == Token.STAR)
 630                                 goto again;
 631                         else if (the_token == Token.OP_GENERICS_LT) {
 632                                 if (!parse_less_than ())
 633                                         return false;
 634                                 goto again;
 635                         } else if (the_token == Token.OPEN_BRACKET) {
 636                         rank_specifiers:
 637                                 the_token = token ();
 638                                 if (the_token == Token.CLOSE_BRACKET)
 639                                         goto again;
 640                                 else if (the_token == Token.COMMA)
 641                                         goto rank_specifiers;
 642                                 return false;
 643                         }
 644
 645                         return false;
 646                 }
 647
 648 #if GMCS_SOURCE
 649                 public void PutbackNullable ()
 650                 {
 651                         if (nullable_pos < 0)
 652                                 throw new Exception ();
 653
 654                         current_token = -1;
 655                         val = null;
 656                         reader.Position = nullable_pos;
 657
 658                         putback_char = '?';
 659                 }
 660
 661                 public void PutbackCloseParens ()
 662                 {
 663                         putback_char = ')';
 664                 }
 665
 666
 667                 int nullable_pos = -1;
 668
 669                 public void CheckNullable (bool is_nullable)
 670                 {
 671                         if (is_nullable)
 672                                 nullable_pos = reader.Position;
 673                         else
 674                                 nullable_pos = -1;
 675                 }
 676 #endif
 677
 678                 public int peek_token ()
 679                 {
 680                         int the_token;
 681
 682                         PushPosition ();
 683                         the_token = token ();
 684                         PopPosition ();
 685
 686                         return the_token;
 687                 }
 688
 689                 bool parse_namespace_or_typename (int next)
 690                 {
 691                         if (next == -1)
 692                                 next = peek_token ();
 693                         while (next == Token.IDENTIFIER){
 694                                 token ();
 695                           again:
 696                                 next = peek_token ();
 697                                 if (next == Token.DOT || next == Token.DOUBLE_COLON){
 698                                         token ();
 699                                         next = peek_token ();
 700                                         continue;
 701                                 }
 702                                 if (next == Token.OP_GENERICS_LT){
 703                                         token ();
 704                                         if (!parse_less_than ())
 705                                                 return false;
 706                                         goto again;
 707                                 }
 708                                 return true;
 709                         }
 710
 711                         return false;
 712                 }
 713
 714                 bool is_simple_type (int token)
 715                 {
 716                         return  (token == Token.BOOL ||
 717                                  token == Token.DECIMAL ||
 718                                  token == Token.SBYTE ||
 719                                  token == Token.BYTE ||
 720                                  token == Token.SHORT ||
 721                                  token == Token.USHORT ||
 722                                  token == Token.INT ||
 723                                  token == Token.UINT ||
 724                                  token == Token.LONG ||
 725                                  token == Token.ULONG ||
 726                                  token == Token.CHAR ||
 727                                  token == Token.FLOAT ||
 728                                  token == Token.DOUBLE);
 729                 }
 730
 731                 bool is_builtin_reference_type (int token)
 732                 {
 733                         return (token == Token.OBJECT || token == Token.STRING);
 734                 }
 735
 736                 bool parse_opt_rank (int next)
 737                 {
 738                         while (true){
 739                                 if (next != Token.OPEN_BRACKET)
 740                                         return true;
 741
 742                                 token ();
 743                                 while (true){
 744                                         next = token ();
 745                                         if (next == Token.CLOSE_BRACKET){
 746                                                 next = peek_token ();
 747                                                 break;
 748                                         }
 749                                         if (next == Token.COMMA)
 750                                                 continue;
 751
 752                                         return false;
 753                                 }
 754                         }
 755                 }
 756
 757                 bool parse_type ()
 758                 {
 759                         int next = peek_token ();
 760
 761                         if (is_simple_type (next)){
 762                                 token ();
 763                                 next = peek_token ();
 764                                 if (next == Token.INTERR)
 765                                         token ();
 766                                 return parse_opt_rank (peek_token ());
 767                         }
 768                         if (parse_namespace_or_typename (next)){
 769                                 next = peek_token ();
 770                                 if (next == Token.INTERR)
 771                                         token ();
 772                                 return parse_opt_rank (peek_token ());
 773                         } else if (is_builtin_reference_type (next)){
 774                                 token ();
 775                                 return parse_opt_rank (peek_token ());
 776                         }
 777
 778                         return false;
 779                 }
 780
 781                 //
 782                 // Invoked after '(' has been seen and tries to parse:
 783                 // type identifier [, type identifier]*
 784                 //
 785                 // if this is the case, instead of returning an
 786                 // OPEN_PARENS token we return a special token that
 787                 // triggers lambda parsing.
 788                 //
 789                 // This is needed because we can not introduce the
 790                 // explicitly_typed_lambda_parameter_list after a '(' in the
 791                 // grammar without introducing reduce/reduce conflicts.
 792                 //
 793                 // We need to parse a type and if it is followed by an
 794                 // identifier, we know it has to be parsed as a lambda
 795                 // expression.
 796                 //
 797                 // the type expression can be prefixed with `ref' or `out'
 798                 //
 799                 public bool parse_lambda_parameters ()
 800                 {
 801                         while (true){
 802                                 int next = peek_token ();
 803
 804                                 if (next == Token.REF || next == Token.OUT)
 805                                         token ();
 806
 807                                 if (parse_type ()){
 808                                         next = peek_token ();
 809                                         if (next == Token.IDENTIFIER){
 810                                                 token ();
 811                                                 next = peek_token ();
 812                                                 if (next == Token.COMMA){
 813                                                         token ();
 814                                                         continue;
 815                                                 }
 816                                                 if (next == Token.CLOSE_PARENS)
 817                                                         return true;
 818                                         }
 819                                 }
 820                                 return false;
 821                         }
 822                 }
 823
 824                 int parsing_generic_less_than = 0;
 825
 826                 int is_punct (char c, ref bool doread)
 827                 {
 828                         int d;
 829                         int t;
 830
 831                         doread = false;
 832
 833                         switch (c){
 834                         case '{':
 835                                 val = Location;
 836                                 return Token.OPEN_BRACE;
 837                         case '}':
 838                                 val = Location;
 839                                 return Token.CLOSE_BRACE;
 840                         case '[':
 841                                 // To block doccomment inside attribute declaration.
 842                                 if (doc_state == XmlCommentState.Allowed)
 843                                         doc_state = XmlCommentState.NotAllowed;
 844                                 return Token.OPEN_BRACKET;
 845                         case ']':
 846                                 return Token.CLOSE_BRACKET;
 847                         case '(':
 848                                 if (IsLinqEnabled){
 849                                         PushPosition ();
 850                                         bool have_lambda_parameter = parse_lambda_parameters ();
 851                                         PopPosition ();
 852
 853                                         if (have_lambda_parameter)
 854                                                 return Token.OPEN_PARENS_LAMBDA;
 855                                         else
 856                                                 return Token.OPEN_PARENS;
 857                                 } else
 858                                         return Token.OPEN_PARENS;
 859                         case ')': {
 860                                 if (deambiguate_close_parens == 0)
 861                                         return Token.CLOSE_PARENS;
 862
 863                                 --deambiguate_close_parens;
 864
 865                                 PushPosition ();
 866
 867                                 int new_token = xtoken ();
 868
 869                                 PopPosition ();
 870
 871                                 if (new_token == Token.OPEN_PARENS)
 872                                         return Token.CLOSE_PARENS_OPEN_PARENS;
 873                                 else if (new_token == Token.MINUS)
 874                                         return Token.CLOSE_PARENS_MINUS;
 875                                 else if (IsCastToken (new_token))
 876                                         return Token.CLOSE_PARENS_CAST;
 877                                 else
 878                                         return Token.CLOSE_PARENS_NO_CAST;
 879                         }
 880
 881                         case ',':
 882                                 return Token.COMMA;
 883                         case ';':
 884                                 val = Location;
 885                                 return Token.SEMICOLON;
 886                         case '~':
 887                                 val = Location;
 888                                 return Token.TILDE;
 889                         case '?':
 890                                 return Token.INTERR;
 891                         }
 892 #if GMCS_SOURCE
 893                         if (c == '<') {
 894                                 if (parsing_generic_less_than++ > 0)
 895                                         return Token.OP_GENERICS_LT;
 896
 897                                 if (handle_typeof) {
 898                                         int dimension;
 899                                         PushPosition ();
 900                                         if (parse_generic_dimension (out dimension)) {
 901                                                 val = dimension;
 902                                                 DiscardPosition ();
 903                                                 return Token.GENERIC_DIMENSION;
 904                                         }
 905                                         PopPosition ();
 906                                 }
 907
 908                                 // Save current position and parse next token.
 909                                 PushPosition ();
 910                                 bool is_generic_lt = parse_less_than ();
 911                                 PopPosition ();
 912
 913                                 if (is_generic_lt) {
 914                                         return Token.OP_GENERICS_LT;
 915                                 } else
 916                                         parsing_generic_less_than = 0;
 917
 918                                 d = peekChar ();
 919                                 if (d == '<'){
 920                                         getChar ();
 921                                         d = peekChar ();
 922
 923                                         if (d == '='){
 924                                                 doread = true;
 925                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 926                                         }
 927                                         return Token.OP_SHIFT_LEFT;
 928                                 } else if (d == '='){
 929                                         doread = true;
 930                                         return Token.OP_LE;
 931                                 }
 932                                 return Token.OP_LT;
 933                         } else if (c == '>') {
 934                                 if (parsing_generic_less_than > 0) {
 935                                         parsing_generic_less_than--;
 936                                         return Token.OP_GENERICS_GT;
 937                                 }
 938
 939                                 d = peekChar ();
 940                                 if (d == '>'){
 941                                         getChar ();
 942                                         d = peekChar ();
 943
 944                                         if (d == '='){
 945                                                 doread = true;
 946                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 947                                         }
 948                                         return Token.OP_SHIFT_RIGHT;
 949                                 } else if (d == '='){
 950                                         doread = true;
 951                                         return Token.OP_GE;
 952                                 }
 953                                 return Token.OP_GT;
 954                         }
 955 #endif
 956                         d = peekChar ();
 957                         if (c == '+'){
 958
 959                                 if (d == '+') {
 960                                         val = Location;
 961                                         t = Token.OP_INC;
 962                                 }
 963                                 else if (d == '=')
 964                                         t = Token.OP_ADD_ASSIGN;
 965                                 else {
 966                                         val = Location;
 967                                         return Token.PLUS;
 968                                 }
 969                                 doread = true;
 970                                 return t;
 971                         }
 972                         if (c == '-'){
 973                                 if (d == '-') {
 974                                         val = Location;
 975                                         t = Token.OP_DEC;
 976                                 }
 977                                 else if (d == '=')
 978                                         t = Token.OP_SUB_ASSIGN;
 979                                 else if (d == '>')
 980                                         t = Token.OP_PTR;
 981                                 else {
 982                                         val = Location;
 983                                         return Token.MINUS;
 984                                 }
 985                                 doread = true;
 986                                 return t;
 987                         }
 988
 989                         if (c == '!'){
 990                                 if (d == '='){
 991                                         doread = true;
 992                                         return Token.OP_NE;
 993                                 }
 994                                 val = Location;
 995                                 return Token.BANG;
 996                         }
 997
 998                         if (c == '='){
 999                                 if (d == '='){
1000                                         doread = true;
1001                                         return Token.OP_EQ;
1002                                 }
1003                                 if (d == '>'){
1004                                         doread = true;
1005                                         val = Location;
1006                                         return Token.ARROW;
1007                                 }
1008                                 return Token.ASSIGN;
1009                         }
1010
1011                         if (c == '&'){
1012                                 if (d == '&'){
1013                                         doread = true;
1014                                         return Token.OP_AND;
1015                                 } else if (d == '='){
1016                                         doread = true;
1017                                         return Token.OP_AND_ASSIGN;
1018                                 }
1019                                 val = Location;
1020                                 return Token.BITWISE_AND;
1021                         }
1022
1023                         if (c == '|'){
1024                                 if (d == '|'){
1025                                         doread = true;
1026                                         return Token.OP_OR;
1027                                 } else if (d == '='){
1028                                         doread = true;
1029                                         return Token.OP_OR_ASSIGN;
1030                                 }
1031                                 return Token.BITWISE_OR;
1032                         }
1033
1034                         if (c == '*'){
1035                                 if (d == '='){
1036                                         doread = true;
1037                                         return Token.OP_MULT_ASSIGN;
1038                                 }
1039                                 val = Location;
1040                                 return Token.STAR;
1041                         }
1042
1043                         if (c == '/'){
1044                                 if (d == '='){
1045                                         doread = true;
1046                                         return Token.OP_DIV_ASSIGN;
1047                                 }
1048                                 return Token.DIV;
1049                         }
1050
1051                         if (c == '%'){
1052                                 if (d == '='){
1053                                         doread = true;
1054                                         return Token.OP_MOD_ASSIGN;
1055                                 }
1056                                 return Token.PERCENT;
1057                         }
1058
1059                         if (c == '^'){
1060                                 if (d == '='){
1061                                         doread = true;
1062                                         return Token.OP_XOR_ASSIGN;
1063                                 }
1064                                 return Token.CARRET;
1065                         }
1066
1067 #if !GMCS_SOURCE
1068                         if (c == '<'){
1069                                 if (d == '<'){
1070                                         getChar ();
1071                                         d = peekChar ();
1072
1073                                         if (d == '='){
1074                                                 doread = true;
1075                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
1076                                         }
1077                                         return Token.OP_SHIFT_LEFT;
1078                                 } else if (d == '='){
1079                                         doread = true;
1080                                         return Token.OP_LE;
1081                                 }
1082                                 return Token.OP_LT;
1083                         }
1084
1085                         if (c == '>'){
1086                                 if (d == '>'){
1087                                         getChar ();
1088                                         d = peekChar ();
1089
1090                                         if (d == '='){
1091                                                 doread = true;
1092                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
1093                                         }
1094                                         return Token.OP_SHIFT_RIGHT;
1095                                 } else if (d == '='){
1096                                         doread = true;
1097                                         return Token.OP_GE;
1098                                 }
1099                                 return Token.OP_GT;
1100                         }
1101 #endif
1102                         if (c == ':'){
1103                                 if (d == ':'){
1104                                         doread = true;
1105                                         return Token.DOUBLE_COLON;
1106                                 }
1107                                 val = Location;
1108                                 return Token.COLON;
1109                         }
1110
1111                         return Token.ERROR;
1112                 }
1113
1114                 int deambiguate_close_parens = 0;
1115
1116                 public void Deambiguate_CloseParens (object expression)
1117                 {
1118                         putback (')');
1119
1120                         // When any binary operation is used we are sure it is not a cast
1121                         if (expression is Binary)
1122                                 return;
1123
1124                         deambiguate_close_parens++;
1125                 }
1126
1127                 bool decimal_digits (int c)
1128                 {
1129                         int d;
1130                         bool seen_digits = false;
1131
1132                         if (c != -1){
1133                                 if (number_pos == max_number_size)
1134                                         Error_NumericConstantTooLong ();
1135                                 number_builder [number_pos++] = (char) c;
1136                         }
1137
1138                         //
1139                         // We use peekChar2, because decimal_digits needs to do a
1140                         // 2-character look-ahead (5.ToString for example).
1141                         //
1142                         while ((d = peekChar2 ()) != -1){
1143                                 if (d >= '0' && d <= '9'){
1144                                         if (number_pos == max_number_size)
1145                                                 Error_NumericConstantTooLong ();
1146                                         number_builder [number_pos++] = (char) d;
1147                                         getChar ();
1148                                         seen_digits = true;
1149                                 } else
1150                                         break;
1151                         }
1152
1153                         return seen_digits;
1154                 }
1155
1156                 static bool is_hex (int e)
1157                 {
1158                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1159                 }
1160
1161                 static int real_type_suffix (int c)
1162                 {
1163                         int t;
1164
1165                         switch (c){
1166                         case 'F': case 'f':
1167                                 t =  Token.LITERAL_FLOAT;
1168                                 break;
1169                         case 'D': case 'd':
1170                                 t = Token.LITERAL_DOUBLE;
1171                                 break;
1172                         case 'M': case 'm':
1173                                  t= Token.LITERAL_DECIMAL;
1174                                 break;
1175                         default:
1176                                 return Token.NONE;
1177                         }
1178                         return t;
1179                 }
1180
1181                 int integer_type_suffix (ulong ul, int c)
1182                 {
1183                         bool is_unsigned = false;
1184                         bool is_long = false;
1185
1186                         if (c != -1){
1187                                 bool scanning = true;
1188                                 do {
1189                                         switch (c){
1190                                         case 'U': case 'u':
1191                                                 if (is_unsigned)
1192                                                         scanning = false;
1193                                                 is_unsigned = true;
1194                                                 getChar ();
1195                                                 break;
1196
1197                                         case 'l':
1198                                                 if (!is_unsigned && (RootContext.WarningLevel >= 4)){
1199                                                         //
1200                                                         // if we have not seen anything in between
1201                                                         // report this error
1202                                                         //
1203                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1204                                                 }
1205                                                 //
1206                                                 // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
1207                                                 // work around that.
1208                                                 //
1209                                                 //goto case 'L';
1210                                                 if (is_long)
1211                                                         scanning = false;
1212                                                 is_long = true;
1213                                                 getChar ();
1214                                                 break;
1215
1216                                         case 'L':
1217                                                 if (is_long)
1218                                                         scanning = false;
1219                                                 is_long = true;
1220                                                 getChar ();
1221                                                 break;
1222
1223                                         default:
1224                                                 scanning = false;
1225                                                 break;
1226                                         }
1227                                         c = peekChar ();
1228                                 } while (scanning);
1229                         }
1230
1231                         if (is_long && is_unsigned){
1232                                 val = ul;
1233                                 return Token.LITERAL_INTEGER;
1234                         } else if (is_unsigned){
1235                                 // uint if possible, or ulong else.
1236
1237                                 if ((ul & 0xffffffff00000000) == 0)
1238                                         val = (uint) ul;
1239                                 else
1240                                         val = ul;
1241                         } else if (is_long){
1242                                 // long if possible, ulong otherwise
1243                                 if ((ul & 0x8000000000000000) != 0)
1244                                         val = ul;
1245                                 else
1246                                         val = (long) ul;
1247                         } else {
1248                                 // int, uint, long or ulong in that order
1249                                 if ((ul & 0xffffffff00000000) == 0){
1250                                         uint ui = (uint) ul;
1251
1252                                         if ((ui & 0x80000000) != 0)
1253                                                 val = ui;
1254                                         else
1255                                                 val = (int) ui;
1256                                 } else {
1257                                         if ((ul & 0x8000000000000000) != 0)
1258                                                 val = ul;
1259                                         else
1260                                                 val = (long) ul;
1261                                 }
1262                         }
1263                         return Token.LITERAL_INTEGER;
1264                 }
1265
1266                 //
1267                 // given `c' as the next char in the input decide whether
1268                 // we need to convert to a special type, and then choose
1269                 // the best representation for the integer
1270                 //
1271                 int adjust_int (int c)
1272                 {
1273                         try {
1274                                 if (number_pos > 9){
1275                                         ulong ul = (uint) (number_builder [0] - '0');
1276
1277                                         for (int i = 1; i < number_pos; i++){
1278                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1279                                         }
1280                                         return integer_type_suffix (ul, c);
1281                                 } else {
1282                                         uint ui = (uint) (number_builder [0] - '0');
1283
1284                                         for (int i = 1; i < number_pos; i++){
1285                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1286                                         }
1287                                         return integer_type_suffix (ui, c);
1288                                 }
1289                         } catch (OverflowException) {
1290                                 error_details = "Integral constant is too large";
1291                                 Report.Error (1021, Location, error_details);
1292                                 val = 0ul;
1293                                 return Token.LITERAL_INTEGER;
1294                         }
1295                         catch (FormatException) {
1296                                 Report.Error (1013, Location, "Invalid number");
1297                                 val = 0ul;
1298                                 return Token.LITERAL_INTEGER;
1299                         }
1300                 }
1301
1302                 int adjust_real (int t)
1303                 {
1304                         string s = new String (number_builder, 0, number_pos);
1305                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1306
1307                         switch (t){
1308                         case Token.LITERAL_DECIMAL:
1309                                 try {
1310                                         val = System.Decimal.Parse (s, styles, csharp_format_info);
1311                                 } catch (OverflowException) {
1312                                         val = 0m;
1313                                         Report.Error (594, Location, error_details, "decimal");
1314                                 }
1315                                 break;
1316                         case Token.LITERAL_FLOAT:
1317                                 try {
1318                                         val = float.Parse (s, styles, csharp_format_info);
1319                                 } catch (OverflowException) {
1320                                         val = 0.0f;
1321                                         Report.Error (594, Location, error_details, "float");
1322                                 }
1323                                 break;
1324
1325                         case Token.LITERAL_DOUBLE:
1326                         case Token.NONE:
1327                                 t = Token.LITERAL_DOUBLE;
1328                                 try {
1329                                         val = System.Double.Parse (s, styles, csharp_format_info);
1330                                 } catch (OverflowException) {
1331                                         val = 0.0;
1332                                         Report.Error (594, Location, error_details, "double");
1333                                 }
1334                                 break;
1335                         }
1336                         return t;
1337                 }
1338
1339                 int handle_hex ()
1340                 {
1341                         int d;
1342                         ulong ul;
1343
1344                         getChar ();
1345                         while ((d = peekChar ()) != -1){
1346                                 if (is_hex (d)){
1347                                         number_builder [number_pos++] = (char) d;
1348                                         getChar ();
1349                                 } else
1350                                         break;
1351                         }
1352
1353                         string s = new String (number_builder, 0, number_pos);
1354                         try {
1355                                 if (number_pos <= 8)
1356                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1357                                 else
1358                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1359                         } catch (OverflowException){
1360                                 error_details = "Integral constant is too large";
1361                                 Report.Error (1021, Location, error_details);
1362                                 val = 0ul;
1363                                 return Token.LITERAL_INTEGER;
1364                         }
1365                         catch (FormatException) {
1366                                 Report.Error (1013, Location, "Invalid number");
1367                                 val = 0ul;
1368                                 return Token.LITERAL_INTEGER;
1369                         }
1370
1371                         return integer_type_suffix (ul, peekChar ());
1372                 }
1373
1374                 //
1375                 // Invoked if we know we have .digits or digits
1376                 //
1377                 int is_number (int c)
1378                 {
1379                         bool is_real = false;
1380                         int type;
1381
1382                         number_pos = 0;
1383
1384                         if (c >= '0' && c <= '9'){
1385                                 if (c == '0'){
1386                                         int peek = peekChar ();
1387
1388                                         if (peek == 'x' || peek == 'X')
1389                                                 return handle_hex ();
1390                                 }
1391                                 decimal_digits (c);
1392                                 c = getChar ();
1393                         }
1394
1395                         //
1396                         // We need to handle the case of
1397                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1398                         //
1399                         if (c == '.'){
1400                                 if (decimal_digits ('.')){
1401                                         is_real = true;
1402                                         c = getChar ();
1403                                 } else {
1404                                         putback ('.');
1405                                         number_pos--;
1406                                         return adjust_int (-1);
1407                                 }
1408                         }
1409
1410                         if (c == 'e' || c == 'E'){
1411                                 is_real = true;
1412                                 if (number_pos == max_number_size)
1413                                         Error_NumericConstantTooLong ();
1414                                 number_builder [number_pos++] = 'e';
1415                                 c = getChar ();
1416
1417                                 if (c == '+'){
1418                                         if (number_pos == max_number_size)
1419                                                 Error_NumericConstantTooLong ();
1420                                         number_builder [number_pos++] = '+';
1421                                         c = -1;
1422                                 } else if (c == '-') {
1423                                         if (number_pos == max_number_size)
1424                                                 Error_NumericConstantTooLong ();
1425                                         number_builder [number_pos++] = '-';
1426                                         c = -1;
1427                                 } else {
1428                                         if (number_pos == max_number_size)
1429                                                 Error_NumericConstantTooLong ();
1430                                         number_builder [number_pos++] = '+';
1431                                 }
1432
1433                                 decimal_digits (c);
1434                                 c = getChar ();
1435                         }
1436
1437                         type = real_type_suffix (c);
1438                         if (type == Token.NONE && !is_real){
1439                                 putback (c);
1440                                 return adjust_int (c);
1441                         } else
1442                                 is_real = true;
1443
1444                         if (type == Token.NONE){
1445                                 putback (c);
1446                         }
1447
1448                         if (is_real)
1449                                 return adjust_real (type);
1450
1451                         Console.WriteLine ("This should not be reached");
1452                         throw new Exception ("Is Number should never reach this point");
1453                 }
1454
1455                 //
1456                 // Accepts exactly count (4 or 8) hex, no more no less
1457                 //
1458                 int getHex (int count, out bool error)
1459                 {
1460                         int i;
1461                         int total = 0;
1462                         int c;
1463                         int top = count != -1 ? count : 4;
1464
1465                         getChar ();
1466                         error = false;
1467                         for (i = 0; i < top; i++){
1468                                 c = getChar ();
1469
1470                                 if (c >= '0' && c <= '9')
1471                                         c = (int) c - (int) '0';
1472                                 else if (c >= 'A' && c <= 'F')
1473                                         c = (int) c - (int) 'A' + 10;
1474                                 else if (c >= 'a' && c <= 'f')
1475                                         c = (int) c - (int) 'a' + 10;
1476                                 else {
1477                                         error = true;
1478                                         return 0;
1479                                 }
1480
1481                                 total = (total * 16) + c;
1482                                 if (count == -1){
1483                                         int p = peekChar ();
1484                                         if (p == -1)
1485                                                 break;
1486                                         if (!is_hex ((char)p))
1487                                                 break;
1488                                 }
1489                         }
1490                         return total;
1491                 }
1492
1493                 int escape (int c)
1494                 {
1495                         bool error;
1496                         int d;
1497                         int v;
1498
1499                         d = peekChar ();
1500                         if (c != '\\')
1501                                 return c;
1502
1503                         switch (d){
1504                         case 'a':
1505                                 v = '\a'; break;
1506                         case 'b':
1507                                 v = '\b'; break;
1508                         case 'n':
1509                                 v = '\n'; break;
1510                         case 't':
1511                                 v = '\t'; break;
1512                         case 'v':
1513                                 v = '\v'; break;
1514                         case 'r':
1515                                 v = '\r'; break;
1516                         case '\\':
1517                                 v = '\\'; break;
1518                         case 'f':
1519                                 v = '\f'; break;
1520                         case '0':
1521                                 v = 0; break;
1522                         case '"':
1523                                 v = '"'; break;
1524                         case '\'':
1525                                 v = '\''; break;
1526                         case 'x':
1527                                 v = getHex (-1, out error);
1528                                 if (error)
1529                                         goto default;
1530                                 return v;
1531                         case 'u':
1532                                 v = getHex (4, out error);
1533                                 if (error)
1534                                         goto default;
1535                                 return v;
1536                         case 'U':
1537                                 v = getHex (8, out error);
1538                                 if (error)
1539                                         goto default;
1540                                 return v;
1541                         default:
1542                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1543                                 return d;
1544                         }
1545                         getChar ();
1546                         return v;
1547                 }
1548
1549                 int getChar ()
1550                 {
1551                         int x;
1552                         if (putback_char != -1) {
1553                                 x = putback_char;
1554                                 putback_char = -1;
1555                         } else
1556                                 x = reader.Read ();
1557                         if (x == '\n') {
1558                                 line++;
1559                                 ref_line++;
1560                                 previous_col = col;
1561                                 col = 0;
1562                         }
1563                         else
1564                                 col++;
1565                         return x;
1566                 }
1567
1568                 int peekChar ()
1569                 {
1570                         if (putback_char != -1)
1571                                 return putback_char;
1572                         putback_char = reader.Read ();
1573                         return putback_char;
1574                 }
1575
1576                 int peekChar2 ()
1577                 {
1578                         if (putback_char != -1)
1579                                 return putback_char;
1580                         return reader.Peek ();
1581                 }
1582
1583                 void putback (int c)
1584                 {
1585                         if (putback_char != -1){
1586                                 Console.WriteLine ("Col: " + col);
1587                                 Console.WriteLine ("Row: " + line);
1588                                 Console.WriteLine ("Name: " + ref_name.Name);
1589                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1590                                 throw new Exception ("This should not happen putback on putback");
1591                         }
1592                         if (c == '\n' || col == 0) {
1593                                 // It won't happen though.
1594                                 line--;
1595                                 ref_line--;
1596                                 col = previous_col;
1597                         }
1598                         else
1599                                 col--;
1600                         putback_char = c;
1601                 }
1602
1603                 public bool advance ()
1604                 {
1605                         return peekChar () != -1;
1606                 }
1607
1608                 public Object Value {
1609                         get {
1610                                 return val;
1611                         }
1612                 }
1613
1614                 public Object value ()
1615                 {
1616                         return val;
1617                 }
1618
1619                 static bool IsCastToken (int token)
1620                 {
1621                         switch (token) {
1622                         case Token.BANG:
1623                         case Token.TILDE:
1624                         case Token.IDENTIFIER:
1625                         case Token.LITERAL_INTEGER:
1626                         case Token.LITERAL_FLOAT:
1627                         case Token.LITERAL_DOUBLE:
1628                         case Token.LITERAL_DECIMAL:
1629                         case Token.LITERAL_CHARACTER:
1630                         case Token.LITERAL_STRING:
1631                         case Token.BASE:
1632                         case Token.CHECKED:
1633                         case Token.DELEGATE:
1634                         case Token.FALSE:
1635                         case Token.FIXED:
1636                         case Token.NEW:
1637                         case Token.NULL:
1638                         case Token.SIZEOF:
1639                         case Token.THIS:
1640                         case Token.THROW:
1641                         case Token.TRUE:
1642                         case Token.TYPEOF:
1643                         case Token.UNCHECKED:
1644                         case Token.UNSAFE:
1645 #if GMCS_SOURCE
1646                         case Token.DEFAULT:
1647 #endif
1648
1649                                 //
1650                                 // These can be part of a member access
1651                                 //
1652                         case Token.INT:
1653                         case Token.UINT:
1654                         case Token.SHORT:
1655                         case Token.USHORT:
1656                         case Token.LONG:
1657                         case Token.ULONG:
1658                         case Token.DOUBLE:
1659                         case Token.FLOAT:
1660                         case Token.CHAR:
1661                                 return true;
1662
1663                         default:
1664                                 return false;
1665                         }
1666                 }
1667
1668                 public int token ()
1669                 {
1670                         current_token = xtoken ();
1671
1672 #if GMCS_SOURCE
1673                         if (current_token != Token.DEFAULT)
1674                                 return current_token;
1675
1676                         PushPosition();
1677                         int c = xtoken();
1678                         if (c == -1)
1679                                 current_token = Token.ERROR;
1680                         else if (c == Token.OPEN_PARENS)
1681                                 current_token = Token.DEFAULT_OPEN_PARENS;
1682                         else if (c == Token.COLON)
1683                                 current_token = Token.DEFAULT_COLON;
1684                         else
1685                                 PopPosition();
1686 #endif
1687                         return current_token;
1688                 }
1689
1690                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1691
1692                 void get_cmd_arg (out string cmd, out string arg)
1693                 {
1694                         int c;
1695
1696                         tokens_seen = false;
1697                         arg = "";
1698                         static_cmd_arg.Length = 0;
1699
1700                         // skip over white space
1701                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1702                                 ;
1703
1704                         while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
1705                                 if (is_identifier_part_character ((char) c)){
1706                                         static_cmd_arg.Append ((char) c);
1707                                         c = getChar ();
1708                                 } else {
1709                                         putback (c);
1710                                         break;
1711                                 }
1712                         }
1713
1714                         cmd = static_cmd_arg.ToString ();
1715
1716                         if (c == '\n' || c == '\r'){
1717                                 return;
1718                         }
1719
1720                         // skip over white space
1721                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1722                                 ;
1723
1724                         if (c == '\n'){
1725                                 return;
1726                         } else if (c == '\r'){
1727                                 return;
1728                         } else if (c == -1){
1729                                 arg = "";
1730                                 return;
1731                         }
1732
1733                         static_cmd_arg.Length = 0;
1734                         static_cmd_arg.Append ((char) c);
1735
1736                         while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
1737                                 static_cmd_arg.Append ((char) c);
1738                         }
1739
1740                         arg = static_cmd_arg.ToString ();
1741                 }
1742
1743                 //
1744                 // Handles the #line directive
1745                 //
1746                 bool PreProcessLine (string arg)
1747                 {
1748                         if (arg.Length == 0)
1749                                 return false;
1750
1751                         if (arg == "default"){
1752                                 ref_line = line;
1753                                 ref_name = file_name;
1754                                 Location.Push (ref_name);
1755                                 return true;
1756                         } else if (arg == "hidden"){
1757                                 //
1758                                 // We ignore #line hidden
1759                                 //
1760                                 return true;
1761                         }
1762
1763                         try {
1764                                 int pos;
1765
1766                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1767                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1768                                         pos++;
1769
1770                                         char [] quotes = { '\"' };
1771
1772                                         string name = arg.Substring (pos). Trim (quotes);
1773                                         ref_name = Location.LookupFile (name);
1774                                         file_name.HasLineDirective = true;
1775                                         ref_name.HasLineDirective = true;
1776                                         Location.Push (ref_name);
1777                                 } else {
1778                                         ref_line = System.Int32.Parse (arg);
1779                                 }
1780                         } catch {
1781                                 return false;
1782                         }
1783
1784                         return true;
1785                 }
1786
1787                 //
1788                 // Handles #define and #undef
1789                 //
1790                 void PreProcessDefinition (bool is_define, string arg, bool caller_is_taking)
1791                 {
1792                         if (arg.Length == 0 || arg == "true" || arg == "false"){
1793                                 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
1794                                 return;
1795                         }
1796
1797                         if (arg.IndexOfAny (simple_whitespaces) != -1){
1798                                 Error_EndLineExpected ();
1799                                 return;
1800                         }
1801
1802                         if (!is_identifier_start_character (arg [0]))
1803                                 Report.Error (1001, Location, "Identifier expected: " + arg);
1804
1805                         foreach (char c in arg.Substring (1)){
1806                                 if (!is_identifier_part_character (c)){
1807                                         Report.Error (1001, Location, "Identifier expected: " + arg);
1808                                         return;
1809                                 }
1810                         }
1811
1812                         if (!caller_is_taking)
1813                                 return;
1814
1815                         if (is_define){
1816                                 if (defines == null)
1817                                         defines = new Hashtable ();
1818                                 define (arg);
1819                         } else {
1820                                 if (defines == null)
1821                                         return;
1822                                 if (defines.Contains (arg))
1823                                         defines.Remove (arg);
1824                         }
1825                 }
1826
1827                 /// <summary>
1828                 /// Handles #pragma directive
1829                 /// </summary>
1830                 void PreProcessPragma (string arg)
1831                 {
1832                         const string warning = "warning";
1833                         const string w_disable = "warning disable";
1834                         const string w_restore = "warning restore";
1835
1836                         if (arg == w_disable) {
1837                                 Report.RegisterWarningRegion (Location).WarningDisable (line);
1838                                 return;
1839                         }
1840
1841                         if (arg == w_restore) {
1842                                 Report.RegisterWarningRegion (Location).WarningEnable (line);
1843                                 return;
1844                         }
1845
1846                         if (arg.StartsWith (w_disable)) {
1847                                 int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
1848                                 foreach (int code in codes) {
1849                                         if (code != 0)
1850                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1851                                 }
1852                                 return;
1853                         }
1854
1855                         if (arg.StartsWith (w_restore)) {
1856                                 int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
1857                                 Hashtable w_table = Report.warning_ignore_table;
1858                                 foreach (int code in codes) {
1859                                         if (w_table != null && w_table.Contains (code))
1860                                                 Report.Warning (1635, 1, Location, String.Format ("Cannot restore warning `CS{0:0000}' because it was disabled globally", code));
1861                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1862                                 }
1863                                 return;
1864                         }
1865
1866                         if (arg.StartsWith (warning)) {
1867                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
1868                                 return;
1869                         }
1870
1871                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
1872                 }
1873
1874                 int[] ParseNumbers (string text)
1875                 {
1876                         string[] string_array = text.Split (',');
1877                         int[] values = new int [string_array.Length];
1878                         int index = 0;
1879                         foreach (string string_code in string_array) {
1880                                 try {
1881                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1882                                 }
1883                                 catch (FormatException) {
1884                                         Report.Warning (1692, 1, Location, "Invalid number");
1885                                 }
1886                         }
1887                         return values;
1888                 }
1889
1890                 bool eval_val (string s)
1891                 {
1892                         if (s == "true")
1893                                 return true;
1894                         if (s == "false")
1895                                 return false;
1896
1897                         if (defines == null)
1898                                 return false;
1899                         if (defines.Contains (s))
1900                                 return true;
1901
1902                         return false;
1903                 }
1904
1905                 bool pp_primary (ref string s)
1906                 {
1907                         s = s.Trim ();
1908                         int len = s.Length;
1909
1910                         if (len > 0){
1911                                 char c = s [0];
1912
1913                                 if (c == '('){
1914                                         s = s.Substring (1);
1915                                         bool val = pp_expr (ref s, false);
1916                                         if (s.Length > 0 && s [0] == ')'){
1917                                                 s = s.Substring (1);
1918                                                 return val;
1919                                         }
1920                                         Error_InvalidDirective ();
1921                                         return false;
1922                                 }
1923
1924                                 if (is_identifier_start_character (c)){
1925                                         int j = 1;
1926
1927                                         while (j < len){
1928                                                 c = s [j];
1929
1930                                                 if (is_identifier_part_character (c)){
1931                                                         j++;
1932                                                         continue;
1933                                                 }
1934                                                 bool v = eval_val (s.Substring (0, j));
1935                                                 s = s.Substring (j);
1936                                                 return v;
1937                                         }
1938                                         bool vv = eval_val (s);
1939                                         s = "";
1940                                         return vv;
1941                                 }
1942                         }
1943                         Error_InvalidDirective ();
1944                         return false;
1945                 }
1946
1947                 bool pp_unary (ref string s)
1948                 {
1949                         s = s.Trim ();
1950                         int len = s.Length;
1951
1952                         if (len > 0){
1953                                 if (s [0] == '!'){
1954                                         if (len > 1 && s [1] == '='){
1955                                                 Error_InvalidDirective ();
1956                                                 return false;
1957                                         }
1958                                         s = s.Substring (1);
1959                                         return ! pp_primary (ref s);
1960                                 } else
1961                                         return pp_primary (ref s);
1962                         } else {
1963                                 Error_InvalidDirective ();
1964                                 return false;
1965                         }
1966                 }
1967
1968                 bool pp_eq (ref string s)
1969                 {
1970                         bool va = pp_unary (ref s);
1971
1972                         s = s.Trim ();
1973                         int len = s.Length;
1974                         if (len > 0){
1975                                 if (s [0] == '='){
1976                                         if (len > 2 && s [1] == '='){
1977                                                 s = s.Substring (2);
1978                                                 return va == pp_unary (ref s);
1979                                         } else {
1980                                                 Error_InvalidDirective ();
1981                                                 return false;
1982                                         }
1983                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
1984                                         s = s.Substring (2);
1985
1986                                         return va != pp_unary (ref s);
1987
1988                                 }
1989                         }
1990
1991                         return va;
1992
1993                 }
1994
1995                 bool pp_and (ref string s)
1996                 {
1997                         bool va = pp_eq (ref s);
1998
1999                         s = s.Trim ();
2000                         int len = s.Length;
2001                         if (len > 0){
2002                                 if (s [0] == '&'){
2003                                         if (len > 2 && s [1] == '&'){
2004                                                 s = s.Substring (2);
2005                                                 return (va & pp_and (ref s));
2006                                         } else {
2007                                                 Error_InvalidDirective ();
2008                                                 return false;
2009                                         }
2010                                 }
2011                         }
2012                         return va;
2013                 }
2014
2015                 //
2016                 // Evaluates an expression for `#if' or `#elif'
2017                 //
2018                 bool pp_expr (ref string s, bool isTerm)
2019                 {
2020                         bool va = pp_and (ref s);
2021                         s = s.Trim ();
2022                         int len = s.Length;
2023                         if (len > 0){
2024                                 char c = s [0];
2025
2026                                 if (c == '|'){
2027                                         if (len > 2 && s [1] == '|'){
2028                                                 s = s.Substring (2);
2029                                                 return va | pp_expr (ref s, isTerm);
2030                                         } else {
2031                                                 Error_InvalidDirective ();
2032                                                 return false;
2033                                         }
2034                                 }
2035                                 if (isTerm) {
2036                                         Error_EndLineExpected ();
2037                                         return false;
2038                                 }
2039                         }
2040
2041                         return va;
2042                 }
2043
2044                 bool eval (string s)
2045                 {
2046                         bool v = pp_expr (ref s, true);
2047                         s = s.Trim ();
2048                         if (s.Length != 0){
2049                                 return false;
2050                         }
2051
2052                         return v;
2053                 }
2054
2055                 void Error_NumericConstantTooLong ()
2056                 {
2057                         Report.Error (1021, Location, "Numeric constant too long");
2058                 }
2059
2060                 void Error_InvalidDirective ()
2061                 {
2062                         Report.Error (1517, Location, "Invalid preprocessor directive");
2063                 }
2064
2065                 void Error_UnexpectedDirective (string extra)
2066                 {
2067                         Report.Error (
2068                                 1028, Location,
2069                                 "Unexpected processor directive (" + extra + ")");
2070                 }
2071
2072                 void Error_TokensSeen ()
2073                 {
2074                         Report.Error (1032, Location,
2075                                 "Cannot define or undefine preprocessor symbols after first token in file");
2076                 }
2077
2078                 void Eror_WrongPreprocessorLocation ()
2079                 {
2080                         Report.Error (1040, Location,
2081                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2082                 }
2083
2084                 void Error_EndLineExpected ()
2085                 {
2086                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2087                 }
2088
2089                 //
2090                 // if true, then the code continues processing the code
2091                 // if false, the code stays in a loop until another directive is
2092                 // reached.
2093                 // When caller_is_taking is false we ignore all directives except the ones
2094                 // which can help us to identify where the #if block ends
2095                 bool handle_preprocessing_directive (bool caller_is_taking)
2096                 {
2097                         string cmd, arg;
2098                         bool region_directive = false;
2099
2100                         get_cmd_arg (out cmd, out arg);
2101
2102                         // Eat any trailing whitespaces and single-line comments
2103                         if (arg.IndexOf ("//") != -1)
2104                                 arg = arg.Substring (0, arg.IndexOf ("//"));
2105                         arg = arg.Trim (simple_whitespaces);
2106
2107                         //
2108                         // The first group of pre-processing instructions is always processed
2109                         //
2110                         switch (cmd){
2111                         case "region":
2112                                 region_directive = true;
2113                                 arg = "true";
2114                                 goto case "if";
2115
2116                         case "endregion":
2117                                 if (ifstack == null || ifstack.Count == 0){
2118                                         Error_UnexpectedDirective ("no #region for this #endregion");
2119                                         return true;
2120                                 }
2121                                 int pop = (int) ifstack.Pop ();
2122
2123                                 if ((pop & REGION) == 0)
2124                                         Report.Error (1027, Location, "Expected `#endif' directive");
2125
2126                                 return caller_is_taking;
2127
2128                         case "if":
2129                                 if (ifstack == null)
2130                                         ifstack = new Stack (2);
2131
2132                                 int flags = region_directive ? REGION : 0;
2133                                 if (ifstack.Count == 0){
2134                                         flags |= PARENT_TAKING;
2135                                 } else {
2136                                         int state = (int) ifstack.Peek ();
2137                                         if ((state & TAKING) != 0) {
2138                                                 flags |= PARENT_TAKING;
2139                                         }
2140                                 }
2141
2142                                 if (caller_is_taking && eval (arg)) {
2143                                         ifstack.Push (flags | TAKING);
2144                                         return true;
2145                                 }
2146                                 ifstack.Push (flags);
2147                                 return false;
2148
2149                         case "endif":
2150                                 if (ifstack == null || ifstack.Count == 0){
2151                                         Error_UnexpectedDirective ("no #if for this #endif");
2152                                         return true;
2153                                 } else {
2154                                         pop = (int) ifstack.Pop ();
2155
2156                                         if ((pop & REGION) != 0)
2157                                                 Report.Error (1038, Location, "#endregion directive expected");
2158
2159                                         if (arg.Length != 0) {
2160                                                 Error_EndLineExpected ();
2161                                         }
2162
2163                                         if (ifstack.Count == 0)
2164                                                 return true;
2165
2166                                         int state = (int) ifstack.Peek ();
2167                                         return (state & TAKING) != 0;
2168                                 }
2169
2170                         case "elif":
2171                                 if (ifstack == null || ifstack.Count == 0){
2172                                         Error_UnexpectedDirective ("no #if for this #elif");
2173                                         return true;
2174                                 } else {
2175                                         int state = (int) ifstack.Pop ();
2176
2177                                         if ((state & REGION) != 0) {
2178                                                 Report.Error (1038, Location, "#endregion directive expected");
2179                                                 return true;
2180                                         }
2181
2182                                         if ((state & ELSE_SEEN) != 0){
2183                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2184                                                 return true;
2185                                         }
2186
2187                                         if ((state & TAKING) != 0) {
2188                                                 ifstack.Push (0);
2189                                                 return false;
2190                                         }
2191
2192                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2193                                                 ifstack.Push (state | TAKING);
2194                                                 return true;
2195                                         }
2196
2197                                         ifstack.Push (state);
2198                                         return false;
2199                                 }
2200
2201                         case "else":
2202                                 if (ifstack == null || ifstack.Count == 0){
2203                                         Error_UnexpectedDirective ("no #if for this #else");
2204                                         return true;
2205                                 } else {
2206                                         int state = (int) ifstack.Peek ();
2207
2208                                         if ((state & REGION) != 0) {
2209                                                 Report.Error (1038, Location, "#endregion directive expected");
2210                                                 return true;
2211                                         }
2212
2213                                         if ((state & ELSE_SEEN) != 0){
2214                                                 Error_UnexpectedDirective ("#else within #else");
2215                                                 return true;
2216                                         }
2217
2218                                         ifstack.Pop ();
2219
2220                                         if (arg.Length != 0) {
2221                                                 Error_EndLineExpected ();
2222                                                 return true;
2223                                         }
2224
2225                                         bool ret = false;
2226                                         if ((state & PARENT_TAKING) != 0) {
2227                                                 ret = (state & TAKING) == 0;
2228
2229                                                 if (ret)
2230                                                         state |= TAKING;
2231                                                 else
2232                                                         state &= ~TAKING;
2233                                         }
2234
2235                                         ifstack.Push (state | ELSE_SEEN);
2236
2237                                         return ret;
2238                                 }
2239                                 case "define":
2240                                         if (any_token_seen){
2241                                                 Error_TokensSeen ();
2242                                                 return caller_is_taking;
2243                                         }
2244                                         PreProcessDefinition (true, arg, caller_is_taking);
2245                                         return caller_is_taking;
2246
2247                                 case "undef":
2248                                         if (any_token_seen){
2249                                                 Error_TokensSeen ();
2250                                                 return caller_is_taking;
2251                                         }
2252                                         PreProcessDefinition (false, arg, caller_is_taking);
2253                                         return caller_is_taking;
2254                         }
2255
2256                         //
2257                         // These are only processed if we are in a `taking' block
2258                         //
2259                         if (!caller_is_taking)
2260                                 return false;
2261
2262                         switch (cmd){
2263                         case "error":
2264                                 Report.Error (1029, Location, "#error: '" + arg + "'");
2265                                 return true;
2266
2267                         case "warning":
2268                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2269                                 return true;
2270
2271                         case "pragma":
2272                                 if (RootContext.Version == LanguageVersion.ISO_1) {
2273                                         Report.FeatureIsNotISO1 (Location, "#pragma");
2274                                         return true;
2275                                 }
2276
2277                                 PreProcessPragma (arg);
2278                                 return true;
2279
2280                         case "line":
2281                                 if (!PreProcessLine (arg))
2282                                         Report.Error (
2283                                                 1576, Location,
2284                                                 "The line number specified for #line directive is missing or invalid");
2285                                 return caller_is_taking;
2286                         }
2287
2288                         Report.Error (1024, Location, "Wrong preprocessor directive");
2289                         return true;
2290
2291                 }
2292
2293                 private int consume_string (bool quoted)
2294                 {
2295                         int c;
2296                         string_builder.Length = 0;
2297
2298                         while ((c = getChar ()) != -1){
2299                                 if (c == '"'){
2300                                         if (quoted && peekChar () == '"'){
2301                                                 string_builder.Append ((char) c);
2302                                                 getChar ();
2303                                                 continue;
2304                                         } else {
2305                                                 val = string_builder.ToString ();
2306                                                 return Token.LITERAL_STRING;
2307                                         }
2308                                 }
2309
2310                                 if (c == '\n'){
2311                                         if (!quoted)
2312                                                 Report.Error (1010, Location, "Newline in constant");
2313                                 }
2314
2315                                 if (!quoted){
2316                                         c = escape (c);
2317                                         if (c == -1)
2318                                                 return Token.ERROR;
2319                                 }
2320                                 string_builder.Append ((char) c);
2321                         }
2322
2323                         Report.Error (1039, Location, "Unterminated string literal");
2324                         return Token.EOF;
2325                 }
2326
2327                 private int consume_identifier (int s)
2328                 {
2329                         int res = consume_identifier (s, false);
2330
2331                         if (doc_state == XmlCommentState.Allowed)
2332                                 doc_state = XmlCommentState.NotAllowed;
2333                         switch (res) {
2334                         case Token.USING:
2335                         case Token.NAMESPACE:
2336                                 check_incorrect_doc_comment ();
2337                                 break;
2338                         }
2339
2340                         if (res == Token.PARTIAL) {
2341                                 // Save current position and parse next token.
2342                                 PushPosition ();
2343
2344                                 int next_token = token ();
2345                                 bool ok = (next_token == Token.CLASS) ||
2346                                         (next_token == Token.STRUCT) ||
2347                                         (next_token == Token.INTERFACE);
2348
2349                                 PopPosition ();
2350
2351                                 if (ok)
2352                                         return res;
2353
2354                                 if (next_token < Token.LAST_KEYWORD)
2355                                         Report.Error (267, Location, "The `partial' modifier can be used only immediately before keyword `class', `struct', or `interface'");
2356
2357                                 val = new LocatedToken (Location, "partial");
2358                                 return Token.IDENTIFIER;
2359                         }
2360
2361                         return res;
2362                 }
2363
2364                 private int consume_identifier (int s, bool quoted)
2365                 {
2366                         int pos = 1;
2367                         int c = -1;
2368
2369                         id_builder [0] = (char) s;
2370
2371                         current_location = new Location (ref_line, Col);
2372
2373                         while ((c = getChar ()) != -1) {
2374                         loop:
2375                                 if (is_identifier_part_character ((char) c)){
2376                                         if (pos == max_id_size){
2377                                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2378                                                 return Token.ERROR;
2379                                         }
2380
2381                                         id_builder [pos++] = (char) c;
2382 //                                      putback_char = -1;
2383                                 } else if (c == '\\') {
2384                                         c = escape (c);
2385                                         goto loop;
2386                                 } else {
2387 //                                      putback_char = c;
2388                                         putback (c);
2389                                         break;
2390                                 }
2391                         }
2392
2393                         //
2394                         // Optimization: avoids doing the keyword lookup
2395                         // on uppercase letters and _
2396                         //
2397                         if (!quoted && (s >= 'a' || s == '_')){
2398                                 int keyword = GetKeyword (id_builder, pos);
2399                                 if (keyword != -1) {
2400                                         val = Location;
2401                                         return keyword;
2402                                 }
2403                         }
2404
2405                         //
2406                         // Keep identifiers in an array of hashtables to avoid needless
2407                         // allocations
2408                         //
2409
2410                         if (identifiers [pos] != null) {
2411                                 val = identifiers [pos][id_builder];
2412                                 if (val != null) {
2413                                         val = new LocatedToken (Location, (string) val);
2414                                         if (quoted)
2415                                                 escapedIdentifiers.Add (val);
2416                                         return Token.IDENTIFIER;
2417                                 }
2418                         }
2419                         else
2420                                 identifiers [pos] = new CharArrayHashtable (pos);
2421
2422                         val = new String (id_builder, 0, pos);
2423                         if (RootContext.Version == LanguageVersion.ISO_1) {
2424                                 for (int i = 1; i < id_builder.Length; i += 3) {
2425                                         if (id_builder [i] == '_' && (id_builder [i - 1] == '_' || id_builder [i + 1] == '_')) {
2426                                                 Report.Error (1638, Location,
2427                                                         "`{0}': Any identifier with double underscores cannot be used when ISO language version mode is specified", val.ToString ());
2428                                                 break;
2429                                         }
2430                                 }
2431                         }
2432
2433                         char [] chars = new char [pos];
2434                         Array.Copy (id_builder, chars, pos);
2435
2436                         identifiers [pos] [chars] = val;
2437
2438                         val = new LocatedToken (Location, (string) val);
2439                         if (quoted)
2440                                 escapedIdentifiers.Add (val);
2441                         return Token.IDENTIFIER;
2442                 }
2443
2444                 public int xtoken ()
2445                 {
2446                         int t;
2447                         bool doread = false;
2448                         int c;
2449
2450                         // Whether we have seen comments on the current line
2451                         bool comments_seen = false;
2452                         val = null;
2453                         for (;(c = getChar ()) != -1;) {
2454                                 if (c == '\t'){
2455                                         col = ((col + 8) / 8) * 8;
2456                                         continue;
2457                                 }
2458
2459                                 if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0 || c == 0)
2460                                         continue;
2461
2462                                 if (c == '\r') {
2463                                         if (peekChar () == '\n')
2464                                                 getChar ();
2465
2466                                         any_token_seen |= tokens_seen;
2467                                         tokens_seen = false;
2468                                         comments_seen = false;
2469                                         continue;
2470                                 }
2471
2472                                 // Handle double-slash comments.
2473                                 if (c == '/'){
2474                                         int d = peekChar ();
2475
2476                                         if (d == '/'){
2477                                                 getChar ();
2478                                                 if (RootContext.Documentation != null && peekChar () == '/') {
2479                                                         getChar ();
2480                                                         // Don't allow ////.
2481                                                         if ((d = peekChar ()) != '/') {
2482                                                                 update_comment_location ();
2483                                                                 if (doc_state == XmlCommentState.Allowed)
2484                                                                         handle_one_line_xml_comment ();
2485                                                                 else if (doc_state == XmlCommentState.NotAllowed)
2486                                                                         warn_incorrect_doc_comment ();
2487                                                         }
2488                                                 }
2489                                                 while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
2490                                                         if (d == '\n'){
2491                                                         }
2492                                                 any_token_seen |= tokens_seen;
2493                                                 tokens_seen = false;
2494                                                 comments_seen = false;
2495                                                 continue;
2496                                         } else if (d == '*'){
2497                                                 getChar ();
2498                                                 bool docAppend = false;
2499                                                 if (RootContext.Documentation != null && peekChar () == '*') {
2500                                                         getChar ();
2501                                                         update_comment_location ();
2502                                                         // But when it is /**/, just do nothing.
2503                                                         if (peekChar () == '/') {
2504                                                                 getChar ();
2505                                                                 continue;
2506                                                         }
2507                                                         if (doc_state == XmlCommentState.Allowed)
2508                                                                 docAppend = true;
2509                                                         else if (doc_state == XmlCommentState.NotAllowed)
2510                                                                 warn_incorrect_doc_comment ();
2511                                                 }
2512
2513                                                 int current_comment_start = 0;
2514                                                 if (docAppend) {
2515                                                         current_comment_start = xml_comment_buffer.Length;
2516                                                         xml_comment_buffer.Append (Environment.NewLine);
2517                                                 }
2518
2519                                                 Location start_location = Location;
2520
2521                                                 while ((d = getChar ()) != -1){
2522                                                         if (d == '*' && peekChar () == '/'){
2523                                                                 getChar ();
2524                                                                 comments_seen = true;
2525                                                                 break;
2526                                                         }
2527                                                         if (docAppend)
2528                                                                 xml_comment_buffer.Append ((char) d);
2529
2530                                                         if (d == '\n'){
2531                                                                 any_token_seen |= tokens_seen;
2532                                                                 tokens_seen = false;
2533                                                                 //
2534                                                                 // Reset 'comments_seen' just to be consistent.
2535                                                                 // It doesn't matter either way, here.
2536                                                                 //
2537                                                                 comments_seen = false;
2538                                                         }
2539                                                 }
2540                                                 if (!comments_seen)
2541                                                         Report.Error (1035, start_location, "End-of-file found, '*/' expected");
2542
2543                                                 if (docAppend)
2544                                                         update_formatted_doc_comment (current_comment_start);
2545                                                 continue;
2546                                         }
2547                                         goto is_punct_label;
2548                                 }
2549
2550
2551                                 if (c == '\\' || is_identifier_start_character ((char)c)){
2552                                         tokens_seen = true;
2553                                         return consume_identifier (c);
2554                                 }
2555
2556                         is_punct_label:
2557                                 current_location = new Location (ref_line, Col);
2558                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
2559                                         tokens_seen = true;
2560                                         if (doread){
2561                                                 getChar ();
2562                                         }
2563                                         return t;
2564                                 }
2565
2566                                 // white space
2567                                 if (c == '\n'){
2568                                         any_token_seen |= tokens_seen;
2569                                         tokens_seen = false;
2570                                         comments_seen = false;
2571                                         continue;
2572                                 }
2573
2574                                 if (c >= '0' && c <= '9'){
2575                                         tokens_seen = true;
2576                                         return is_number (c);
2577                                 }
2578
2579                                 if (c == '.'){
2580                                         tokens_seen = true;
2581                                         int peek = peekChar ();
2582                                         if (peek >= '0' && peek <= '9')
2583                                                 return is_number (c);
2584                                         return Token.DOT;
2585                                 }
2586
2587                                 if (c == '#') {
2588                                         if (tokens_seen || comments_seen) {
2589                                                 Eror_WrongPreprocessorLocation ();
2590                                                 return Token.ERROR;
2591                                         }
2592
2593                                         if (handle_preprocessing_directive (true))
2594                                                 continue;
2595
2596                                         bool directive_expected = false;
2597                                         while ((c = getChar ()) != -1) {
2598                                                 if (col == 1) {
2599                                                         directive_expected = true;
2600                                                 } else if (!directive_expected) {
2601                                                         // TODO: Implement comment support for disabled code and uncomment this code
2602 //                                                      if (c == '#') {
2603 //                                                              Eror_WrongPreprocessorLocation ();
2604 //                                                              return Token.ERROR;
2605 //                                                      }
2606                                                         continue;
2607                                                 }
2608
2609                                                 if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
2610                                                         continue;
2611
2612                                                 if (c == '#') {
2613                                                         if (handle_preprocessing_directive (false))
2614                                                                 break;
2615                                                 }
2616                                                 directive_expected = false;
2617                                         }
2618
2619                                         if (c != -1) {
2620                                                 tokens_seen = false;
2621                                                 continue;
2622                                         }
2623
2624                                         return Token.EOF;
2625                                 }
2626
2627                                 if (c == '"')
2628                                         return consume_string (false);
2629
2630                                 if (c == '\''){
2631                                         c = getChar ();
2632                                         tokens_seen = true;
2633                                         if (c == '\''){
2634                                                 error_details = "Empty character literal";
2635                                                 Report.Error (1011, Location, error_details);
2636                                                 return Token.ERROR;
2637                                         }
2638                                         if (c == '\r' || c == '\n') {
2639                                                 Report.Error (1010, Location, "Newline in constant");
2640                                                 return Token.ERROR;
2641                                         }
2642                                         c = escape (c);
2643                                         if (c == -1)
2644                                                 return Token.ERROR;
2645                                         val = new System.Char ();
2646                                         val = (char) c;
2647                                         c = getChar ();
2648
2649                                         if (c != '\''){
2650                                                 error_details = "Too many characters in character literal";
2651                                                 Report.Error (1012, Location, error_details);
2652
2653                                                 // Try to recover, read until newline or next "'"
2654                                                 while ((c = getChar ()) != -1){
2655                                                         if (c == '\n'){
2656                                                                 break;
2657                                                         }
2658                                                         else if (c == '\'')
2659                                                                 break;
2660                                                 }
2661                                                 return Token.ERROR;
2662                                         }
2663                                         return Token.LITERAL_CHARACTER;
2664                                 }
2665
2666                                 if (c == '@') {
2667                                         c = getChar ();
2668                                         if (c == '"') {
2669                                                 tokens_seen = true;
2670                                                 return consume_string (true);
2671                                         } else if (is_identifier_start_character ((char) c)){
2672                                                 return consume_identifier (c, true);
2673                                         } else {
2674                                                 Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
2675                                         }
2676                                 }
2677
2678                                 error_details = ((char)c).ToString ();
2679
2680                                 return Token.ERROR;
2681                         }
2682
2683                         return Token.EOF;
2684                 }
2685
2686                 //
2687                 // Handles one line xml comment
2688                 //
2689                 private void handle_one_line_xml_comment ()
2690                 {
2691                         int c;
2692                         while ((c = peekChar ()) == ' ')
2693                                 getChar (); // skip heading whitespaces.
2694                         while ((c = peekChar ()) != -1 && c != '\n' && c != '\r') {
2695                                 xml_comment_buffer.Append ((char) getChar ());
2696                         }
2697                         if (c == '\r' || c == '\n')
2698                                 xml_comment_buffer.Append (Environment.NewLine);
2699                 }
2700
2701                 //
2702                 // Remove heading "*" in Javadoc-like xml documentation.
2703                 //
2704                 private void update_formatted_doc_comment (int current_comment_start)
2705                 {
2706                         int length = xml_comment_buffer.Length - current_comment_start;
2707                         string [] lines = xml_comment_buffer.ToString (
2708                                 current_comment_start,
2709                                 length).Replace ("\r", "").Split ('\n');
2710
2711                         // The first line starts with /**, thus it is not target
2712                         // for the format check.
2713                         for (int i = 1; i < lines.Length; i++) {
2714                                 string s = lines [i];
2715                                 int idx = s.IndexOf ('*');
2716                                 string head = null;
2717                                 if (idx < 0) {
2718                                         if (i < lines.Length - 1)
2719                                                 return;
2720                                         head = s;
2721                                 } else
2722                                         head = s.Substring (0, idx);
2723                                 foreach (char c in head)
2724                                         if (c != ' ')
2725                                                 return;
2726                                 lines [i] = s.Substring (idx + 1);
2727                         }
2728                         xml_comment_buffer.Remove (current_comment_start, length);
2729                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
2730                 }
2731
2732                 //
2733                 // Updates current comment location.
2734                 //
2735                 private void update_comment_location ()
2736                 {
2737                         if (current_comment_location.IsNull) {
2738                                 // "-2" is for heading "//" or "/*"
2739                                 current_comment_location =
2740                                         new Location (ref_line, col - 2);
2741                         }
2742                 }
2743
2744                 //
2745                 // Checks if there was incorrect doc comments and raise
2746                 // warnings.
2747                 //
2748                 public void check_incorrect_doc_comment ()
2749                 {
2750                         if (xml_comment_buffer.Length > 0)
2751                                 warn_incorrect_doc_comment ();
2752                 }
2753
2754                 //
2755                 // Raises a warning when tokenizer found incorrect doccomment
2756                 // markup.
2757                 //
2758                 private void warn_incorrect_doc_comment ()
2759                 {
2760                         if (doc_state != XmlCommentState.Error) {
2761                                 doc_state = XmlCommentState.Error;
2762                                 // in csc, it is 'XML comment is not placed on
2763                                 // a valid language element'. But that does not
2764                                 // make sense.
2765                                 Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
2766                         }
2767                 }
2768
2769                 //
2770                 // Consumes the saved xml comment lines (if any)
2771                 // as for current target member or type.
2772                 //
2773                 public string consume_doc_comment ()
2774                 {
2775                         if (xml_comment_buffer.Length > 0) {
2776                                 string ret = xml_comment_buffer.ToString ();
2777                                 reset_doc_comment ();
2778                                 return ret;
2779                         }
2780                         return null;
2781                 }
2782
2783                 void reset_doc_comment ()
2784                 {
2785                         xml_comment_buffer.Length = 0;
2786                         current_comment_location = Location.Null;
2787                 }
2788
2789                 public void cleanup ()
2790                 {
2791                         if (ifstack != null && ifstack.Count >= 1) {
2792                                 current_location = new Location (ref_line, Col);
2793                                 int state = (int) ifstack.Pop ();
2794                                 if ((state & REGION) != 0)
2795                                         Report.Error (1038, Location, "#endregion directive expected");
2796                                 else
2797                                         Report.Error (1027, Location, "Expected `#endif' directive");
2798                         }
2799                 }
2800         }
2801
2802         //
2803         // Indicates whether it accepts XML documentation or not.
2804         //
2805         public enum XmlCommentState {
2806                 // comment is allowed in this state.
2807                 Allowed,
2808                 // comment is not allowed in this state.
2809                 NotAllowed,
2810                 // once comments appeared when it is NotAllowed, then the
2811                 // state is changed to it, until the state is changed to
2812                 // .Allowed.
2813                 Error
2814         }
2815 }
2816