mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Licensed under the terms of the GNU GPL
   9 //
  10 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // (C) 2004 Novell, Inc
  12 //
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections;
  18 using System.IO;
  19 using System.Globalization;
  20 using System.Reflection;
  21
  22 namespace Mono.CSharp
  23 {
  24         /// <summary>
  25         ///    Tokenizer for C# source code.
  26         /// </summary>
  27
  28         public class Tokenizer : yyParser.yyInput
  29         {
  30                 SeekableStreamReader reader;
  31                 SourceFile ref_name;
  32                 SourceFile file_name;
  33                 int ref_line = 1;
  34                 int line = 1;
  35                 int col = 0;
  36                 int previous_col;
  37                 int current_token;
  38                 bool handle_get_set = false;
  39                 bool handle_remove_add = false;
  40                 bool handle_assembly = false;
  41                 bool handle_constraints = false;
  42                 bool handle_typeof = false;
  43                 bool linq;
  44                 Location current_location;
  45                 Location current_comment_location = Location.Null;
  46                 ArrayList escapedIdentifiers = new ArrayList ();
  47
  48                 //
  49                 // XML documentation buffer. The save point is used to divide
  50                 // comments on types and comments on members.
  51                 //
  52                 StringBuilder xml_comment_buffer;
  53
  54                 //
  55                 // See comment on XmlCommentState enumeration.
  56                 //
  57                 XmlCommentState xmlDocState = XmlCommentState.Allowed;
  58
  59                 //
  60                 // Whether tokens have been seen on this line
  61                 //
  62                 bool tokens_seen = false;
  63
  64                 //
  65                 // Whether a token has been seen on the file
  66                 // This is needed because `define' is not allowed to be used
  67                 // after a token has been seen.
  68                 //
  69                 bool any_token_seen = false;
  70
  71                 static Hashtable tokenValues;
  72                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
  73
  74                 private static Hashtable TokenValueName
  75                 {
  76                         get {
  77                                 if (tokenValues == null)
  78                                         tokenValues = GetTokenValueNameHash ();
  79
  80                                 return tokenValues;
  81                         }
  82                 }
  83
  84                 private static Hashtable GetTokenValueNameHash ()
  85                 {
  86                         Type t = typeof (Token);
  87                         FieldInfo [] fields = t.GetFields ();
  88                         Hashtable hash = new Hashtable ();
  89                         foreach (FieldInfo field in fields) {
  90                                 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))
  91                                         hash.Add (field.GetValue (null), field.Name);
  92                         }
  93                         return hash;
  94                 }
  95
  96                 //
  97                 // Returns a verbose representation of the current location
  98                 //
  99                 public string location {
 100                         get {
 101                                 string det;
 102
 103                                 if (current_token == Token.ERROR)
 104                                         det = "detail: " + error_details;
 105                                 else
 106                                         det = "";
 107
 108                                 // return "Line:     "+line+" Col: "+col + "\n" +
 109                                 //       "VirtLine: "+ref_line +
 110                                 //       " Token: "+current_token + " " + det;
 111                                 string current_token_name = TokenValueName [current_token] as string;
 112                                 if (current_token_name == null)
 113                                         current_token_name = current_token.ToString ();
 114
 115                                 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,
 116                                                                                        ref_line,
 117                                                                                        col,
 118                                                                                        current_token_name,
 119                                                                                        det);
 120                         }
 121                 }
 122
 123                 public bool PropertyParsing {
 124                         get {
 125                                 return handle_get_set;
 126                         }
 127
 128                         set {
 129                                 handle_get_set = value;
 130                         }
 131                 }
 132
 133                 public bool AssemblyTargetParsing {
 134                         get {
 135                                 return handle_assembly;
 136                         }
 137
 138                         set {
 139                                 handle_assembly = value;
 140                         }
 141                 }
 142
 143                 public bool EventParsing {
 144                         get {
 145                                 return handle_remove_add;
 146                         }
 147
 148                         set {
 149                                 handle_remove_add = value;
 150                         }
 151                 }
 152
 153                 public bool ConstraintsParsing {
 154                         get {
 155                                 return handle_constraints;
 156                         }
 157
 158                         set {
 159                                 handle_constraints = value;
 160                         }
 161                 }
 162
 163                 public bool TypeOfParsing {
 164                         get {
 165                                 return handle_typeof;
 166                         }
 167
 168                         set {
 169                                 handle_typeof = value;
 170                         }
 171                 }
 172
 173                 public XmlCommentState doc_state {
 174                         get { return xmlDocState; }
 175                         set {
 176                                 if (value == XmlCommentState.Allowed) {
 177                                         check_incorrect_doc_comment ();
 178                                         reset_doc_comment ();
 179                                 }
 180                                 xmlDocState = value;
 181                         }
 182                 }
 183
 184                 public bool IsEscapedIdentifier (Location loc)
 185                 {
 186                         foreach (LocatedToken lt in escapedIdentifiers)
 187                                 if (lt.Location.Equals (loc))
 188                                         return true;
 189                         return false;
 190                 }
 191
 192                 //
 193                 // Class variables
 194                 //
 195                 static CharArrayHashtable[] keywords;
 196                 static Hashtable keywordStrings = new Hashtable ();
 197                 static NumberStyles styles;
 198                 static NumberFormatInfo csharp_format_info;
 199
 200                 //
 201                 // Values for the associated token returned
 202                 //
 203                 int putback_char;
 204                 Object val;
 205
 206                 //
 207                 // Pre-processor
 208                 //
 209                 Hashtable defines;
 210
 211                 const int TAKING        = 1;
 212                 const int ELSE_SEEN     = 4;
 213                 const int PARENT_TAKING = 8;
 214                 const int REGION        = 16;
 215
 216                 //
 217                 // pre-processor if stack state:
 218                 //
 219                 Stack ifstack;
 220
 221                 static System.Text.StringBuilder string_builder;
 222
 223                 const int max_id_size = 512;
 224                 static char [] id_builder = new char [max_id_size];
 225
 226                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 227
 228                 const int max_number_size = 512;
 229                 static char [] number_builder = new char [max_number_size];
 230                 static int number_pos;
 231
 232                 //
 233                 // Details about the error encoutered by the tokenizer
 234                 //
 235                 string error_details;
 236
 237                 public string error {
 238                         get {
 239                                 return error_details;
 240                         }
 241                 }
 242
 243                 public int Line {
 244                         get {
 245                                 return ref_line;
 246                         }
 247                 }
 248
 249                 public int Col {
 250                         get {
 251                                 return col;
 252                         }
 253                 }
 254
 255                 //
 256                 // This is used when the tokenizer needs to save
 257                 // the current position as it needs to do some parsing
 258                 // on its own to deamiguate a token in behalf of the
 259                 // parser.
 260                 //
 261                 Stack position_stack = new Stack (2);
 262                 class Position {
 263                         public int position;
 264                         public int ref_line;
 265                         public int col;
 266                         public int putback_char;
 267                         public int previous_col;
 268                         public Stack ifstack;
 269                         public int parsing_generic_less_than;
 270                         public int current_token;
 271
 272                         public Position (Tokenizer t)
 273                         {
 274                                 position = t.reader.Position;
 275                                 ref_line = t.ref_line;
 276                                 col = t.col;
 277                                 putback_char = t.putback_char;
 278                                 previous_col = t.previous_col;
 279                                 if (t.ifstack != null && t.ifstack.Count != 0)
 280                                         ifstack = (Stack)t.ifstack.Clone ();
 281                                 parsing_generic_less_than = t.parsing_generic_less_than;
 282                                 current_token = t.current_token;
 283                         }
 284                 }
 285
 286                 public void PushPosition ()
 287                 {
 288                         position_stack.Push (new Position (this));
 289                 }
 290
 291                 public void PopPosition ()
 292                 {
 293                         Position p = (Position) position_stack.Pop ();
 294
 295                         reader.Position = p.position;
 296                         ref_line = p.ref_line;
 297                         col = p.col;
 298                         putback_char = p.putback_char;
 299                         previous_col = p.previous_col;
 300                         ifstack = p.ifstack;
 301                         parsing_generic_less_than = p.parsing_generic_less_than;
 302                         current_token = p.current_token;
 303                 }
 304
 305                 // Do not reset the position, ignore it.
 306                 public void DiscardPosition ()
 307                 {
 308                         position_stack.Pop ();
 309                 }
 310
 311                 static void AddKeyword (string kw, int token) {
 312                         keywordStrings.Add (kw, kw);
 313                         if (keywords [kw.Length] == null) {
 314                                 keywords [kw.Length] = new CharArrayHashtable (kw.Length);
 315                         }
 316                         keywords [kw.Length] [kw.ToCharArray ()] = token;
 317                 }
 318
 319                 static void InitTokens ()
 320                 {
 321                         keywords = new CharArrayHashtable [64];
 322
 323                         AddKeyword ("__arglist", Token.ARGLIST);
 324                         AddKeyword ("abstract", Token.ABSTRACT);
 325                         AddKeyword ("as", Token.AS);
 326                         AddKeyword ("add", Token.ADD);
 327                         AddKeyword ("assembly", Token.ASSEMBLY);
 328                         AddKeyword ("base", Token.BASE);
 329                         AddKeyword ("bool", Token.BOOL);
 330                         AddKeyword ("break", Token.BREAK);
 331                         AddKeyword ("byte", Token.BYTE);
 332                         AddKeyword ("case", Token.CASE);
 333                         AddKeyword ("catch", Token.CATCH);
 334                         AddKeyword ("char", Token.CHAR);
 335                         AddKeyword ("checked", Token.CHECKED);
 336                         AddKeyword ("class", Token.CLASS);
 337                         AddKeyword ("const", Token.CONST);
 338                         AddKeyword ("continue", Token.CONTINUE);
 339                         AddKeyword ("decimal", Token.DECIMAL);
 340                         AddKeyword ("default", Token.DEFAULT);
 341                         AddKeyword ("delegate", Token.DELEGATE);
 342                         AddKeyword ("do", Token.DO);
 343                         AddKeyword ("double", Token.DOUBLE);
 344                         AddKeyword ("else", Token.ELSE);
 345                         AddKeyword ("enum", Token.ENUM);
 346                         AddKeyword ("event", Token.EVENT);
 347                         AddKeyword ("explicit", Token.EXPLICIT);
 348                         AddKeyword ("extern", Token.EXTERN);
 349                         AddKeyword ("false", Token.FALSE);
 350                         AddKeyword ("finally", Token.FINALLY);
 351                         AddKeyword ("fixed", Token.FIXED);
 352                         AddKeyword ("float", Token.FLOAT);
 353                         AddKeyword ("for", Token.FOR);
 354                         AddKeyword ("foreach", Token.FOREACH);
 355                         AddKeyword ("goto", Token.GOTO);
 356                         AddKeyword ("get", Token.GET);
 357                         AddKeyword ("if", Token.IF);
 358                         AddKeyword ("implicit", Token.IMPLICIT);
 359                         AddKeyword ("in", Token.IN);
 360                         AddKeyword ("int", Token.INT);
 361                         AddKeyword ("interface", Token.INTERFACE);
 362                         AddKeyword ("internal", Token.INTERNAL);
 363                         AddKeyword ("is", Token.IS);
 364                         AddKeyword ("lock", Token.LOCK);
 365                         AddKeyword ("long", Token.LONG);
 366                         AddKeyword ("namespace", Token.NAMESPACE);
 367                         AddKeyword ("new", Token.NEW);
 368                         AddKeyword ("null", Token.NULL);
 369                         AddKeyword ("object", Token.OBJECT);
 370                         AddKeyword ("operator", Token.OPERATOR);
 371                         AddKeyword ("out", Token.OUT);
 372                         AddKeyword ("override", Token.OVERRIDE);
 373                         AddKeyword ("params", Token.PARAMS);
 374                         AddKeyword ("private", Token.PRIVATE);
 375                         AddKeyword ("protected", Token.PROTECTED);
 376                         AddKeyword ("public", Token.PUBLIC);
 377                         AddKeyword ("readonly", Token.READONLY);
 378                         AddKeyword ("ref", Token.REF);
 379                         AddKeyword ("remove", Token.REMOVE);
 380                         AddKeyword ("return", Token.RETURN);
 381                         AddKeyword ("sbyte", Token.SBYTE);
 382                         AddKeyword ("sealed", Token.SEALED);
 383                         AddKeyword ("set", Token.SET);
 384                         AddKeyword ("short", Token.SHORT);
 385                         AddKeyword ("sizeof", Token.SIZEOF);
 386                         AddKeyword ("stackalloc", Token.STACKALLOC);
 387                         AddKeyword ("static", Token.STATIC);
 388                         AddKeyword ("string", Token.STRING);
 389                         AddKeyword ("struct", Token.STRUCT);
 390                         AddKeyword ("switch", Token.SWITCH);
 391                         AddKeyword ("this", Token.THIS);
 392                         AddKeyword ("throw", Token.THROW);
 393                         AddKeyword ("true", Token.TRUE);
 394                         AddKeyword ("try", Token.TRY);
 395                         AddKeyword ("typeof", Token.TYPEOF);
 396                         AddKeyword ("uint", Token.UINT);
 397                         AddKeyword ("ulong", Token.ULONG);
 398                         AddKeyword ("unchecked", Token.UNCHECKED);
 399                         AddKeyword ("unsafe", Token.UNSAFE);
 400                         AddKeyword ("ushort", Token.USHORT);
 401                         AddKeyword ("using", Token.USING);
 402                         AddKeyword ("virtual", Token.VIRTUAL);
 403                         AddKeyword ("void", Token.VOID);
 404                         AddKeyword ("volatile", Token.VOLATILE);
 405                         AddKeyword ("while", Token.WHILE);
 406                         AddKeyword ("partial", Token.PARTIAL);
 407 #if GMCS_SOURCE
 408                         AddKeyword ("where", Token.WHERE);
 409 #endif
 410                 }
 411
 412                 //
 413                 // Class initializer
 414                 //
 415                 static Tokenizer ()
 416                 {
 417                         InitTokens ();
 418                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 419                         styles = NumberStyles.Float;
 420
 421                         string_builder = new System.Text.StringBuilder ();
 422                 }
 423
 424                 int GetKeyword (char[] id, int id_len)
 425                 {
 426                         /*
 427                          * Keywords are stored in an array of hashtables grouped by their
 428                          * length.
 429                          */
 430
 431                         if ((id_len >= keywords.Length) || (keywords [id_len] == null))
 432                                 return -1;
 433                         object o = keywords [id_len] [id];
 434
 435                         if (o == null)
 436                                 return -1;
 437
 438                         int res = (int) o;
 439
 440                         if (handle_get_set == false && (res == Token.GET || res == Token.SET))
 441                                 return -1;
 442                         if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
 443                                 return -1;
 444                         if (handle_assembly == false && res == Token.ASSEMBLY)
 445                                 return -1;
 446 #if GMCS_SOURCE
 447                         if (handle_constraints == false && res == Token.WHERE)
 448                                 return -1;
 449 #endif
 450                         return res;
 451
 452                 }
 453
 454                 public Location Location {
 455                         get { return current_location; }
 456                 }
 457
 458                 void define (string def)
 459                 {
 460                         if (!RootContext.AllDefines.Contains (def)){
 461                                 RootContext.AllDefines [def] = true;
 462                         }
 463                         if (defines.Contains (def))
 464                                 return;
 465                         defines [def] = true;
 466                 }
 467
 468                 public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)
 469                 {
 470                         this.ref_name = file;
 471                         this.file_name = file;
 472                         linq = RootContext.Version == LanguageVersion.LINQ;
 473                         reader = input;
 474
 475                         putback_char = -1;
 476
 477                         if (defs != null){
 478                                 defines = new Hashtable ();
 479                                 foreach (string def in defs)
 480                                         define (def);
 481                         }
 482
 483                         xml_comment_buffer = new StringBuilder ();
 484
 485                         //
 486                         // FIXME: This could be `Location.Push' but we have to
 487                         // find out why the MS compiler allows this
 488                         //
 489                         Mono.CSharp.Location.Push (file);
 490                 }
 491
 492                 static bool is_identifier_start_character (char c)
 493                 {
 494                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter (c);
 495                 }
 496
 497                 static bool is_identifier_part_character (char c)
 498                 {
 499                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') ||
 500                                 Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 501                 }
 502
 503                 public static bool IsKeyword (string s)
 504                 {
 505                         return keywordStrings [s] != null;
 506                 }
 507
 508                 public static bool IsValidIdentifier (string s)
 509                 {
 510                         if (s == null || s.Length == 0)
 511                                 return false;
 512
 513                         if (!is_identifier_start_character (s [0]))
 514                                 return false;
 515
 516                         for (int i = 1; i < s.Length; i ++)
 517                                 if (! is_identifier_part_character (s [i]))
 518                                         return false;
 519
 520                         return true;
 521                 }
 522
 523                 bool parse_generic_dimension (out int dimension)
 524                 {
 525                         dimension = 1;
 526
 527                 again:
 528                         int the_token = token ();
 529                         if (the_token == Token.OP_GENERICS_GT)
 530                                 return true;
 531                         else if (the_token == Token.COMMA) {
 532                                 dimension++;
 533                                 goto again;
 534                         }
 535
 536                         return false;
 537                 }
 538
 539                 bool parse_less_than ()
 540                 {
 541                 start:
 542                         int the_token = token ();
 543                         if (the_token == Token.OPEN_BRACKET) {
 544                                 do {
 545                                         the_token = token ();
 546                                 } while (the_token != Token.CLOSE_BRACKET);
 547                                 the_token = token ();
 548                         }
 549                         switch (the_token) {
 550                         case Token.IDENTIFIER:
 551                         case Token.OBJECT:
 552                         case Token.STRING:
 553                         case Token.BOOL:
 554                         case Token.DECIMAL:
 555                         case Token.FLOAT:
 556                         case Token.DOUBLE:
 557                         case Token.SBYTE:
 558                         case Token.BYTE:
 559                         case Token.SHORT:
 560                         case Token.USHORT:
 561                         case Token.INT:
 562                         case Token.UINT:
 563                         case Token.LONG:
 564                         case Token.ULONG:
 565                         case Token.CHAR:
 566                         case Token.VOID:
 567                                 break;
 568
 569                         default:
 570                                 return false;
 571                         }
 572                 again:
 573                         the_token = token ();
 574
 575                         if (the_token == Token.OP_GENERICS_GT)
 576                                 return true;
 577                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
 578                                 goto start;
 579                         else if (the_token == Token.INTERR || the_token == Token.STAR)
 580                                 goto again;
 581                         else if (the_token == Token.OP_GENERICS_LT) {
 582                                 if (!parse_less_than ())
 583                                         return false;
 584                                 goto again;
 585                         } else if (the_token == Token.OPEN_BRACKET) {
 586                         rank_specifiers:
 587                                 the_token = token ();
 588                                 if (the_token == Token.CLOSE_BRACKET)
 589                                         goto again;
 590                                 else if (the_token == Token.COMMA)
 591                                         goto rank_specifiers;
 592                                 return false;
 593                         }
 594
 595                         return false;
 596                 }
 597
 598 #if GMCS_SOURCE
 599                 public void PutbackNullable ()
 600                 {
 601                         if (nullable_pos < 0)
 602                                 throw new Exception ();
 603
 604                         current_token = -1;
 605                         val = null;
 606                         reader.Position = nullable_pos;
 607
 608                         putback_char = '?';
 609                 }
 610
 611                 public void PutbackCloseParens ()
 612                 {
 613                         putback_char = ')';
 614                 }
 615
 616
 617                 int nullable_pos = -1;
 618
 619                 public void CheckNullable (bool is_nullable)
 620                 {
 621                         if (is_nullable)
 622                                 nullable_pos = reader.Position;
 623                         else
 624                                 nullable_pos = -1;
 625                 }
 626 #endif
 627
 628                 public int peek_token ()
 629                 {
 630                         int the_token;
 631
 632                         PushPosition ();
 633                         the_token = token ();
 634                         PopPosition ();
 635
 636                         return the_token;
 637                 }
 638
 639                 bool parse_namespace_or_typename (int next)
 640                 {
 641                         if (next == -1)
 642                                 next = peek_token ();
 643                         while (next == Token.IDENTIFIER){
 644                                 token ();
 645                           again:
 646                                 next = peek_token ();
 647                                 if (next == Token.DOT || next == Token.DOUBLE_COLON){
 648                                         token ();
 649                                         next = peek_token ();
 650                                         continue;
 651                                 }
 652                                 if (next == Token.OP_GENERICS_LT){
 653                                         token ();
 654                                         if (!parse_less_than ())
 655                                                 return false;
 656                                         goto again;
 657                                 }
 658                                 return true;
 659                         }
 660
 661                         return false;
 662                 }
 663
 664                 bool is_simple_type (int token)
 665                 {
 666                         return  (token == Token.BOOL ||
 667                                  token == Token.DECIMAL ||
 668                                  token == Token.SBYTE ||
 669                                  token == Token.BYTE ||
 670                                  token == Token.SHORT ||
 671                                  token == Token.USHORT ||
 672                                  token == Token.INT ||
 673                                  token == Token.UINT ||
 674                                  token == Token.LONG ||
 675                                  token == Token.ULONG ||
 676                                  token == Token.CHAR ||
 677                                  token == Token.FLOAT ||
 678                                  token == Token.DOUBLE);
 679                 }
 680
 681                 bool is_builtin_reference_type (int token)
 682                 {
 683                         return (token == Token.OBJECT || token == Token.STRING);
 684                 }
 685
 686                 bool parse_opt_rank (int next)
 687                 {
 688                         while (true){
 689                                 if (next != Token.OPEN_BRACKET)
 690                                         return true;
 691
 692                                 token ();
 693                                 while (true){
 694                                         next = token ();
 695                                         if (next == Token.CLOSE_BRACKET){
 696                                                 next = peek_token ();
 697                                                 break;
 698                                         }
 699                                         if (next == Token.COMMA)
 700                                                 continue;
 701
 702                                         return false;
 703                                 }
 704                         }
 705                 }
 706
 707                 bool parse_type ()
 708                 {
 709                         int next = peek_token ();
 710
 711                         if (is_simple_type (next)){
 712                                 token ();
 713                                 next = peek_token ();
 714                                 if (next == Token.INTERR)
 715                                         token ();
 716                                 return parse_opt_rank (peek_token ());
 717                         }
 718                         if (parse_namespace_or_typename (next)){
 719                                 next = peek_token ();
 720                                 if (next == Token.INTERR)
 721                                         token ();
 722                                 return parse_opt_rank (peek_token ());
 723                         } else if (is_builtin_reference_type (next)){
 724                                 token ();
 725                                 return parse_opt_rank (peek_token ());
 726                         }
 727
 728                         return false;
 729                 }
 730
 731                 //
 732                 // Invoked after '(' has been seen and tries to parse:
 733                 // type identifier [, type identifier]*
 734                 //
 735                 // if this is the case, instead of returning an
 736                 // OPEN_PARENS token we return a special token that
 737                 // triggers lambda parsing.
 738                 //
 739                 // This is needed because we can not introduce the
 740                 // explicitly_typed_lambda_parameter_list after a '(' in the
 741                 // grammar without introducing reduce/reduce conflicts.
 742                 //
 743                 // We need to parse a type and if it is followed by an
 744                 // identifier, we know it has to be parsed as a lambda
 745                 // expression.
 746                 //
 747                 // the type expression can be prefixed with `ref' or `out'
 748                 //
 749                 public bool parse_lambda_parameters ()
 750                 {
 751                         while (true){
 752                                 int next = peek_token ();
 753
 754                                 if (next == Token.REF || next == Token.OUT)
 755                                         token ();
 756
 757                                 if (parse_type ()){
 758                                         next = peek_token ();
 759                                         if (next == Token.IDENTIFIER){
 760                                                 token ();
 761                                                 next = peek_token ();
 762                                                 if (next == Token.COMMA){
 763                                                         token ();
 764                                                         continue;
 765                                                 }
 766                                                 if (next == Token.CLOSE_PARENS)
 767                                                         return true;
 768                                         }
 769                                 }
 770                                 return false;
 771                         }
 772                 }
 773
 774                 int parsing_generic_less_than = 0;
 775
 776                 int is_punct (char c, ref bool doread)
 777                 {
 778                         int d;
 779                         int t;
 780
 781                         doread = false;
 782
 783                         switch (c){
 784                         case '{':
 785                                 val = Location;
 786                                 return Token.OPEN_BRACE;
 787                         case '}':
 788                                 val = Location;
 789                                 return Token.CLOSE_BRACE;
 790                         case '[':
 791                                 // To block doccomment inside attribute declaration.
 792                                 if (doc_state == XmlCommentState.Allowed)
 793                                         doc_state = XmlCommentState.NotAllowed;
 794                                 return Token.OPEN_BRACKET;
 795                         case ']':
 796                                 return Token.CLOSE_BRACKET;
 797                         case '(':
 798                                 if (linq){
 799                                         PushPosition ();
 800                                         bool have_lambda_parameter = parse_lambda_parameters ();
 801                                         PopPosition ();
 802
 803                                         if (have_lambda_parameter)
 804                                                 return Token.OPEN_PARENS_LAMBDA;
 805                                         else
 806                                                 return Token.OPEN_PARENS;
 807                                 } else
 808                                         return Token.OPEN_PARENS;
 809                         case ')': {
 810                                 if (deambiguate_close_parens == 0)
 811                                         return Token.CLOSE_PARENS;
 812
 813                                 --deambiguate_close_parens;
 814
 815                                 PushPosition ();
 816
 817                                 int new_token = xtoken ();
 818
 819                                 PopPosition ();
 820
 821                                 if (new_token == Token.OPEN_PARENS)
 822                                         return Token.CLOSE_PARENS_OPEN_PARENS;
 823                                 else if (new_token == Token.MINUS)
 824                                         return Token.CLOSE_PARENS_MINUS;
 825                                 else if (IsCastToken (new_token))
 826                                         return Token.CLOSE_PARENS_CAST;
 827                                 else
 828                                         return Token.CLOSE_PARENS_NO_CAST;
 829                         }
 830
 831                         case ',':
 832                                 return Token.COMMA;
 833                         case ';':
 834                                 val = Location;
 835                                 return Token.SEMICOLON;
 836                         case '~':
 837                                 val = Location;
 838                                 return Token.TILDE;
 839                         case '?':
 840                                 return Token.INTERR;
 841                         }
 842 #if GMCS_SOURCE
 843                         if (c == '<') {
 844                                 if (parsing_generic_less_than++ > 0)
 845                                         return Token.OP_GENERICS_LT;
 846
 847                                 if (handle_typeof) {
 848                                         int dimension;
 849                                         PushPosition ();
 850                                         if (parse_generic_dimension (out dimension)) {
 851                                                 val = dimension;
 852                                                 DiscardPosition ();
 853                                                 return Token.GENERIC_DIMENSION;
 854                                         }
 855                                         PopPosition ();
 856                                 }
 857
 858                                 // Save current position and parse next token.
 859                                 PushPosition ();
 860                                 bool is_generic_lt = parse_less_than ();
 861                                 PopPosition ();
 862
 863                                 if (is_generic_lt) {
 864                                         return Token.OP_GENERICS_LT;
 865                                 } else
 866                                         parsing_generic_less_than = 0;
 867
 868                                 d = peekChar ();
 869                                 if (d == '<'){
 870                                         getChar ();
 871                                         d = peekChar ();
 872
 873                                         if (d == '='){
 874                                                 doread = true;
 875                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 876                                         }
 877                                         return Token.OP_SHIFT_LEFT;
 878                                 } else if (d == '='){
 879                                         doread = true;
 880                                         return Token.OP_LE;
 881                                 }
 882                                 return Token.OP_LT;
 883                         } else if (c == '>') {
 884                                 if (parsing_generic_less_than > 0) {
 885                                         parsing_generic_less_than--;
 886                                         return Token.OP_GENERICS_GT;
 887                                 }
 888
 889                                 d = peekChar ();
 890                                 if (d == '>'){
 891                                         getChar ();
 892                                         d = peekChar ();
 893
 894                                         if (d == '='){
 895                                                 doread = true;
 896                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 897                                         }
 898                                         return Token.OP_SHIFT_RIGHT;
 899                                 } else if (d == '='){
 900                                         doread = true;
 901                                         return Token.OP_GE;
 902                                 }
 903                                 return Token.OP_GT;
 904                         }
 905 #endif
 906                         d = peekChar ();
 907                         if (c == '+'){
 908
 909                                 if (d == '+') {
 910                                         val = Location;
 911                                         t = Token.OP_INC;
 912                                 }
 913                                 else if (d == '=')
 914                                         t = Token.OP_ADD_ASSIGN;
 915                                 else {
 916                                         val = Location;
 917                                         return Token.PLUS;
 918                                 }
 919                                 doread = true;
 920                                 return t;
 921                         }
 922                         if (c == '-'){
 923                                 if (d == '-') {
 924                                         val = Location;
 925                                         t = Token.OP_DEC;
 926                                 }
 927                                 else if (d == '=')
 928                                         t = Token.OP_SUB_ASSIGN;
 929                                 else if (d == '>')
 930                                         t = Token.OP_PTR;
 931                                 else {
 932                                         val = Location;
 933                                         return Token.MINUS;
 934                                 }
 935                                 doread = true;
 936                                 return t;
 937                         }
 938
 939                         if (c == '!'){
 940                                 if (d == '='){
 941                                         doread = true;
 942                                         return Token.OP_NE;
 943                                 }
 944                                 val = Location;
 945                                 return Token.BANG;
 946                         }
 947
 948                         if (c == '='){
 949                                 if (d == '='){
 950                                         doread = true;
 951                                         return Token.OP_EQ;
 952                                 }
 953                                 if (d == '>'){
 954                                         doread = true;
 955                                         val = Location;
 956                                         return Token.ARROW;
 957                                 }
 958                                 return Token.ASSIGN;
 959                         }
 960
 961                         if (c == '&'){
 962                                 if (d == '&'){
 963                                         doread = true;
 964                                         return Token.OP_AND;
 965                                 } else if (d == '='){
 966                                         doread = true;
 967                                         return Token.OP_AND_ASSIGN;
 968                                 }
 969                                 val = Location;
 970                                 return Token.BITWISE_AND;
 971                         }
 972
 973                         if (c == '|'){
 974                                 if (d == '|'){
 975                                         doread = true;
 976                                         return Token.OP_OR;
 977                                 } else if (d == '='){
 978                                         doread = true;
 979                                         return Token.OP_OR_ASSIGN;
 980                                 }
 981                                 return Token.BITWISE_OR;
 982                         }
 983
 984                         if (c == '*'){
 985                                 if (d == '='){
 986                                         doread = true;
 987                                         return Token.OP_MULT_ASSIGN;
 988                                 }
 989                                 val = Location;
 990                                 return Token.STAR;
 991                         }
 992
 993                         if (c == '/'){
 994                                 if (d == '='){
 995                                         doread = true;
 996                                         return Token.OP_DIV_ASSIGN;
 997                                 }
 998                                 return Token.DIV;
 999                         }
1000
1001                         if (c == '%'){
1002                                 if (d == '='){
1003                                         doread = true;
1004                                         return Token.OP_MOD_ASSIGN;
1005                                 }
1006                                 return Token.PERCENT;
1007                         }
1008
1009                         if (c == '^'){
1010                                 if (d == '='){
1011                                         doread = true;
1012                                         return Token.OP_XOR_ASSIGN;
1013                                 }
1014                                 return Token.CARRET;
1015                         }
1016
1017 #if !GMCS_SOURCE
1018                         if (c == '<'){
1019                                 if (d == '<'){
1020                                         getChar ();
1021                                         d = peekChar ();
1022
1023                                         if (d == '='){
1024                                                 doread = true;
1025                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
1026                                         }
1027                                         return Token.OP_SHIFT_LEFT;
1028                                 } else if (d == '='){
1029                                         doread = true;
1030                                         return Token.OP_LE;
1031                                 }
1032                                 return Token.OP_LT;
1033                         }
1034
1035                         if (c == '>'){
1036                                 if (d == '>'){
1037                                         getChar ();
1038                                         d = peekChar ();
1039
1040                                         if (d == '='){
1041                                                 doread = true;
1042                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
1043                                         }
1044                                         return Token.OP_SHIFT_RIGHT;
1045                                 } else if (d == '='){
1046                                         doread = true;
1047                                         return Token.OP_GE;
1048                                 }
1049                                 return Token.OP_GT;
1050                         }
1051 #endif
1052                         if (c == ':'){
1053                                 if (d == ':'){
1054                                         doread = true;
1055                                         return Token.DOUBLE_COLON;
1056                                 }
1057                                 val = Location;
1058                                 return Token.COLON;
1059                         }
1060
1061                         return Token.ERROR;
1062                 }
1063
1064                 int deambiguate_close_parens = 0;
1065
1066                 public void Deambiguate_CloseParens (object expression)
1067                 {
1068                         putback (')');
1069
1070                         // When any binary operation is used we are sure it is not a cast
1071                         if (expression is Binary)
1072                                 return;
1073
1074                         deambiguate_close_parens++;
1075                 }
1076
1077                 bool decimal_digits (int c)
1078                 {
1079                         int d;
1080                         bool seen_digits = false;
1081
1082                         if (c != -1){
1083                                 if (number_pos == max_number_size)
1084                                         Error_NumericConstantTooLong ();
1085                                 number_builder [number_pos++] = (char) c;
1086                         }
1087
1088                         //
1089                         // We use peekChar2, because decimal_digits needs to do a
1090                         // 2-character look-ahead (5.ToString for example).
1091                         //
1092                         while ((d = peekChar2 ()) != -1){
1093                                 if (d >= '0' && d <= '9'){
1094                                         if (number_pos == max_number_size)
1095                                                 Error_NumericConstantTooLong ();
1096                                         number_builder [number_pos++] = (char) d;
1097                                         getChar ();
1098                                         seen_digits = true;
1099                                 } else
1100                                         break;
1101                         }
1102
1103                         return seen_digits;
1104                 }
1105
1106                 static bool is_hex (int e)
1107                 {
1108                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1109                 }
1110
1111                 static int real_type_suffix (int c)
1112                 {
1113                         int t;
1114
1115                         switch (c){
1116                         case 'F': case 'f':
1117                                 t =  Token.LITERAL_FLOAT;
1118                                 break;
1119                         case 'D': case 'd':
1120                                 t = Token.LITERAL_DOUBLE;
1121                                 break;
1122                         case 'M': case 'm':
1123                                  t= Token.LITERAL_DECIMAL;
1124                                 break;
1125                         default:
1126                                 return Token.NONE;
1127                         }
1128                         return t;
1129                 }
1130
1131                 int integer_type_suffix (ulong ul, int c)
1132                 {
1133                         bool is_unsigned = false;
1134                         bool is_long = false;
1135
1136                         if (c != -1){
1137                                 bool scanning = true;
1138                                 do {
1139                                         switch (c){
1140                                         case 'U': case 'u':
1141                                                 if (is_unsigned)
1142                                                         scanning = false;
1143                                                 is_unsigned = true;
1144                                                 getChar ();
1145                                                 break;
1146
1147                                         case 'l':
1148                                                 if (!is_unsigned && (RootContext.WarningLevel >= 4)){
1149                                                         //
1150                                                         // if we have not seen anything in between
1151                                                         // report this error
1152                                                         //
1153                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1154                                                 }
1155                                                 //
1156                                                 // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
1157                                                 // work around that.
1158                                                 //
1159                                                 //goto case 'L';
1160                                                 if (is_long)
1161                                                         scanning = false;
1162                                                 is_long = true;
1163                                                 getChar ();
1164                                                 break;
1165
1166                                         case 'L':
1167                                                 if (is_long)
1168                                                         scanning = false;
1169                                                 is_long = true;
1170                                                 getChar ();
1171                                                 break;
1172
1173                                         default:
1174                                                 scanning = false;
1175                                                 break;
1176                                         }
1177                                         c = peekChar ();
1178                                 } while (scanning);
1179                         }
1180
1181                         if (is_long && is_unsigned){
1182                                 val = ul;
1183                                 return Token.LITERAL_INTEGER;
1184                         } else if (is_unsigned){
1185                                 // uint if possible, or ulong else.
1186
1187                                 if ((ul & 0xffffffff00000000) == 0)
1188                                         val = (uint) ul;
1189                                 else
1190                                         val = ul;
1191                         } else if (is_long){
1192                                 // long if possible, ulong otherwise
1193                                 if ((ul & 0x8000000000000000) != 0)
1194                                         val = ul;
1195                                 else
1196                                         val = (long) ul;
1197                         } else {
1198                                 // int, uint, long or ulong in that order
1199                                 if ((ul & 0xffffffff00000000) == 0){
1200                                         uint ui = (uint) ul;
1201
1202                                         if ((ui & 0x80000000) != 0)
1203                                                 val = ui;
1204                                         else
1205                                                 val = (int) ui;
1206                                 } else {
1207                                         if ((ul & 0x8000000000000000) != 0)
1208                                                 val = ul;
1209                                         else
1210                                                 val = (long) ul;
1211                                 }
1212                         }
1213                         return Token.LITERAL_INTEGER;
1214                 }
1215
1216                 //
1217                 // given `c' as the next char in the input decide whether
1218                 // we need to convert to a special type, and then choose
1219                 // the best representation for the integer
1220                 //
1221                 int adjust_int (int c)
1222                 {
1223                         try {
1224                                 if (number_pos > 9){
1225                                         ulong ul = (uint) (number_builder [0] - '0');
1226
1227                                         for (int i = 1; i < number_pos; i++){
1228                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1229                                         }
1230                                         return integer_type_suffix (ul, c);
1231                                 } else {
1232                                         uint ui = (uint) (number_builder [0] - '0');
1233
1234                                         for (int i = 1; i < number_pos; i++){
1235                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1236                                         }
1237                                         return integer_type_suffix (ui, c);
1238                                 }
1239                         } catch (OverflowException) {
1240                                 error_details = "Integral constant is too large";
1241                                 Report.Error (1021, Location, error_details);
1242                                 val = 0ul;
1243                                 return Token.LITERAL_INTEGER;
1244                         }
1245                         catch (FormatException) {
1246                                 Report.Error (1013, Location, "Invalid number");
1247                                 val = 0ul;
1248                                 return Token.LITERAL_INTEGER;
1249                         }
1250                 }
1251
1252                 int adjust_real (int t)
1253                 {
1254                         string s = new String (number_builder, 0, number_pos);
1255                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1256
1257                         switch (t){
1258                         case Token.LITERAL_DECIMAL:
1259                                 try {
1260                                         val = System.Decimal.Parse (s, styles, csharp_format_info);
1261                                 } catch (OverflowException) {
1262                                         val = 0m;
1263                                         Report.Error (594, Location, error_details, "decimal");
1264                                 }
1265                                 break;
1266                         case Token.LITERAL_FLOAT:
1267                                 try {
1268                                         val = float.Parse (s, styles, csharp_format_info);
1269                                 } catch (OverflowException) {
1270                                         val = 0.0f;
1271                                         Report.Error (594, Location, error_details, "float");
1272                                 }
1273                                 break;
1274
1275                         case Token.LITERAL_DOUBLE:
1276                         case Token.NONE:
1277                                 t = Token.LITERAL_DOUBLE;
1278                                 try {
1279                                         val = System.Double.Parse (s, styles, csharp_format_info);
1280                                 } catch (OverflowException) {
1281                                         val = 0.0;
1282                                         Report.Error (594, Location, error_details, "double");
1283                                 }
1284                                 break;
1285                         }
1286                         return t;
1287                 }
1288
1289                 int handle_hex ()
1290                 {
1291                         int d;
1292                         ulong ul;
1293
1294                         getChar ();
1295                         while ((d = peekChar ()) != -1){
1296                                 if (is_hex (d)){
1297                                         number_builder [number_pos++] = (char) d;
1298                                         getChar ();
1299                                 } else
1300                                         break;
1301                         }
1302
1303                         string s = new String (number_builder, 0, number_pos);
1304                         try {
1305                                 if (number_pos <= 8)
1306                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1307                                 else
1308                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1309                         } catch (OverflowException){
1310                                 error_details = "Integral constant is too large";
1311                                 Report.Error (1021, Location, error_details);
1312                                 val = 0ul;
1313                                 return Token.LITERAL_INTEGER;
1314                         }
1315                         catch (FormatException) {
1316                                 Report.Error (1013, Location, "Invalid number");
1317                                 val = 0ul;
1318                                 return Token.LITERAL_INTEGER;
1319                         }
1320
1321                         return integer_type_suffix (ul, peekChar ());
1322                 }
1323
1324                 //
1325                 // Invoked if we know we have .digits or digits
1326                 //
1327                 int is_number (int c)
1328                 {
1329                         bool is_real = false;
1330                         int type;
1331
1332                         number_pos = 0;
1333
1334                         if (c >= '0' && c <= '9'){
1335                                 if (c == '0'){
1336                                         int peek = peekChar ();
1337
1338                                         if (peek == 'x' || peek == 'X')
1339                                                 return handle_hex ();
1340                                 }
1341                                 decimal_digits (c);
1342                                 c = getChar ();
1343                         }
1344
1345                         //
1346                         // We need to handle the case of
1347                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1348                         //
1349                         if (c == '.'){
1350                                 if (decimal_digits ('.')){
1351                                         is_real = true;
1352                                         c = getChar ();
1353                                 } else {
1354                                         putback ('.');
1355                                         number_pos--;
1356                                         return adjust_int (-1);
1357                                 }
1358                         }
1359
1360                         if (c == 'e' || c == 'E'){
1361                                 is_real = true;
1362                                 if (number_pos == max_number_size)
1363                                         Error_NumericConstantTooLong ();
1364                                 number_builder [number_pos++] = 'e';
1365                                 c = getChar ();
1366
1367                                 if (c == '+'){
1368                                         if (number_pos == max_number_size)
1369                                                 Error_NumericConstantTooLong ();
1370                                         number_builder [number_pos++] = '+';
1371                                         c = -1;
1372                                 } else if (c == '-') {
1373                                         if (number_pos == max_number_size)
1374                                                 Error_NumericConstantTooLong ();
1375                                         number_builder [number_pos++] = '-';
1376                                         c = -1;
1377                                 } else {
1378                                         if (number_pos == max_number_size)
1379                                                 Error_NumericConstantTooLong ();
1380                                         number_builder [number_pos++] = '+';
1381                                 }
1382
1383                                 decimal_digits (c);
1384                                 c = getChar ();
1385                         }
1386
1387                         type = real_type_suffix (c);
1388                         if (type == Token.NONE && !is_real){
1389                                 putback (c);
1390                                 return adjust_int (c);
1391                         } else
1392                                 is_real = true;
1393
1394                         if (type == Token.NONE){
1395                                 putback (c);
1396                         }
1397
1398                         if (is_real)
1399                                 return adjust_real (type);
1400
1401                         Console.WriteLine ("This should not be reached");
1402                         throw new Exception ("Is Number should never reach this point");
1403                 }
1404
1405                 //
1406                 // Accepts exactly count (4 or 8) hex, no more no less
1407                 //
1408                 int getHex (int count, out bool error)
1409                 {
1410                         int i;
1411                         int total = 0;
1412                         int c;
1413                         int top = count != -1 ? count : 4;
1414
1415                         getChar ();
1416                         error = false;
1417                         for (i = 0; i < top; i++){
1418                                 c = getChar ();
1419
1420                                 if (c >= '0' && c <= '9')
1421                                         c = (int) c - (int) '0';
1422                                 else if (c >= 'A' && c <= 'F')
1423                                         c = (int) c - (int) 'A' + 10;
1424                                 else if (c >= 'a' && c <= 'f')
1425                                         c = (int) c - (int) 'a' + 10;
1426                                 else {
1427                                         error = true;
1428                                         return 0;
1429                                 }
1430
1431                                 total = (total * 16) + c;
1432                                 if (count == -1){
1433                                         int p = peekChar ();
1434                                         if (p == -1)
1435                                                 break;
1436                                         if (!is_hex ((char)p))
1437                                                 break;
1438                                 }
1439                         }
1440                         return total;
1441                 }
1442
1443                 int escape (int c)
1444                 {
1445                         bool error;
1446                         int d;
1447                         int v;
1448
1449                         d = peekChar ();
1450                         if (c != '\\')
1451                                 return c;
1452
1453                         switch (d){
1454                         case 'a':
1455                                 v = '\a'; break;
1456                         case 'b':
1457                                 v = '\b'; break;
1458                         case 'n':
1459                                 v = '\n'; break;
1460                         case 't':
1461                                 v = '\t'; break;
1462                         case 'v':
1463                                 v = '\v'; break;
1464                         case 'r':
1465                                 v = '\r'; break;
1466                         case '\\':
1467                                 v = '\\'; break;
1468                         case 'f':
1469                                 v = '\f'; break;
1470                         case '0':
1471                                 v = 0; break;
1472                         case '"':
1473                                 v = '"'; break;
1474                         case '\'':
1475                                 v = '\''; break;
1476                         case 'x':
1477                                 v = getHex (-1, out error);
1478                                 if (error)
1479                                         goto default;
1480                                 return v;
1481                         case 'u':
1482                                 v = getHex (4, out error);
1483                                 if (error)
1484                                         goto default;
1485                                 return v;
1486                         case 'U':
1487                                 v = getHex (8, out error);
1488                                 if (error)
1489                                         goto default;
1490                                 return v;
1491                         default:
1492                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1493                                 return d;
1494                         }
1495                         getChar ();
1496                         return v;
1497                 }
1498
1499                 int getChar ()
1500                 {
1501                         int x;
1502                         if (putback_char != -1) {
1503                                 x = putback_char;
1504                                 putback_char = -1;
1505                         } else
1506                                 x = reader.Read ();
1507                         if (x == '\n') {
1508                                 line++;
1509                                 ref_line++;
1510                                 previous_col = col;
1511                                 col = 0;
1512                         }
1513                         else
1514                                 col++;
1515                         return x;
1516                 }
1517
1518                 int peekChar ()
1519                 {
1520                         if (putback_char != -1)
1521                                 return putback_char;
1522                         putback_char = reader.Read ();
1523                         return putback_char;
1524                 }
1525
1526                 int peekChar2 ()
1527                 {
1528                         if (putback_char != -1)
1529                                 return putback_char;
1530                         return reader.Peek ();
1531                 }
1532
1533                 void putback (int c)
1534                 {
1535                         if (putback_char != -1){
1536                                 Console.WriteLine ("Col: " + col);
1537                                 Console.WriteLine ("Row: " + line);
1538                                 Console.WriteLine ("Name: " + ref_name.Name);
1539                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1540                                 throw new Exception ("This should not happen putback on putback");
1541                         }
1542                         if (c == '\n' || col == 0) {
1543                                 // It won't happen though.
1544                                 line--;
1545                                 ref_line--;
1546                                 col = previous_col;
1547                         }
1548                         else
1549                                 col--;
1550                         putback_char = c;
1551                 }
1552
1553                 public bool advance ()
1554                 {
1555                         return peekChar () != -1;
1556                 }
1557
1558                 public Object Value {
1559                         get {
1560                                 return val;
1561                         }
1562                 }
1563
1564                 public Object value ()
1565                 {
1566                         return val;
1567                 }
1568
1569                 static bool IsCastToken (int token)
1570                 {
1571                         switch (token) {
1572                         case Token.BANG:
1573                         case Token.TILDE:
1574                         case Token.IDENTIFIER:
1575                         case Token.LITERAL_INTEGER:
1576                         case Token.LITERAL_FLOAT:
1577                         case Token.LITERAL_DOUBLE:
1578                         case Token.LITERAL_DECIMAL:
1579                         case Token.LITERAL_CHARACTER:
1580                         case Token.LITERAL_STRING:
1581                         case Token.BASE:
1582                         case Token.CHECKED:
1583                         case Token.DELEGATE:
1584                         case Token.FALSE:
1585                         case Token.FIXED:
1586                         case Token.NEW:
1587                         case Token.NULL:
1588                         case Token.SIZEOF:
1589                         case Token.THIS:
1590                         case Token.THROW:
1591                         case Token.TRUE:
1592                         case Token.TYPEOF:
1593                         case Token.UNCHECKED:
1594                         case Token.UNSAFE:
1595 #if GMCS_SOURCE
1596                         case Token.DEFAULT:
1597 #endif
1598
1599                                 //
1600                                 // These can be part of a member access
1601                                 //
1602                         case Token.INT:
1603                         case Token.UINT:
1604                         case Token.SHORT:
1605                         case Token.USHORT:
1606                         case Token.LONG:
1607                         case Token.ULONG:
1608                         case Token.DOUBLE:
1609                         case Token.FLOAT:
1610                         case Token.CHAR:
1611                                 return true;
1612
1613                         default:
1614                                 return false;
1615                         }
1616                 }
1617
1618                 public int token ()
1619                 {
1620                         current_token = xtoken ();
1621
1622 #if GMCS_SOURCE
1623                         if (current_token != Token.DEFAULT)
1624                                 return current_token;
1625
1626                         PushPosition();
1627                         int c = xtoken();
1628                         if (c == -1)
1629                                 current_token = Token.ERROR;
1630                         else if (c == Token.OPEN_PARENS)
1631                                 current_token = Token.DEFAULT_OPEN_PARENS;
1632                         else if (c == Token.COLON)
1633                                 current_token = Token.DEFAULT_COLON;
1634                         else
1635                                 PopPosition();
1636 #endif
1637                         return current_token;
1638                 }
1639
1640                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1641
1642                 void get_cmd_arg (out string cmd, out string arg)
1643                 {
1644                         int c;
1645
1646                         tokens_seen = false;
1647                         arg = "";
1648                         static_cmd_arg.Length = 0;
1649
1650                         // skip over white space
1651                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1652                                 ;
1653
1654                         while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
1655                                 if (is_identifier_part_character ((char) c)){
1656                                         static_cmd_arg.Append ((char) c);
1657                                         c = getChar ();
1658                                 } else {
1659                                         putback (c);
1660                                         break;
1661                                 }
1662                         }
1663
1664                         cmd = static_cmd_arg.ToString ();
1665
1666                         if (c == '\n' || c == '\r'){
1667                                 return;
1668                         }
1669
1670                         // skip over white space
1671                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1672                                 ;
1673
1674                         if (c == '\n'){
1675                                 return;
1676                         } else if (c == '\r'){
1677                                 return;
1678                         } else if (c == -1){
1679                                 arg = "";
1680                                 return;
1681                         }
1682
1683                         static_cmd_arg.Length = 0;
1684                         static_cmd_arg.Append ((char) c);
1685
1686                         while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
1687                                 static_cmd_arg.Append ((char) c);
1688                         }
1689
1690                         arg = static_cmd_arg.ToString ();
1691                 }
1692
1693                 //
1694                 // Handles the #line directive
1695                 //
1696                 bool PreProcessLine (string arg)
1697                 {
1698                         if (arg.Length == 0)
1699                                 return false;
1700
1701                         if (arg == "default"){
1702                                 ref_line = line;
1703                                 ref_name = file_name;
1704                                 Location.Push (ref_name);
1705                                 return true;
1706                         } else if (arg == "hidden"){
1707                                 //
1708                                 // We ignore #line hidden
1709                                 //
1710                                 return true;
1711                         }
1712
1713                         try {
1714                                 int pos;
1715
1716                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1717                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1718                                         pos++;
1719
1720                                         char [] quotes = { '\"' };
1721
1722                                         string name = arg.Substring (pos). Trim (quotes);
1723                                         ref_name = Location.LookupFile (name);
1724                                         file_name.HasLineDirective = true;
1725                                         ref_name.HasLineDirective = true;
1726                                         Location.Push (ref_name);
1727                                 } else {
1728                                         ref_line = System.Int32.Parse (arg);
1729                                 }
1730                         } catch {
1731                                 return false;
1732                         }
1733
1734                         return true;
1735                 }
1736
1737                 //
1738                 // Handles #define and #undef
1739                 //
1740                 void PreProcessDefinition (bool is_define, string arg, bool caller_is_taking)
1741                 {
1742                         if (arg.Length == 0 || arg == "true" || arg == "false"){
1743                                 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
1744                                 return;
1745                         }
1746
1747                         if (arg.IndexOfAny (simple_whitespaces) != -1){
1748                                 Error_EndLineExpected ();
1749                                 return;
1750                         }
1751
1752                         if (!is_identifier_start_character (arg [0]))
1753                                 Report.Error (1001, Location, "Identifier expected: " + arg);
1754
1755                         foreach (char c in arg.Substring (1)){
1756                                 if (!is_identifier_part_character (c)){
1757                                         Report.Error (1001, Location, "Identifier expected: " + arg);
1758                                         return;
1759                                 }
1760                         }
1761
1762                         if (!caller_is_taking)
1763                                 return;
1764
1765                         if (is_define){
1766                                 if (defines == null)
1767                                         defines = new Hashtable ();
1768                                 define (arg);
1769                         } else {
1770                                 if (defines == null)
1771                                         return;
1772                                 if (defines.Contains (arg))
1773                                         defines.Remove (arg);
1774                         }
1775                 }
1776
1777                 /// <summary>
1778                 /// Handles #pragma directive
1779                 /// </summary>
1780                 void PreProcessPragma (string arg)
1781                 {
1782                         const string warning = "warning";
1783                         const string w_disable = "warning disable";
1784                         const string w_restore = "warning restore";
1785
1786                         if (arg == w_disable) {
1787                                 Report.RegisterWarningRegion (Location).WarningDisable (line);
1788                                 return;
1789                         }
1790
1791                         if (arg == w_restore) {
1792                                 Report.RegisterWarningRegion (Location).WarningEnable (line);
1793                                 return;
1794                         }
1795
1796                         if (arg.StartsWith (w_disable)) {
1797                                 int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
1798                                 foreach (int code in codes) {
1799                                         if (code != 0)
1800                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1801                                 }
1802                                 return;
1803                         }
1804
1805                         if (arg.StartsWith (w_restore)) {
1806                                 int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
1807                                 Hashtable w_table = Report.warning_ignore_table;
1808                                 foreach (int code in codes) {
1809                                         if (w_table != null && w_table.Contains (code))
1810                                                 Report.Warning (1635, 1, Location, String.Format ("Cannot restore warning `CS{0:0000}' because it was disabled globally", code));
1811                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1812                                 }
1813                                 return;
1814                         }
1815
1816                         if (arg.StartsWith (warning)) {
1817                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
1818                                 return;
1819                         }
1820
1821                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
1822                 }
1823
1824                 int[] ParseNumbers (string text)
1825                 {
1826                         string[] string_array = text.Split (',');
1827                         int[] values = new int [string_array.Length];
1828                         int index = 0;
1829                         foreach (string string_code in string_array) {
1830                                 try {
1831                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1832                                 }
1833                                 catch (FormatException) {
1834                                         Report.Warning (1692, 1, Location, "Invalid number");
1835                                 }
1836                         }
1837                         return values;
1838                 }
1839
1840                 bool eval_val (string s)
1841                 {
1842                         if (s == "true")
1843                                 return true;
1844                         if (s == "false")
1845                                 return false;
1846
1847                         if (defines == null)
1848                                 return false;
1849                         if (defines.Contains (s))
1850                                 return true;
1851
1852                         return false;
1853                 }
1854
1855                 bool pp_primary (ref string s)
1856                 {
1857                         s = s.Trim ();
1858                         int len = s.Length;
1859
1860                         if (len > 0){
1861                                 char c = s [0];
1862
1863                                 if (c == '('){
1864                                         s = s.Substring (1);
1865                                         bool val = pp_expr (ref s, false);
1866                                         if (s.Length > 0 && s [0] == ')'){
1867                                                 s = s.Substring (1);
1868                                                 return val;
1869                                         }
1870                                         Error_InvalidDirective ();
1871                                         return false;
1872                                 }
1873
1874                                 if (is_identifier_start_character (c)){
1875                                         int j = 1;
1876
1877                                         while (j < len){
1878                                                 c = s [j];
1879
1880                                                 if (is_identifier_part_character (c)){
1881                                                         j++;
1882                                                         continue;
1883                                                 }
1884                                                 bool v = eval_val (s.Substring (0, j));
1885                                                 s = s.Substring (j);
1886                                                 return v;
1887                                         }
1888                                         bool vv = eval_val (s);
1889                                         s = "";
1890                                         return vv;
1891                                 }
1892                         }
1893                         Error_InvalidDirective ();
1894                         return false;
1895                 }
1896
1897                 bool pp_unary (ref string s)
1898                 {
1899                         s = s.Trim ();
1900                         int len = s.Length;
1901
1902                         if (len > 0){
1903                                 if (s [0] == '!'){
1904                                         if (len > 1 && s [1] == '='){
1905                                                 Error_InvalidDirective ();
1906                                                 return false;
1907                                         }
1908                                         s = s.Substring (1);
1909                                         return ! pp_primary (ref s);
1910                                 } else
1911                                         return pp_primary (ref s);
1912                         } else {
1913                                 Error_InvalidDirective ();
1914                                 return false;
1915                         }
1916                 }
1917
1918                 bool pp_eq (ref string s)
1919                 {
1920                         bool va = pp_unary (ref s);
1921
1922                         s = s.Trim ();
1923                         int len = s.Length;
1924                         if (len > 0){
1925                                 if (s [0] == '='){
1926                                         if (len > 2 && s [1] == '='){
1927                                                 s = s.Substring (2);
1928                                                 return va == pp_unary (ref s);
1929                                         } else {
1930                                                 Error_InvalidDirective ();
1931                                                 return false;
1932                                         }
1933                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
1934                                         s = s.Substring (2);
1935
1936                                         return va != pp_unary (ref s);
1937
1938                                 }
1939                         }
1940
1941                         return va;
1942
1943                 }
1944
1945                 bool pp_and (ref string s)
1946                 {
1947                         bool va = pp_eq (ref s);
1948
1949                         s = s.Trim ();
1950                         int len = s.Length;
1951                         if (len > 0){
1952                                 if (s [0] == '&'){
1953                                         if (len > 2 && s [1] == '&'){
1954                                                 s = s.Substring (2);
1955                                                 return (va & pp_and (ref s));
1956                                         } else {
1957                                                 Error_InvalidDirective ();
1958                                                 return false;
1959                                         }
1960                                 }
1961                         }
1962                         return va;
1963                 }
1964
1965                 //
1966                 // Evaluates an expression for `#if' or `#elif'
1967                 //
1968                 bool pp_expr (ref string s, bool isTerm)
1969                 {
1970                         bool va = pp_and (ref s);
1971                         s = s.Trim ();
1972                         int len = s.Length;
1973                         if (len > 0){
1974                                 char c = s [0];
1975
1976                                 if (c == '|'){
1977                                         if (len > 2 && s [1] == '|'){
1978                                                 s = s.Substring (2);
1979                                                 return va | pp_expr (ref s, isTerm);
1980                                         } else {
1981                                                 Error_InvalidDirective ();
1982                                                 return false;
1983                                         }
1984                                 }
1985                                 if (isTerm) {
1986                                         Error_EndLineExpected ();
1987                                         return false;
1988                                 }
1989                         }
1990
1991                         return va;
1992                 }
1993
1994                 bool eval (string s)
1995                 {
1996                         bool v = pp_expr (ref s, true);
1997                         s = s.Trim ();
1998                         if (s.Length != 0){
1999                                 return false;
2000                         }
2001
2002                         return v;
2003                 }
2004
2005                 void Error_NumericConstantTooLong ()
2006                 {
2007                         Report.Error (1021, Location, "Numeric constant too long");
2008                 }
2009
2010                 void Error_InvalidDirective ()
2011                 {
2012                         Report.Error (1517, Location, "Invalid preprocessor directive");
2013                 }
2014
2015                 void Error_UnexpectedDirective (string extra)
2016                 {
2017                         Report.Error (
2018                                 1028, Location,
2019                                 "Unexpected processor directive (" + extra + ")");
2020                 }
2021
2022                 void Error_TokensSeen ()
2023                 {
2024                         Report.Error (1032, Location,
2025                                 "Cannot define or undefine preprocessor symbols after first token in file");
2026                 }
2027
2028                 void Eror_WrongPreprocessorLocation ()
2029                 {
2030                         Report.Error (1040, Location,
2031                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2032                 }
2033
2034                 void Error_EndLineExpected ()
2035                 {
2036                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2037                 }
2038
2039                 //
2040                 // if true, then the code continues processing the code
2041                 // if false, the code stays in a loop until another directive is
2042                 // reached.
2043                 // When caller_is_taking is false we ignore all directives except the ones
2044                 // which can help us to identify where the #if block ends
2045                 bool handle_preprocessing_directive (bool caller_is_taking)
2046                 {
2047                         string cmd, arg;
2048                         bool region_directive = false;
2049
2050                         get_cmd_arg (out cmd, out arg);
2051
2052                         // Eat any trailing whitespaces and single-line comments
2053                         if (arg.IndexOf ("//") != -1)
2054                                 arg = arg.Substring (0, arg.IndexOf ("//"));
2055                         arg = arg.Trim (simple_whitespaces);
2056
2057                         //
2058                         // The first group of pre-processing instructions is always processed
2059                         //
2060                         switch (cmd){
2061                         case "region":
2062                                 region_directive = true;
2063                                 arg = "true";
2064                                 goto case "if";
2065
2066                         case "endregion":
2067                                 if (ifstack == null || ifstack.Count == 0){
2068                                         Error_UnexpectedDirective ("no #region for this #endregion");
2069                                         return true;
2070                                 }
2071                                 int pop = (int) ifstack.Pop ();
2072
2073                                 if ((pop & REGION) == 0)
2074                                         Report.Error (1027, Location, "Expected `#endif' directive");
2075
2076                                 return caller_is_taking;
2077
2078                         case "if":
2079                                 if (ifstack == null)
2080                                         ifstack = new Stack (2);
2081
2082                                 int flags = region_directive ? REGION : 0;
2083                                 if (ifstack.Count == 0){
2084                                         flags |= PARENT_TAKING;
2085                                 } else {
2086                                         int state = (int) ifstack.Peek ();
2087                                         if ((state & TAKING) != 0) {
2088                                                 flags |= PARENT_TAKING;
2089                                         }
2090                                 }
2091
2092                                 if (caller_is_taking && eval (arg)) {
2093                                         ifstack.Push (flags | TAKING);
2094                                         return true;
2095                                 }
2096                                 ifstack.Push (flags);
2097                                 return false;
2098
2099                         case "endif":
2100                                 if (ifstack == null || ifstack.Count == 0){
2101                                         Error_UnexpectedDirective ("no #if for this #endif");
2102                                         return true;
2103                                 } else {
2104                                         pop = (int) ifstack.Pop ();
2105
2106                                         if ((pop & REGION) != 0)
2107                                                 Report.Error (1038, Location, "#endregion directive expected");
2108
2109                                         if (arg.Length != 0) {
2110                                                 Error_EndLineExpected ();
2111                                         }
2112
2113                                         if (ifstack.Count == 0)
2114                                                 return true;
2115
2116                                         int state = (int) ifstack.Peek ();
2117                                         return (state & TAKING) != 0;
2118                                 }
2119
2120                         case "elif":
2121                                 if (ifstack == null || ifstack.Count == 0){
2122                                         Error_UnexpectedDirective ("no #if for this #elif");
2123                                         return true;
2124                                 } else {
2125                                         int state = (int) ifstack.Pop ();
2126
2127                                         if ((state & REGION) != 0) {
2128                                                 Report.Error (1038, Location, "#endregion directive expected");
2129                                                 return true;
2130                                         }
2131
2132                                         if ((state & ELSE_SEEN) != 0){
2133                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2134                                                 return true;
2135                                         }
2136
2137                                         if ((state & TAKING) != 0) {
2138                                                 ifstack.Push (0);
2139                                                 return false;
2140                                         }
2141
2142                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2143                                                 ifstack.Push (state | TAKING);
2144                                                 return true;
2145                                         }
2146
2147                                         ifstack.Push (state);
2148                                         return false;
2149                                 }
2150
2151                         case "else":
2152                                 if (ifstack == null || ifstack.Count == 0){
2153                                         Error_UnexpectedDirective ("no #if for this #else");
2154                                         return true;
2155                                 } else {
2156                                         int state = (int) ifstack.Peek ();
2157
2158                                         if ((state & REGION) != 0) {
2159                                                 Report.Error (1038, Location, "#endregion directive expected");
2160                                                 return true;
2161                                         }
2162
2163                                         if ((state & ELSE_SEEN) != 0){
2164                                                 Error_UnexpectedDirective ("#else within #else");
2165                                                 return true;
2166                                         }
2167
2168                                         ifstack.Pop ();
2169
2170                                         if (arg.Length != 0) {
2171                                                 Error_EndLineExpected ();
2172                                                 return true;
2173                                         }
2174
2175                                         bool ret = false;
2176                                         if ((state & PARENT_TAKING) != 0) {
2177                                                 ret = (state & TAKING) == 0;
2178
2179                                                 if (ret)
2180                                                         state |= TAKING;
2181                                                 else
2182                                                         state &= ~TAKING;
2183                                         }
2184
2185                                         ifstack.Push (state | ELSE_SEEN);
2186
2187                                         return ret;
2188                                 }
2189                                 case "define":
2190                                         if (any_token_seen){
2191                                                 Error_TokensSeen ();
2192                                                 return caller_is_taking;
2193                                         }
2194                                         PreProcessDefinition (true, arg, caller_is_taking);
2195                                         return caller_is_taking;
2196
2197                                 case "undef":
2198                                         if (any_token_seen){
2199                                                 Error_TokensSeen ();
2200                                                 return caller_is_taking;
2201                                         }
2202                                         PreProcessDefinition (false, arg, caller_is_taking);
2203                                         return caller_is_taking;
2204                         }
2205
2206                         //
2207                         // These are only processed if we are in a `taking' block
2208                         //
2209                         if (!caller_is_taking)
2210                                 return false;
2211
2212                         switch (cmd){
2213                         case "error":
2214                                 Report.Error (1029, Location, "#error: '" + arg + "'");
2215                                 return true;
2216
2217                         case "warning":
2218                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2219                                 return true;
2220
2221                         case "pragma":
2222                                 if (RootContext.Version == LanguageVersion.ISO_1) {
2223                                         Report.FeatureIsNotISO1 (Location, "#pragma");
2224                                         return true;
2225                                 }
2226
2227                                 PreProcessPragma (arg);
2228                                 return true;
2229
2230                         case "line":
2231                                 if (!PreProcessLine (arg))
2232                                         Report.Error (
2233                                                 1576, Location,
2234                                                 "The line number specified for #line directive is missing or invalid");
2235                                 return caller_is_taking;
2236                         }
2237
2238                         Report.Error (1024, Location, "Wrong preprocessor directive");
2239                         return true;
2240
2241                 }
2242
2243                 private int consume_string (bool quoted)
2244                 {
2245                         int c;
2246                         string_builder.Length = 0;
2247
2248                         while ((c = getChar ()) != -1){
2249                                 if (c == '"'){
2250                                         if (quoted && peekChar () == '"'){
2251                                                 string_builder.Append ((char) c);
2252                                                 getChar ();
2253                                                 continue;
2254                                         } else {
2255                                                 val = string_builder.ToString ();
2256                                                 return Token.LITERAL_STRING;
2257                                         }
2258                                 }
2259
2260                                 if (c == '\n'){
2261                                         if (!quoted)
2262                                                 Report.Error (1010, Location, "Newline in constant");
2263                                 }
2264
2265                                 if (!quoted){
2266                                         c = escape (c);
2267                                         if (c == -1)
2268                                                 return Token.ERROR;
2269                                 }
2270                                 string_builder.Append ((char) c);
2271                         }
2272
2273                         Report.Error (1039, Location, "Unterminated string literal");
2274                         return Token.EOF;
2275                 }
2276
2277                 private int consume_identifier (int s)
2278                 {
2279                         int res = consume_identifier (s, false);
2280
2281                         if (doc_state == XmlCommentState.Allowed)
2282                                 doc_state = XmlCommentState.NotAllowed;
2283                         switch (res) {
2284                         case Token.USING:
2285                         case Token.NAMESPACE:
2286                                 check_incorrect_doc_comment ();
2287                                 break;
2288                         }
2289
2290                         if (res == Token.PARTIAL) {
2291                                 // Save current position and parse next token.
2292                                 PushPosition ();
2293
2294                                 int next_token = token ();
2295                                 bool ok = (next_token == Token.CLASS) ||
2296                                         (next_token == Token.STRUCT) ||
2297                                         (next_token == Token.INTERFACE);
2298
2299                                 PopPosition ();
2300
2301                                 if (ok)
2302                                         return res;
2303
2304                                 if (next_token < Token.LAST_KEYWORD)
2305                                         Report.Error (267, Location, "The `partial' modifier can be used only immediately before keyword `class', `struct', or `interface'");
2306
2307                                 val = new LocatedToken (Location, "partial");
2308                                 return Token.IDENTIFIER;
2309                         }
2310
2311                         return res;
2312                 }
2313
2314                 private int consume_identifier (int s, bool quoted)
2315                 {
2316                         int pos = 1;
2317                         int c = -1;
2318
2319                         id_builder [0] = (char) s;
2320
2321                         current_location = new Location (ref_line, Col);
2322
2323                         while ((c = getChar ()) != -1) {
2324                         loop:
2325                                 if (is_identifier_part_character ((char) c)){
2326                                         if (pos == max_id_size){
2327                                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2328                                                 return Token.ERROR;
2329                                         }
2330
2331                                         id_builder [pos++] = (char) c;
2332 //                                      putback_char = -1;
2333                                 } else if (c == '\\') {
2334                                         c = escape (c);
2335                                         goto loop;
2336                                 } else {
2337 //                                      putback_char = c;
2338                                         putback (c);
2339                                         break;
2340                                 }
2341                         }
2342
2343                         //
2344                         // Optimization: avoids doing the keyword lookup
2345                         // on uppercase letters and _
2346                         //
2347                         if (!quoted && (s >= 'a' || s == '_')){
2348                                 int keyword = GetKeyword (id_builder, pos);
2349                                 if (keyword != -1) {
2350                                         val = Location;
2351                                         return keyword;
2352                                 }
2353                         }
2354
2355                         //
2356                         // Keep identifiers in an array of hashtables to avoid needless
2357                         // allocations
2358                         //
2359
2360                         if (identifiers [pos] != null) {
2361                                 val = identifiers [pos][id_builder];
2362                                 if (val != null) {
2363                                         val = new LocatedToken (Location, (string) val);
2364                                         if (quoted)
2365                                                 escapedIdentifiers.Add (val);
2366                                         return Token.IDENTIFIER;
2367                                 }
2368                         }
2369                         else
2370                                 identifiers [pos] = new CharArrayHashtable (pos);
2371
2372                         val = new String (id_builder, 0, pos);
2373                         if (RootContext.Version == LanguageVersion.ISO_1) {
2374                                 for (int i = 1; i < id_builder.Length; i += 3) {
2375                                         if (id_builder [i] == '_' && (id_builder [i - 1] == '_' || id_builder [i + 1] == '_')) {
2376                                                 Report.Error (1638, Location,
2377                                                         "`{0}': Any identifier with double underscores cannot be used when ISO language version mode is specified", val.ToString ());
2378                                                 break;
2379                                         }
2380                                 }
2381                         }
2382
2383                         char [] chars = new char [pos];
2384                         Array.Copy (id_builder, chars, pos);
2385
2386                         identifiers [pos] [chars] = val;
2387
2388                         val = new LocatedToken (Location, (string) val);
2389                         if (quoted)
2390                                 escapedIdentifiers.Add (val);
2391                         return Token.IDENTIFIER;
2392                 }
2393
2394                 public int xtoken ()
2395                 {
2396                         int t;
2397                         bool doread = false;
2398                         int c;
2399
2400                         // Whether we have seen comments on the current line
2401                         bool comments_seen = false;
2402                         val = null;
2403                         for (;(c = getChar ()) != -1;) {
2404                                 if (c == '\t'){
2405                                         col = ((col + 8) / 8) * 8;
2406                                         continue;
2407                                 }
2408
2409                                 if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0 || c == 0)
2410                                         continue;
2411
2412                                 if (c == '\r') {
2413                                         if (peekChar () == '\n')
2414                                                 getChar ();
2415
2416                                         any_token_seen |= tokens_seen;
2417                                         tokens_seen = false;
2418                                         comments_seen = false;
2419                                         continue;
2420                                 }
2421
2422                                 // Handle double-slash comments.
2423                                 if (c == '/'){
2424                                         int d = peekChar ();
2425
2426                                         if (d == '/'){
2427                                                 getChar ();
2428                                                 if (RootContext.Documentation != null && peekChar () == '/') {
2429                                                         getChar ();
2430                                                         // Don't allow ////.
2431                                                         if ((d = peekChar ()) != '/') {
2432                                                                 update_comment_location ();
2433                                                                 if (doc_state == XmlCommentState.Allowed)
2434                                                                         handle_one_line_xml_comment ();
2435                                                                 else if (doc_state == XmlCommentState.NotAllowed)
2436                                                                         warn_incorrect_doc_comment ();
2437                                                         }
2438                                                 }
2439                                                 while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
2440                                                         if (d == '\n'){
2441                                                         }
2442                                                 any_token_seen |= tokens_seen;
2443                                                 tokens_seen = false;
2444                                                 comments_seen = false;
2445                                                 continue;
2446                                         } else if (d == '*'){
2447                                                 getChar ();
2448                                                 bool docAppend = false;
2449                                                 if (RootContext.Documentation != null && peekChar () == '*') {
2450                                                         getChar ();
2451                                                         update_comment_location ();
2452                                                         // But when it is /**/, just do nothing.
2453                                                         if (peekChar () == '/') {
2454                                                                 getChar ();
2455                                                                 continue;
2456                                                         }
2457                                                         if (doc_state == XmlCommentState.Allowed)
2458                                                                 docAppend = true;
2459                                                         else if (doc_state == XmlCommentState.NotAllowed)
2460                                                                 warn_incorrect_doc_comment ();
2461                                                 }
2462
2463                                                 int current_comment_start = 0;
2464                                                 if (docAppend) {
2465                                                         current_comment_start = xml_comment_buffer.Length;
2466                                                         xml_comment_buffer.Append (Environment.NewLine);
2467                                                 }
2468
2469                                                 Location start_location = Location;
2470
2471                                                 while ((d = getChar ()) != -1){
2472                                                         if (d == '*' && peekChar () == '/'){
2473                                                                 getChar ();
2474                                                                 comments_seen = true;
2475                                                                 break;
2476                                                         }
2477                                                         if (docAppend)
2478                                                                 xml_comment_buffer.Append ((char) d);
2479
2480                                                         if (d == '\n'){
2481                                                                 any_token_seen |= tokens_seen;
2482                                                                 tokens_seen = false;
2483                                                                 //
2484                                                                 // Reset 'comments_seen' just to be consistent.
2485                                                                 // It doesn't matter either way, here.
2486                                                                 //
2487                                                                 comments_seen = false;
2488                                                         }
2489                                                 }
2490                                                 if (!comments_seen)
2491                                                         Report.Error (1035, start_location, "End-of-file found, '*/' expected");
2492
2493                                                 if (docAppend)
2494                                                         update_formatted_doc_comment (current_comment_start);
2495                                                 continue;
2496                                         }
2497                                         goto is_punct_label;
2498                                 }
2499
2500
2501                                 if (c == '\\' || is_identifier_start_character ((char)c)){
2502                                         tokens_seen = true;
2503                                         return consume_identifier (c);
2504                                 }
2505
2506                         is_punct_label:
2507                                 current_location = new Location (ref_line, Col);
2508                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
2509                                         tokens_seen = true;
2510                                         if (doread){
2511                                                 getChar ();
2512                                         }
2513                                         return t;
2514                                 }
2515
2516                                 // white space
2517                                 if (c == '\n'){
2518                                         any_token_seen |= tokens_seen;
2519                                         tokens_seen = false;
2520                                         comments_seen = false;
2521                                         continue;
2522                                 }
2523
2524                                 if (c >= '0' && c <= '9'){
2525                                         tokens_seen = true;
2526                                         return is_number (c);
2527                                 }
2528
2529                                 if (c == '.'){
2530                                         tokens_seen = true;
2531                                         int peek = peekChar ();
2532                                         if (peek >= '0' && peek <= '9')
2533                                                 return is_number (c);
2534                                         return Token.DOT;
2535                                 }
2536
2537                                 if (c == '#') {
2538                                         if (tokens_seen || comments_seen) {
2539                                                 Eror_WrongPreprocessorLocation ();
2540                                                 return Token.ERROR;
2541                                         }
2542
2543                                         if (handle_preprocessing_directive (true))
2544                                                 continue;
2545
2546                                         bool directive_expected = false;
2547                                         while ((c = getChar ()) != -1) {
2548                                                 if (col == 1) {
2549                                                         directive_expected = true;
2550                                                 } else if (!directive_expected) {
2551                                                         // TODO: Implement comment support for disabled code and uncomment this code
2552 //                                                      if (c == '#') {
2553 //                                                              Eror_WrongPreprocessorLocation ();
2554 //                                                              return Token.ERROR;
2555 //                                                      }
2556                                                         continue;
2557                                                 }
2558
2559                                                 if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
2560                                                         continue;
2561
2562                                                 if (c == '#') {
2563                                                         if (handle_preprocessing_directive (false))
2564                                                                 break;
2565                                                 }
2566                                                 directive_expected = false;
2567                                         }
2568
2569                                         if (c != -1) {
2570                                                 tokens_seen = false;
2571                                                 continue;
2572                                         }
2573
2574                                         return Token.EOF;
2575                                 }
2576
2577                                 if (c == '"')
2578                                         return consume_string (false);
2579
2580                                 if (c == '\''){
2581                                         c = getChar ();
2582                                         tokens_seen = true;
2583                                         if (c == '\''){
2584                                                 error_details = "Empty character literal";
2585                                                 Report.Error (1011, Location, error_details);
2586                                                 return Token.ERROR;
2587                                         }
2588                                         if (c == '\r' || c == '\n') {
2589                                                 Report.Error (1010, Location, "Newline in constant");
2590                                                 return Token.ERROR;
2591                                         }
2592                                         c = escape (c);
2593                                         if (c == -1)
2594                                                 return Token.ERROR;
2595                                         val = new System.Char ();
2596                                         val = (char) c;
2597                                         c = getChar ();
2598
2599                                         if (c != '\''){
2600                                                 error_details = "Too many characters in character literal";
2601                                                 Report.Error (1012, Location, error_details);
2602
2603                                                 // Try to recover, read until newline or next "'"
2604                                                 while ((c = getChar ()) != -1){
2605                                                         if (c == '\n'){
2606                                                                 break;
2607                                                         }
2608                                                         else if (c == '\'')
2609                                                                 break;
2610                                                 }
2611                                                 return Token.ERROR;
2612                                         }
2613                                         return Token.LITERAL_CHARACTER;
2614                                 }
2615
2616                                 if (c == '@') {
2617                                         c = getChar ();
2618                                         if (c == '"') {
2619                                                 tokens_seen = true;
2620                                                 return consume_string (true);
2621                                         } else if (is_identifier_start_character ((char) c)){
2622                                                 return consume_identifier (c, true);
2623                                         } else {
2624                                                 Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
2625                                         }
2626                                 }
2627
2628                                 error_details = ((char)c).ToString ();
2629
2630                                 return Token.ERROR;
2631                         }
2632
2633                         return Token.EOF;
2634                 }
2635
2636                 //
2637                 // Handles one line xml comment
2638                 //
2639                 private void handle_one_line_xml_comment ()
2640                 {
2641                         int c;
2642                         while ((c = peekChar ()) == ' ')
2643                                 getChar (); // skip heading whitespaces.
2644                         while ((c = peekChar ()) != -1 && c != '\n' && c != '\r') {
2645                                 xml_comment_buffer.Append ((char) getChar ());
2646                         }
2647                         if (c == '\r' || c == '\n')
2648                                 xml_comment_buffer.Append (Environment.NewLine);
2649                 }
2650
2651                 //
2652                 // Remove heading "*" in Javadoc-like xml documentation.
2653                 //
2654                 private void update_formatted_doc_comment (int current_comment_start)
2655                 {
2656                         int length = xml_comment_buffer.Length - current_comment_start;
2657                         string [] lines = xml_comment_buffer.ToString (
2658                                 current_comment_start,
2659                                 length).Replace ("\r", "").Split ('\n');
2660
2661                         // The first line starts with /**, thus it is not target
2662                         // for the format check.
2663                         for (int i = 1; i < lines.Length; i++) {
2664                                 string s = lines [i];
2665                                 int idx = s.IndexOf ('*');
2666                                 string head = null;
2667                                 if (idx < 0) {
2668                                         if (i < lines.Length - 1)
2669                                                 return;
2670                                         head = s;
2671                                 } else
2672                                         head = s.Substring (0, idx);
2673                                 foreach (char c in head)
2674                                         if (c != ' ')
2675                                                 return;
2676                                 lines [i] = s.Substring (idx + 1);
2677                         }
2678                         xml_comment_buffer.Remove (current_comment_start, length);
2679                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
2680                 }
2681
2682                 //
2683                 // Updates current comment location.
2684                 //
2685                 private void update_comment_location ()
2686                 {
2687                         if (current_comment_location.IsNull) {
2688                                 // "-2" is for heading "//" or "/*"
2689                                 current_comment_location =
2690                                         new Location (ref_line, col - 2);
2691                         }
2692                 }
2693
2694                 //
2695                 // Checks if there was incorrect doc comments and raise
2696                 // warnings.
2697                 //
2698                 public void check_incorrect_doc_comment ()
2699                 {
2700                         if (xml_comment_buffer.Length > 0)
2701                                 warn_incorrect_doc_comment ();
2702                 }
2703
2704                 //
2705                 // Raises a warning when tokenizer found incorrect doccomment
2706                 // markup.
2707                 //
2708                 private void warn_incorrect_doc_comment ()
2709                 {
2710                         if (doc_state != XmlCommentState.Error) {
2711                                 doc_state = XmlCommentState.Error;
2712                                 // in csc, it is 'XML comment is not placed on
2713                                 // a valid language element'. But that does not
2714                                 // make sense.
2715                                 Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
2716                         }
2717                 }
2718
2719                 //
2720                 // Consumes the saved xml comment lines (if any)
2721                 // as for current target member or type.
2722                 //
2723                 public string consume_doc_comment ()
2724                 {
2725                         if (xml_comment_buffer.Length > 0) {
2726                                 string ret = xml_comment_buffer.ToString ();
2727                                 reset_doc_comment ();
2728                                 return ret;
2729                         }
2730                         return null;
2731                 }
2732
2733                 void reset_doc_comment ()
2734                 {
2735                         xml_comment_buffer.Length = 0;
2736                         current_comment_location = Location.Null;
2737                 }
2738
2739                 public void cleanup ()
2740                 {
2741                         if (ifstack != null && ifstack.Count >= 1) {
2742                                 current_location = new Location (ref_line, Col);
2743                                 int state = (int) ifstack.Pop ();
2744                                 if ((state & REGION) != 0)
2745                                         Report.Error (1038, Location, "#endregion directive expected");
2746                                 else
2747                                         Report.Error (1027, Location, "Expected `#endif' directive");
2748                         }
2749                 }
2750         }
2751
2752         //
2753         // Indicates whether it accepts XML documentation or not.
2754         //
2755         public enum XmlCommentState {
2756                 // comment is allowed in this state.
2757                 Allowed,
2758                 // comment is not allowed in this state.
2759                 NotAllowed,
2760                 // once comments appeared when it is NotAllowed, then the
2761                 // state is changed to it, until the state is changed to
2762                 // .Allowed.
2763                 Error
2764         }
2765 }
2766