mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Licensed under the terms of the GNU GPL
   9 //
  10 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // (C) 2004 Novell, Inc
  12 //
  13
  14 /*
  15  * TODO:
  16  *   Make sure we accept the proper Unicode ranges, per the spec.
  17  *   Report error 1032
  18 */
  19
  20 using System;
  21 using System.Text;
  22 using System.Collections;
  23 using System.IO;
  24 using System.Globalization;
  25 using System.Reflection;
  26
  27 namespace Mono.CSharp
  28 {
  29         /// <summary>
  30         ///    Tokenizer for C# source code.
  31         /// </summary>
  32
  33         public class Tokenizer : yyParser.yyInput
  34         {
  35                 SeekableStreamReader reader;
  36                 SourceFile ref_name;
  37                 SourceFile file_name;
  38                 int ref_line = 1;
  39                 int line = 1;
  40                 int col = 0;
  41                 int previous_col;
  42                 int current_token;
  43                 bool handle_get_set = false;
  44                 bool handle_remove_add = false;
  45                 bool handle_assembly = false;
  46                 bool handle_constraints = false;
  47                 bool handle_typeof = false;
  48                 Location current_location;
  49                 Location current_comment_location = Location.Null;
  50                 ArrayList escapedIdentifiers = new ArrayList ();
  51
  52                 //
  53                 // XML documentation buffer. The save point is used to divide
  54                 // comments on types and comments on members.
  55                 //
  56                 StringBuilder xml_comment_buffer;
  57
  58                 //
  59                 // See comment on XmlCommentState enumeration.
  60                 //
  61                 XmlCommentState xmlDocState = XmlCommentState.Allowed;
  62
  63                 //
  64                 // Whether tokens have been seen on this line
  65                 //
  66                 bool tokens_seen = false;
  67
  68                 //
  69                 // Whether a token has been seen on the file
  70                 // This is needed because `define' is not allowed to be used
  71                 // after a token has been seen.
  72                 //
  73                 bool any_token_seen = false;
  74
  75                 static Hashtable tokenValues;
  76
  77                 private static Hashtable TokenValueName
  78                 {
  79                         get {
  80                                 if (tokenValues == null)
  81                                         tokenValues = GetTokenValueNameHash ();
  82
  83                                 return tokenValues;
  84                         }
  85                 }
  86
  87                 private static Hashtable GetTokenValueNameHash ()
  88                 {
  89                         Type t = typeof (Token);
  90                         FieldInfo [] fields = t.GetFields ();
  91                         Hashtable hash = new Hashtable ();
  92                         foreach (FieldInfo field in fields) {
  93                                 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))
  94                                         hash.Add (field.GetValue (null), field.Name);
  95                         }
  96                         return hash;
  97                 }
  98
  99                 //
 100                 // Returns a verbose representation of the current location
 101                 //
 102                 public string location {
 103                         get {
 104                                 string det;
 105
 106                                 if (current_token == Token.ERROR)
 107                                         det = "detail: " + error_details;
 108                                 else
 109                                         det = "";
 110
 111                                 // return "Line:     "+line+" Col: "+col + "\n" +
 112                                 //       "VirtLine: "+ref_line +
 113                                 //       " Token: "+current_token + " " + det;
 114                                 string current_token_name = TokenValueName [current_token] as string;
 115                                 if (current_token_name == null)
 116                                         current_token_name = current_token.ToString ();
 117
 118                                 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,
 119                                                                                        ref_line,
 120                                                                                        col,
 121                                                                                        current_token_name,
 122                                                                                        det);
 123                         }
 124                 }
 125
 126                 public bool PropertyParsing {
 127                         get {
 128                                 return handle_get_set;
 129                         }
 130
 131                         set {
 132                                 handle_get_set = value;
 133                         }
 134                 }
 135
 136                 public bool AssemblyTargetParsing {
 137                         get {
 138                                 return handle_assembly;
 139                         }
 140
 141                         set {
 142                                 handle_assembly = value;
 143                         }
 144                 }
 145
 146                 public bool EventParsing {
 147                         get {
 148                                 return handle_remove_add;
 149                         }
 150
 151                         set {
 152                                 handle_remove_add = value;
 153                         }
 154                 }
 155
 156                 public bool ConstraintsParsing {
 157                         get {
 158                                 return handle_constraints;
 159                         }
 160
 161                         set {
 162                                 handle_constraints = value;
 163                         }
 164                 }
 165
 166                 public bool TypeOfParsing {
 167                         get {
 168                                 return handle_typeof;
 169                         }
 170
 171                         set {
 172                                 handle_typeof = value;
 173                         }
 174                 }
 175
 176                 public XmlCommentState doc_state {
 177                         get { return xmlDocState; }
 178                         set {
 179                                 if (value == XmlCommentState.Allowed) {
 180                                         check_incorrect_doc_comment ();
 181                                         reset_doc_comment ();
 182                                 }
 183                                 xmlDocState = value;
 184                         }
 185                 }
 186
 187                 public bool IsEscapedIdentifier (Location loc)
 188                 {
 189                         foreach (LocatedToken lt in escapedIdentifiers)
 190                                 if (lt.Location.Equals (loc))
 191                                         return true;
 192                         return false;
 193                 }
 194
 195                 //
 196                 // Class variables
 197                 //
 198                 static CharArrayHashtable[] keywords;
 199                 static Hashtable keywordStrings = new Hashtable ();
 200                 static NumberStyles styles;
 201                 static NumberFormatInfo csharp_format_info;
 202
 203                 //
 204                 // Values for the associated token returned
 205                 //
 206                 int putback_char;
 207                 Object val;
 208
 209                 //
 210                 // Pre-processor
 211                 //
 212                 Hashtable defines;
 213
 214                 const int TAKING        = 1;
 215                 const int ELSE_SEEN     = 4;
 216                 const int PARENT_TAKING = 8;
 217                 const int REGION        = 16;
 218
 219                 //
 220                 // pre-processor if stack state:
 221                 //
 222                 Stack ifstack;
 223
 224                 static System.Text.StringBuilder string_builder;
 225
 226                 const int max_id_size = 512;
 227                 static char [] id_builder = new char [max_id_size];
 228
 229                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 230
 231                 const int max_number_size = 512;
 232                 static char [] number_builder = new char [max_number_size];
 233                 static int number_pos;
 234
 235                 //
 236                 // Details about the error encoutered by the tokenizer
 237                 //
 238                 string error_details;
 239
 240                 public string error {
 241                         get {
 242                                 return error_details;
 243                         }
 244                 }
 245
 246                 public int Line {
 247                         get {
 248                                 return ref_line;
 249                         }
 250                 }
 251
 252                 public int Col {
 253                         get {
 254                                 return col;
 255                         }
 256                 }
 257
 258                 //
 259                 // This is used when the tokenizer needs to save
 260                 // the current position as it needs to do some parsing
 261                 // on its own to deamiguate a token in behalf of the
 262                 // parser.
 263                 //
 264                 Stack position_stack = new Stack (2);
 265                 class Position {
 266                         public int position;
 267                         public int ref_line;
 268                         public int col;
 269                         public int putback_char;
 270                         public int previous_col;
 271                         public Stack ifstack;
 272 #if GMCS_SOURCES
 273                         public int parsing_generic_less_than;
 274 #endif
 275                         public Position (Tokenizer t)
 276                         {
 277                                 position = t.reader.Position;
 278                                 ref_line = t.ref_line;
 279                                 col = t.col;
 280                                 putback_char = t.putback_char;
 281                                 previous_col = t.previous_col;
 282                                 if (t.ifstack != null && t.ifstack.Count != 0)
 283                                         ifstack = (Stack)t.ifstack.Clone ();
 284 #if GMCS_SOURCES
 285                                 parsing_generic_less_than = t.parsing_generic_less_than;
 286 #endif
 287                         }
 288                 }
 289
 290                 public void PushPosition ()
 291                 {
 292                         position_stack.Push (new Position (this));
 293                 }
 294
 295                 public void PopPosition ()
 296                 {
 297                         Position p = (Position) position_stack.Pop ();
 298
 299                         reader.Position = p.position;
 300                         ref_line = p.ref_line;
 301                         col = p.col;
 302                         putback_char = p.putback_char;
 303                         previous_col = p.previous_col;
 304                         ifstack = p.ifstack;
 305                 }
 306
 307                 // Do not reset the position, ignore it.
 308                 public void DiscardPosition ()
 309                 {
 310                         position_stack.Pop ();
 311                 }
 312
 313                 static void AddKeyword (string kw, int token) {
 314                         keywordStrings.Add (kw, kw);
 315                         if (keywords [kw.Length] == null) {
 316                                 keywords [kw.Length] = new CharArrayHashtable (kw.Length);
 317                         }
 318                         keywords [kw.Length] [kw.ToCharArray ()] = token;
 319                 }
 320
 321                 static void InitTokens ()
 322                 {
 323                         keywords = new CharArrayHashtable [64];
 324
 325                         AddKeyword ("__arglist", Token.ARGLIST);
 326                         AddKeyword ("abstract", Token.ABSTRACT);
 327                         AddKeyword ("as", Token.AS);
 328                         AddKeyword ("add", Token.ADD);
 329                         AddKeyword ("assembly", Token.ASSEMBLY);
 330                         AddKeyword ("base", Token.BASE);
 331                         AddKeyword ("bool", Token.BOOL);
 332                         AddKeyword ("break", Token.BREAK);
 333                         AddKeyword ("byte", Token.BYTE);
 334                         AddKeyword ("case", Token.CASE);
 335                         AddKeyword ("catch", Token.CATCH);
 336                         AddKeyword ("char", Token.CHAR);
 337                         AddKeyword ("checked", Token.CHECKED);
 338                         AddKeyword ("class", Token.CLASS);
 339                         AddKeyword ("const", Token.CONST);
 340                         AddKeyword ("continue", Token.CONTINUE);
 341                         AddKeyword ("decimal", Token.DECIMAL);
 342                         AddKeyword ("default", Token.DEFAULT);
 343                         AddKeyword ("delegate", Token.DELEGATE);
 344                         AddKeyword ("do", Token.DO);
 345                         AddKeyword ("double", Token.DOUBLE);
 346                         AddKeyword ("else", Token.ELSE);
 347                         AddKeyword ("enum", Token.ENUM);
 348                         AddKeyword ("event", Token.EVENT);
 349                         AddKeyword ("explicit", Token.EXPLICIT);
 350                         AddKeyword ("extern", Token.EXTERN);
 351                         AddKeyword ("false", Token.FALSE);
 352                         AddKeyword ("finally", Token.FINALLY);
 353                         AddKeyword ("fixed", Token.FIXED);
 354                         AddKeyword ("float", Token.FLOAT);
 355                         AddKeyword ("for", Token.FOR);
 356                         AddKeyword ("foreach", Token.FOREACH);
 357                         AddKeyword ("goto", Token.GOTO);
 358                         AddKeyword ("get", Token.GET);
 359                         AddKeyword ("if", Token.IF);
 360                         AddKeyword ("implicit", Token.IMPLICIT);
 361                         AddKeyword ("in", Token.IN);
 362                         AddKeyword ("int", Token.INT);
 363                         AddKeyword ("interface", Token.INTERFACE);
 364                         AddKeyword ("internal", Token.INTERNAL);
 365                         AddKeyword ("is", Token.IS);
 366                         AddKeyword ("lock", Token.LOCK);
 367                         AddKeyword ("long", Token.LONG);
 368                         AddKeyword ("namespace", Token.NAMESPACE);
 369                         AddKeyword ("new", Token.NEW);
 370                         AddKeyword ("null", Token.NULL);
 371                         AddKeyword ("object", Token.OBJECT);
 372                         AddKeyword ("operator", Token.OPERATOR);
 373                         AddKeyword ("out", Token.OUT);
 374                         AddKeyword ("override", Token.OVERRIDE);
 375                         AddKeyword ("params", Token.PARAMS);
 376                         AddKeyword ("private", Token.PRIVATE);
 377                         AddKeyword ("protected", Token.PROTECTED);
 378                         AddKeyword ("public", Token.PUBLIC);
 379                         AddKeyword ("readonly", Token.READONLY);
 380                         AddKeyword ("ref", Token.REF);
 381                         AddKeyword ("remove", Token.REMOVE);
 382                         AddKeyword ("return", Token.RETURN);
 383                         AddKeyword ("sbyte", Token.SBYTE);
 384                         AddKeyword ("sealed", Token.SEALED);
 385                         AddKeyword ("set", Token.SET);
 386                         AddKeyword ("short", Token.SHORT);
 387                         AddKeyword ("sizeof", Token.SIZEOF);
 388                         AddKeyword ("stackalloc", Token.STACKALLOC);
 389                         AddKeyword ("static", Token.STATIC);
 390                         AddKeyword ("string", Token.STRING);
 391                         AddKeyword ("struct", Token.STRUCT);
 392                         AddKeyword ("switch", Token.SWITCH);
 393                         AddKeyword ("this", Token.THIS);
 394                         AddKeyword ("throw", Token.THROW);
 395                         AddKeyword ("true", Token.TRUE);
 396                         AddKeyword ("try", Token.TRY);
 397                         AddKeyword ("typeof", Token.TYPEOF);
 398                         AddKeyword ("uint", Token.UINT);
 399                         AddKeyword ("ulong", Token.ULONG);
 400                         AddKeyword ("unchecked", Token.UNCHECKED);
 401                         AddKeyword ("unsafe", Token.UNSAFE);
 402                         AddKeyword ("ushort", Token.USHORT);
 403                         AddKeyword ("using", Token.USING);
 404                         AddKeyword ("virtual", Token.VIRTUAL);
 405                         AddKeyword ("void", Token.VOID);
 406                         AddKeyword ("volatile", Token.VOLATILE);
 407                         AddKeyword ("while", Token.WHILE);
 408                         AddKeyword ("partial", Token.PARTIAL);
 409 #if GMCS_SOURCE
 410                         AddKeyword ("where", Token.WHERE);
 411 #endif
 412                 }
 413
 414                 //
 415                 // Class initializer
 416                 //
 417                 static Tokenizer ()
 418                 {
 419                         InitTokens ();
 420                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 421                         styles = NumberStyles.Float;
 422
 423                         string_builder = new System.Text.StringBuilder ();
 424                 }
 425
 426                 int GetKeyword (char[] id, int id_len)
 427                 {
 428                         /*
 429                          * Keywords are stored in an array of hashtables grouped by their
 430                          * length.
 431                          */
 432
 433                         if ((id_len >= keywords.Length) || (keywords [id_len] == null))
 434                                 return -1;
 435                         object o = keywords [id_len] [id];
 436
 437                         if (o == null)
 438                                 return -1;
 439
 440                         int res = (int) o;
 441
 442                         if (handle_get_set == false && (res == Token.GET || res == Token.SET))
 443                                 return -1;
 444                         if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
 445                                 return -1;
 446                         if (handle_assembly == false && res == Token.ASSEMBLY)
 447                                 return -1;
 448 #if GMCS_SOURCE
 449                         if (handle_constraints == false && res == Token.WHERE)
 450                                 return -1;
 451 #endif
 452                         return res;
 453
 454                 }
 455
 456                 public Location Location {
 457                         get { return current_location; }
 458                 }
 459
 460                 void define (string def)
 461                 {
 462                         if (!RootContext.AllDefines.Contains (def)){
 463                                 RootContext.AllDefines [def] = true;
 464                         }
 465                         if (defines.Contains (def))
 466                                 return;
 467                         defines [def] = true;
 468                 }
 469
 470                 public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)
 471                 {
 472                         this.ref_name = file;
 473                         this.file_name = file;
 474                         reader = input;
 475
 476                         putback_char = -1;
 477
 478                         if (defs != null){
 479                                 defines = new Hashtable ();
 480                                 foreach (string def in defs)
 481                                         define (def);
 482                         }
 483
 484                         xml_comment_buffer = new StringBuilder ();
 485
 486                         //
 487                         // FIXME: This could be `Location.Push' but we have to
 488                         // find out why the MS compiler allows this
 489                         //
 490                         Mono.CSharp.Location.Push (file);
 491                 }
 492
 493                 static bool is_identifier_start_character (char c)
 494                 {
 495                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter (c);
 496                 }
 497
 498                 static bool is_identifier_part_character (char c)
 499                 {
 500                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
 501                 }
 502
 503                 public static bool IsKeyword (string s)
 504                 {
 505                         return keywordStrings [s] != null;
 506                 }
 507
 508                 public static bool IsValidIdentifier (string s)
 509                 {
 510                         if (s == null || s.Length == 0)
 511                                 return false;
 512
 513                         if (!is_identifier_start_character (s [0]))
 514                                 return false;
 515
 516                         for (int i = 1; i < s.Length; i ++)
 517                                 if (! is_identifier_part_character (s [i]))
 518                                         return false;
 519
 520                         return true;
 521                 }
 522
 523 #if GMCS_SOURCE
 524                 bool parse_generic_dimension (out int dimension)
 525                 {
 526                         dimension = 1;
 527
 528                 again:
 529                         int the_token = token ();
 530                         if (the_token == Token.OP_GENERICS_GT)
 531                                 return true;
 532                         else if (the_token == Token.COMMA) {
 533                                 dimension++;
 534                                 goto again;
 535                         }
 536
 537                         return false;
 538                 }
 539
 540                 bool parse_less_than ()
 541                 {
 542                 start:
 543                         int the_token = token ();
 544                         if (the_token == Token.OPEN_BRACKET) {
 545                                 do {
 546                                         the_token = token ();
 547                                 } while (the_token != Token.CLOSE_BRACKET);
 548                                 the_token = token ();
 549                         }
 550                         switch (the_token) {
 551                         case Token.IDENTIFIER:
 552                         case Token.OBJECT:
 553                         case Token.STRING:
 554                         case Token.BOOL:
 555                         case Token.DECIMAL:
 556                         case Token.FLOAT:
 557                         case Token.DOUBLE:
 558                         case Token.SBYTE:
 559                         case Token.BYTE:
 560                         case Token.SHORT:
 561                         case Token.USHORT:
 562                         case Token.INT:
 563                         case Token.UINT:
 564                         case Token.LONG:
 565                         case Token.ULONG:
 566                         case Token.CHAR:
 567                         case Token.VOID:
 568                                 break;
 569
 570                         default:
 571                                 return false;
 572                         }
 573                 again:
 574                         the_token = token ();
 575
 576                         if (the_token == Token.OP_GENERICS_GT)
 577                                 return true;
 578                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
 579                                 goto start;
 580                         else if (the_token == Token.INTERR || the_token == Token.STAR)
 581                                 goto again;
 582                         else if (the_token == Token.OP_GENERICS_LT) {
 583                                 if (!parse_less_than ())
 584                                         return false;
 585                                 goto again;
 586                         } else if (the_token == Token.OPEN_BRACKET) {
 587                         rank_specifiers:
 588                                 the_token = token ();
 589                                 if (the_token == Token.CLOSE_BRACKET)
 590                                         goto again;
 591                                 else if (the_token == Token.COMMA)
 592                                         goto rank_specifiers;
 593                                 return false;
 594                         }
 595
 596                         return false;
 597                 }
 598
 599                 int parsing_generic_less_than = 0;
 600
 601                 public void PutbackNullable ()
 602                 {
 603                         if (nullable_pos < 0)
 604                                 throw new Exception ();
 605
 606                         current_token = -1;
 607                         val = null;
 608                         reader.Position = nullable_pos;
 609
 610                         putback_char = '?';
 611                 }
 612
 613                 public void PutbackCloseParens ()
 614                 {
 615                         putback_char = ')';
 616                 }
 617
 618
 619                 int nullable_pos = -1;
 620
 621                 public void CheckNullable (bool is_nullable)
 622                 {
 623                         if (is_nullable)
 624                                 nullable_pos = reader.Position;
 625                         else
 626                                 nullable_pos = -1;
 627                 }
 628 #endif
 629                 int is_punct (char c, ref bool doread)
 630                 {
 631                         int d;
 632                         int t;
 633
 634                         doread = false;
 635
 636                         switch (c){
 637                         case '{':
 638                                 val = Location;
 639                                 return Token.OPEN_BRACE;
 640                         case '}':
 641                                 val = Location;
 642                                 return Token.CLOSE_BRACE;
 643                         case '[':
 644                                 // To block doccomment inside attribute declaration.
 645                                 if (doc_state == XmlCommentState.Allowed)
 646                                         doc_state = XmlCommentState.NotAllowed;
 647                                 return Token.OPEN_BRACKET;
 648                         case ']':
 649                                 return Token.CLOSE_BRACKET;
 650                         case '(':
 651                                 return Token.OPEN_PARENS;
 652                         case ')': {
 653                                 if (deambiguate_close_parens == 0)
 654                                         return Token.CLOSE_PARENS;
 655
 656                                 --deambiguate_close_parens;
 657
 658                                 PushPosition ();
 659
 660                                 int new_token = xtoken ();
 661
 662                                 PopPosition ();
 663
 664                                 if (new_token == Token.OPEN_PARENS)
 665                                         return Token.CLOSE_PARENS_OPEN_PARENS;
 666                                 else if (new_token == Token.MINUS)
 667                                         return Token.CLOSE_PARENS_MINUS;
 668                                 else if (IsCastToken (new_token))
 669                                         return Token.CLOSE_PARENS_CAST;
 670                                 else
 671                                         return Token.CLOSE_PARENS_NO_CAST;
 672                         }
 673
 674                         case ',':
 675                                 return Token.COMMA;
 676                         case ';':
 677                                 val = Location;
 678                                 return Token.SEMICOLON;
 679                         case '~':
 680                                 val = Location;
 681                                 return Token.TILDE;
 682                         case '?':
 683                                 return Token.INTERR;
 684                         }
 685 #if GMCS_SOURCE
 686                         if (c == '<') {
 687                                 if (parsing_generic_less_than++ > 0)
 688                                         return Token.OP_GENERICS_LT;
 689
 690                                 if (handle_typeof) {
 691                                         int dimension;
 692                                         PushPosition ();
 693                                         if (parse_generic_dimension (out dimension)) {
 694                                                 val = dimension;
 695                                                 DiscardPosition ();
 696                                                 return Token.GENERIC_DIMENSION;
 697                                         }
 698                                         PopPosition ();
 699                                 }
 700
 701                                 // Save current position and parse next token.
 702                                 PushPosition ();
 703                                 bool is_generic_lt = parse_less_than ();
 704                                 PopPosition ();
 705
 706                                 if (is_generic_lt) {
 707                                         parsing_generic_less_than++;
 708                                         return Token.OP_GENERICS_LT;
 709                                 } else
 710                                         parsing_generic_less_than = 0;
 711
 712                                 d = peekChar ();
 713                                 if (d == '<'){
 714                                         getChar ();
 715                                         d = peekChar ();
 716
 717                                         if (d == '='){
 718                                                 doread = true;
 719                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 720                                         }
 721                                         return Token.OP_SHIFT_LEFT;
 722                                 } else if (d == '='){
 723                                         doread = true;
 724                                         return Token.OP_LE;
 725                                 }
 726                                 return Token.OP_LT;
 727                         } else if (c == '>') {
 728                                 if (parsing_generic_less_than > 0) {
 729                                         parsing_generic_less_than--;
 730                                         return Token.OP_GENERICS_GT;
 731                                 }
 732
 733                                 d = peekChar ();
 734                                 if (d == '>'){
 735                                         getChar ();
 736                                         d = peekChar ();
 737
 738                                         if (d == '='){
 739                                                 doread = true;
 740                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 741                                         }
 742                                         return Token.OP_SHIFT_RIGHT;
 743                                 } else if (d == '='){
 744                                         doread = true;
 745                                         return Token.OP_GE;
 746                                 }
 747                                 return Token.OP_GT;
 748                         }
 749 #endif
 750                         d = peekChar ();
 751                         if (c == '+'){
 752
 753                                 if (d == '+') {
 754                                         val = Location;
 755                                         t = Token.OP_INC;
 756                                 }
 757                                 else if (d == '=')
 758                                         t = Token.OP_ADD_ASSIGN;
 759                                 else {
 760                                         val = Location;
 761                                         return Token.PLUS;
 762                                 }
 763                                 doread = true;
 764                                 return t;
 765                         }
 766                         if (c == '-'){
 767                                 if (d == '-') {
 768                                         val = Location;
 769                                         t = Token.OP_DEC;
 770                                 }
 771                                 else if (d == '=')
 772                                         t = Token.OP_SUB_ASSIGN;
 773                                 else if (d == '>')
 774                                         t = Token.OP_PTR;
 775                                 else {
 776                                         val = Location;
 777                                         return Token.MINUS;
 778                                 }
 779                                 doread = true;
 780                                 return t;
 781                         }
 782
 783                         if (c == '!'){
 784                                 if (d == '='){
 785                                         doread = true;
 786                                         return Token.OP_NE;
 787                                 }
 788                                 val = Location;
 789                                 return Token.BANG;
 790                         }
 791
 792                         if (c == '='){
 793                                 if (d == '='){
 794                                         doread = true;
 795                                         return Token.OP_EQ;
 796                                 }
 797                                 return Token.ASSIGN;
 798                         }
 799
 800                         if (c == '&'){
 801                                 if (d == '&'){
 802                                         doread = true;
 803                                         return Token.OP_AND;
 804                                 } else if (d == '='){
 805                                         doread = true;
 806                                         return Token.OP_AND_ASSIGN;
 807                                 }
 808                                 val = Location;
 809                                 return Token.BITWISE_AND;
 810                         }
 811
 812                         if (c == '|'){
 813                                 if (d == '|'){
 814                                         doread = true;
 815                                         return Token.OP_OR;
 816                                 } else if (d == '='){
 817                                         doread = true;
 818                                         return Token.OP_OR_ASSIGN;
 819                                 }
 820                                 return Token.BITWISE_OR;
 821                         }
 822
 823                         if (c == '*'){
 824                                 if (d == '='){
 825                                         doread = true;
 826                                         return Token.OP_MULT_ASSIGN;
 827                                 }
 828                                 val = Location;
 829                                 return Token.STAR;
 830                         }
 831
 832                         if (c == '/'){
 833                                 if (d == '='){
 834                                         doread = true;
 835                                         return Token.OP_DIV_ASSIGN;
 836                                 }
 837                                 return Token.DIV;
 838                         }
 839
 840                         if (c == '%'){
 841                                 if (d == '='){
 842                                         doread = true;
 843                                         return Token.OP_MOD_ASSIGN;
 844                                 }
 845                                 return Token.PERCENT;
 846                         }
 847
 848                         if (c == '^'){
 849                                 if (d == '='){
 850                                         doread = true;
 851                                         return Token.OP_XOR_ASSIGN;
 852                                 }
 853                                 return Token.CARRET;
 854                         }
 855
 856 #if !GMCS_SOURCE
 857                         if (c == '<'){
 858                                 if (d == '<'){
 859                                         getChar ();
 860                                         d = peekChar ();
 861
 862                                         if (d == '='){
 863                                                 doread = true;
 864                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 865                                         }
 866                                         return Token.OP_SHIFT_LEFT;
 867                                 } else if (d == '='){
 868                                         doread = true;
 869                                         return Token.OP_LE;
 870                                 }
 871                                 return Token.OP_LT;
 872                         }
 873
 874                         if (c == '>'){
 875                                 if (d == '>'){
 876                                         getChar ();
 877                                         d = peekChar ();
 878
 879                                         if (d == '='){
 880                                                 doread = true;
 881                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 882                                         }
 883                                         return Token.OP_SHIFT_RIGHT;
 884                                 } else if (d == '='){
 885                                         doread = true;
 886                                         return Token.OP_GE;
 887                                 }
 888                                 return Token.OP_GT;
 889                         }
 890 #endif
 891                         if (c == ':'){
 892                                 if (d == ':'){
 893                                         doread = true;
 894                                         return Token.DOUBLE_COLON;
 895                                 }
 896                                 val = Location;
 897                                 return Token.COLON;
 898                         }
 899
 900                         return Token.ERROR;
 901                 }
 902
 903                 int deambiguate_close_parens = 0;
 904
 905                 public void Deambiguate_CloseParens (object expression)
 906                 {
 907                         putback (')');
 908
 909                         // When any binary operation is used we are sure it is not a cast
 910                         if (expression is Binary)
 911                                 return;
 912
 913                         deambiguate_close_parens++;
 914                 }
 915
 916                 bool decimal_digits (int c)
 917                 {
 918                         int d;
 919                         bool seen_digits = false;
 920
 921                         if (c != -1){
 922                                 if (number_pos == max_number_size)
 923                                         Error_NumericConstantTooLong ();
 924                                 number_builder [number_pos++] = (char) c;
 925                         }
 926
 927                         //
 928                         // We use peekChar2, because decimal_digits needs to do a
 929                         // 2-character look-ahead (5.ToString for example).
 930                         //
 931                         while ((d = peekChar2 ()) != -1){
 932                                 if (d >= '0' && d <= '9'){
 933                                         if (number_pos == max_number_size)
 934                                                 Error_NumericConstantTooLong ();
 935                                         number_builder [number_pos++] = (char) d;
 936                                         getChar ();
 937                                         seen_digits = true;
 938                                 } else
 939                                         break;
 940                         }
 941
 942                         return seen_digits;
 943                 }
 944
 945                 static bool is_hex (int e)
 946                 {
 947                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
 948                 }
 949
 950                 static int real_type_suffix (int c)
 951                 {
 952                         int t;
 953
 954                         switch (c){
 955                         case 'F': case 'f':
 956                                 t =  Token.LITERAL_FLOAT;
 957                                 break;
 958                         case 'D': case 'd':
 959                                 t = Token.LITERAL_DOUBLE;
 960                                 break;
 961                         case 'M': case 'm':
 962                                  t= Token.LITERAL_DECIMAL;
 963                                 break;
 964                         default:
 965                                 return Token.NONE;
 966                         }
 967                         return t;
 968                 }
 969
 970                 int integer_type_suffix (ulong ul, int c)
 971                 {
 972                         bool is_unsigned = false;
 973                         bool is_long = false;
 974
 975                         if (c != -1){
 976                                 bool scanning = true;
 977                                 do {
 978                                         switch (c){
 979                                         case 'U': case 'u':
 980                                                 if (is_unsigned)
 981                                                         scanning = false;
 982                                                 is_unsigned = true;
 983                                                 getChar ();
 984                                                 break;
 985
 986                                         case 'l':
 987                                                 if (!is_unsigned && (RootContext.WarningLevel >= 4)){
 988                                                         //
 989                                                         // if we have not seen anything in between
 990                                                         // report this error
 991                                                         //
 992                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
 993                                                 }
 994                                                 //
 995                                                 // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
 996                                                 // work around that.
 997                                                 //
 998                                                 //goto case 'L';
 999                                                 if (is_long)
1000                                                         scanning = false;
1001                                                 is_long = true;
1002                                                 getChar ();
1003                                                 break;
1004
1005                                         case 'L':
1006                                                 if (is_long)
1007                                                         scanning = false;
1008                                                 is_long = true;
1009                                                 getChar ();
1010                                                 break;
1011
1012                                         default:
1013                                                 scanning = false;
1014                                                 break;
1015                                         }
1016                                         c = peekChar ();
1017                                 } while (scanning);
1018                         }
1019
1020                         if (is_long && is_unsigned){
1021                                 val = ul;
1022                                 return Token.LITERAL_INTEGER;
1023                         } else if (is_unsigned){
1024                                 // uint if possible, or ulong else.
1025
1026                                 if ((ul & 0xffffffff00000000) == 0)
1027                                         val = (uint) ul;
1028                                 else
1029                                         val = ul;
1030                         } else if (is_long){
1031                                 // long if possible, ulong otherwise
1032                                 if ((ul & 0x8000000000000000) != 0)
1033                                         val = ul;
1034                                 else
1035                                         val = (long) ul;
1036                         } else {
1037                                 // int, uint, long or ulong in that order
1038                                 if ((ul & 0xffffffff00000000) == 0){
1039                                         uint ui = (uint) ul;
1040
1041                                         if ((ui & 0x80000000) != 0)
1042                                                 val = ui;
1043                                         else
1044                                                 val = (int) ui;
1045                                 } else {
1046                                         if ((ul & 0x8000000000000000) != 0)
1047                                                 val = ul;
1048                                         else
1049                                                 val = (long) ul;
1050                                 }
1051                         }
1052                         return Token.LITERAL_INTEGER;
1053                 }
1054
1055                 //
1056                 // given `c' as the next char in the input decide whether
1057                 // we need to convert to a special type, and then choose
1058                 // the best representation for the integer
1059                 //
1060                 int adjust_int (int c)
1061                 {
1062                         try {
1063                                 if (number_pos > 9){
1064                                         ulong ul = (uint) (number_builder [0] - '0');
1065
1066                                         for (int i = 1; i < number_pos; i++){
1067                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1068                                         }
1069                                         return integer_type_suffix (ul, c);
1070                                 } else {
1071                                         uint ui = (uint) (number_builder [0] - '0');
1072
1073                                         for (int i = 1; i < number_pos; i++){
1074                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1075                                         }
1076                                         return integer_type_suffix (ui, c);
1077                                 }
1078                         } catch (OverflowException) {
1079                                 error_details = "Integral constant is too large";
1080                                 Report.Error (1021, Location, error_details);
1081                                 val = 0ul;
1082                                 return Token.LITERAL_INTEGER;
1083                         }
1084                         catch (FormatException) {
1085                                 Report.Error (1013, Location, "Invalid number");
1086                                 val = 0ul;
1087                                 return Token.LITERAL_INTEGER;
1088                         }
1089                 }
1090
1091                 int adjust_real (int t)
1092                 {
1093                         string s = new String (number_builder, 0, number_pos);
1094                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1095
1096                         switch (t){
1097                         case Token.LITERAL_DECIMAL:
1098                                 try {
1099                                         val = System.Decimal.Parse (s, styles, csharp_format_info);
1100                                 } catch (OverflowException) {
1101                                         val = 0m;
1102                                         Report.Error (594, Location, error_details, "decimal");
1103                                 }
1104                                 break;
1105                         case Token.LITERAL_FLOAT:
1106                                 try {
1107                                         val = float.Parse (s, styles, csharp_format_info);
1108                                 } catch (OverflowException) {
1109                                         val = 0.0f;
1110                                         Report.Error (594, Location, error_details, "float");
1111                                 }
1112                                 break;
1113
1114                         case Token.LITERAL_DOUBLE:
1115                         case Token.NONE:
1116                                 t = Token.LITERAL_DOUBLE;
1117                                 try {
1118                                         val = System.Double.Parse (s, styles, csharp_format_info);
1119                                 } catch (OverflowException) {
1120                                         val = 0.0;
1121                                         Report.Error (594, Location, error_details, "double");
1122                                 }
1123                                 break;
1124                         }
1125                         return t;
1126                 }
1127
1128                 int handle_hex ()
1129                 {
1130                         int d;
1131                         ulong ul;
1132
1133                         getChar ();
1134                         while ((d = peekChar ()) != -1){
1135                                 if (is_hex (d)){
1136                                         number_builder [number_pos++] = (char) d;
1137                                         getChar ();
1138                                 } else
1139                                         break;
1140                         }
1141
1142                         string s = new String (number_builder, 0, number_pos);
1143                         try {
1144                                 if (number_pos <= 8)
1145                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1146                                 else
1147                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1148                         } catch (OverflowException){
1149                                 error_details = "Integral constant is too large";
1150                                 Report.Error (1021, Location, error_details);
1151                                 val = 0ul;
1152                                 return Token.LITERAL_INTEGER;
1153                         }
1154                         catch (FormatException) {
1155                                 Report.Error (1013, Location, "Invalid number");
1156                                 val = 0ul;
1157                                 return Token.LITERAL_INTEGER;
1158                         }
1159
1160                         return integer_type_suffix (ul, peekChar ());
1161                 }
1162
1163                 //
1164                 // Invoked if we know we have .digits or digits
1165                 //
1166                 int is_number (int c)
1167                 {
1168                         bool is_real = false;
1169                         int type;
1170
1171                         number_pos = 0;
1172
1173                         if (c >= '0' && c <= '9'){
1174                                 if (c == '0'){
1175                                         int peek = peekChar ();
1176
1177                                         if (peek == 'x' || peek == 'X')
1178                                                 return handle_hex ();
1179                                 }
1180                                 decimal_digits (c);
1181                                 c = getChar ();
1182                         }
1183
1184                         //
1185                         // We need to handle the case of
1186                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1187                         //
1188                         if (c == '.'){
1189                                 if (decimal_digits ('.')){
1190                                         is_real = true;
1191                                         c = getChar ();
1192                                 } else {
1193                                         putback ('.');
1194                                         number_pos--;
1195                                         return adjust_int (-1);
1196                                 }
1197                         }
1198
1199                         if (c == 'e' || c == 'E'){
1200                                 is_real = true;
1201                                 if (number_pos == max_number_size)
1202                                         Error_NumericConstantTooLong ();
1203                                 number_builder [number_pos++] = 'e';
1204                                 c = getChar ();
1205
1206                                 if (c == '+'){
1207                                         if (number_pos == max_number_size)
1208                                                 Error_NumericConstantTooLong ();
1209                                         number_builder [number_pos++] = '+';
1210                                         c = -1;
1211                                 } else if (c == '-') {
1212                                         if (number_pos == max_number_size)
1213                                                 Error_NumericConstantTooLong ();
1214                                         number_builder [number_pos++] = '-';
1215                                         c = -1;
1216                                 } else {
1217                                         if (number_pos == max_number_size)
1218                                                 Error_NumericConstantTooLong ();
1219                                         number_builder [number_pos++] = '+';
1220                                 }
1221
1222                                 decimal_digits (c);
1223                                 c = getChar ();
1224                         }
1225
1226                         type = real_type_suffix (c);
1227                         if (type == Token.NONE && !is_real){
1228                                 putback (c);
1229                                 return adjust_int (c);
1230                         } else
1231                                 is_real = true;
1232
1233                         if (type == Token.NONE){
1234                                 putback (c);
1235                         }
1236
1237                         if (is_real)
1238                                 return adjust_real (type);
1239
1240                         Console.WriteLine ("This should not be reached");
1241                         throw new Exception ("Is Number should never reach this point");
1242                 }
1243
1244                 //
1245                 // Accepts exactly count (4 or 8) hex, no more no less
1246                 //
1247                 int getHex (int count, out bool error)
1248                 {
1249                         int i;
1250                         int total = 0;
1251                         int c;
1252                         int top = count != -1 ? count : 4;
1253
1254                         getChar ();
1255                         error = false;
1256                         for (i = 0; i < top; i++){
1257                                 c = getChar ();
1258
1259                                 if (c >= '0' && c <= '9')
1260                                         c = (int) c - (int) '0';
1261                                 else if (c >= 'A' && c <= 'F')
1262                                         c = (int) c - (int) 'A' + 10;
1263                                 else if (c >= 'a' && c <= 'f')
1264                                         c = (int) c - (int) 'a' + 10;
1265                                 else {
1266                                         error = true;
1267                                         return 0;
1268                                 }
1269
1270                                 total = (total * 16) + c;
1271                                 if (count == -1){
1272                                         int p = peekChar ();
1273                                         if (p == -1)
1274                                                 break;
1275                                         if (!is_hex ((char)p))
1276                                                 break;
1277                                 }
1278                         }
1279                         return total;
1280                 }
1281
1282                 int escape (int c)
1283                 {
1284                         bool error;
1285                         int d;
1286                         int v;
1287
1288                         d = peekChar ();
1289                         if (c != '\\')
1290                                 return c;
1291
1292                         switch (d){
1293                         case 'a':
1294                                 v = '\a'; break;
1295                         case 'b':
1296                                 v = '\b'; break;
1297                         case 'n':
1298                                 v = '\n'; break;
1299                         case 't':
1300                                 v = '\t'; break;
1301                         case 'v':
1302                                 v = '\v'; break;
1303                         case 'r':
1304                                 v = '\r'; break;
1305                         case '\\':
1306                                 v = '\\'; break;
1307                         case 'f':
1308                                 v = '\f'; break;
1309                         case '0':
1310                                 v = 0; break;
1311                         case '"':
1312                                 v = '"'; break;
1313                         case '\'':
1314                                 v = '\''; break;
1315                         case 'x':
1316                                 v = getHex (-1, out error);
1317                                 if (error)
1318                                         goto default;
1319                                 return v;
1320                         case 'u':
1321                                 v = getHex (4, out error);
1322                                 if (error)
1323                                         goto default;
1324                                 return v;
1325                         case 'U':
1326                                 v = getHex (8, out error);
1327                                 if (error)
1328                                         goto default;
1329                                 return v;
1330                         default:
1331                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1332                                 return d;
1333                         }
1334                         getChar ();
1335                         return v;
1336                 }
1337
1338                 int getChar ()
1339                 {
1340                         int x;
1341                         if (putback_char != -1) {
1342                                 x = putback_char;
1343                                 putback_char = -1;
1344                         } else
1345                                 x = reader.Read ();
1346                         if (x == '\n') {
1347                                 line++;
1348                                 ref_line++;
1349                                 previous_col = col;
1350                                 col = 0;
1351                         }
1352                         else
1353                                 col++;
1354                         return x;
1355                 }
1356
1357                 int peekChar ()
1358                 {
1359                         if (putback_char != -1)
1360                                 return putback_char;
1361                         putback_char = reader.Read ();
1362                         return putback_char;
1363                 }
1364
1365                 int peekChar2 ()
1366                 {
1367                         if (putback_char != -1)
1368                                 return putback_char;
1369                         return reader.Peek ();
1370                 }
1371
1372                 void putback (int c)
1373                 {
1374                         if (putback_char != -1){
1375                                 Console.WriteLine ("Col: " + col);
1376                                 Console.WriteLine ("Row: " + line);
1377                                 Console.WriteLine ("Name: " + ref_name.Name);
1378                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1379                                 throw new Exception ("This should not happen putback on putback");
1380                         }
1381                         if (c == '\n' || col == 0) {
1382                                 // It won't happen though.
1383                                 line--;
1384                                 ref_line--;
1385                                 col = previous_col;
1386                         }
1387                         else
1388                                 col--;
1389                         putback_char = c;
1390                 }
1391
1392                 public bool advance ()
1393                 {
1394                         return peekChar () != -1;
1395                 }
1396
1397                 public Object Value {
1398                         get {
1399                                 return val;
1400                         }
1401                 }
1402
1403                 public Object value ()
1404                 {
1405                         return val;
1406                 }
1407
1408                 static bool IsCastToken (int token)
1409                 {
1410                         switch (token) {
1411                         case Token.BANG:
1412                         case Token.TILDE:
1413                         case Token.IDENTIFIER:
1414                         case Token.LITERAL_INTEGER:
1415                         case Token.LITERAL_FLOAT:
1416                         case Token.LITERAL_DOUBLE:
1417                         case Token.LITERAL_DECIMAL:
1418                         case Token.LITERAL_CHARACTER:
1419                         case Token.LITERAL_STRING:
1420                         case Token.BASE:
1421                         case Token.CHECKED:
1422                         case Token.DELEGATE:
1423                         case Token.FALSE:
1424                         case Token.FIXED:
1425                         case Token.NEW:
1426                         case Token.NULL:
1427                         case Token.SIZEOF:
1428                         case Token.THIS:
1429                         case Token.THROW:
1430                         case Token.TRUE:
1431                         case Token.TYPEOF:
1432                         case Token.UNCHECKED:
1433                         case Token.UNSAFE:
1434 #if GMCS_SOURCE
1435                         case Token.DEFAULT:
1436 #endif
1437
1438                                 //
1439                                 // These can be part of a member access
1440                                 //
1441                         case Token.INT:
1442                         case Token.UINT:
1443                         case Token.SHORT:
1444                         case Token.USHORT:
1445                         case Token.LONG:
1446                         case Token.ULONG:
1447                         case Token.DOUBLE:
1448                         case Token.FLOAT:
1449                         case Token.CHAR:
1450                                 return true;
1451
1452                         default:
1453                                 return false;
1454                         }
1455                 }
1456
1457                 public int token ()
1458                 {
1459                         current_token = xtoken ();
1460
1461 #if GMCS_SOURCE
1462                         if (current_token != Token.DEFAULT)
1463                                 return current_token;
1464
1465                         PushPosition();
1466                         int c = xtoken();
1467                         if (c == -1)
1468                                 current_token = Token.ERROR;
1469                         else if (c == Token.OPEN_PARENS)
1470                                 current_token = Token.DEFAULT_OPEN_PARENS;
1471                         else if (c == Token.COLON)
1472                                 current_token = Token.DEFAULT_COLON;
1473                         else
1474                                 PopPosition();
1475 #endif
1476                         return current_token;
1477                 }
1478
1479                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1480
1481                 void get_cmd_arg (out string cmd, out string arg)
1482                 {
1483                         int c;
1484
1485                         tokens_seen = false;
1486                         arg = "";
1487                         static_cmd_arg.Length = 0;
1488
1489                         // skip over white space
1490                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1491                                 ;
1492
1493                         while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
1494                                 if (is_identifier_part_character ((char) c)){
1495                                         static_cmd_arg.Append ((char) c);
1496                                         c = getChar ();
1497                                 } else {
1498                                         putback (c);
1499                                         break;
1500                                 }
1501                         }
1502
1503                         cmd = static_cmd_arg.ToString ();
1504
1505                         if (c == '\n' || c == '\r'){
1506                                 return;
1507                         }
1508
1509                         // skip over white space
1510                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1511                                 ;
1512
1513                         if (c == '\n'){
1514                                 return;
1515                         } else if (c == '\r'){
1516                                 return;
1517                         } else if (c == -1){
1518                                 arg = "";
1519                                 return;
1520                         }
1521
1522                         static_cmd_arg.Length = 0;
1523                         static_cmd_arg.Append ((char) c);
1524
1525                         while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
1526                                 static_cmd_arg.Append ((char) c);
1527                         }
1528
1529                         arg = static_cmd_arg.ToString ();
1530                 }
1531
1532                 //
1533                 // Handles the #line directive
1534                 //
1535                 bool PreProcessLine (string arg)
1536                 {
1537                         if (arg.Length == 0)
1538                                 return false;
1539
1540                         if (arg == "default"){
1541                                 ref_line = line;
1542                                 ref_name = file_name;
1543                                 Location.Push (ref_name);
1544                                 return true;
1545                         } else if (arg == "hidden"){
1546                                 //
1547                                 // We ignore #line hidden
1548                                 //
1549                                 return true;
1550                         }
1551
1552                         try {
1553                                 int pos;
1554
1555                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1556                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1557                                         pos++;
1558
1559                                         char [] quotes = { '\"' };
1560
1561                                         string name = arg.Substring (pos). Trim (quotes);
1562                                         ref_name = Location.LookupFile (name);
1563                                         file_name.HasLineDirective = true;
1564                                         ref_name.HasLineDirective = true;
1565                                         Location.Push (ref_name);
1566                                 } else {
1567                                         ref_line = System.Int32.Parse (arg);
1568                                 }
1569                         } catch {
1570                                 return false;
1571                         }
1572
1573                         return true;
1574                 }
1575
1576                 //
1577                 // Handles #define and #undef
1578                 //
1579                 void PreProcessDefinition (bool is_define, string arg, bool caller_is_taking)
1580                 {
1581                         if (arg.Length == 0 || arg == "true" || arg == "false"){
1582                                 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
1583                                 return;
1584                         }
1585
1586                         char[] whitespace = { ' ', '\t' };
1587                         if (arg.IndexOfAny (whitespace) != -1){
1588                                 Error_EndLineExpected ();
1589                                 return;
1590                         }
1591
1592                         if (!is_identifier_start_character (arg [0]))
1593                                 Report.Error (1001, Location, "Identifier expected: " + arg);
1594
1595                         foreach (char c in arg.Substring (1)){
1596                                 if (!is_identifier_part_character (c)){
1597                                         Report.Error (1001, Location, "Identifier expected: " + arg);
1598                                         return;
1599                                 }
1600                         }
1601
1602                         if (!caller_is_taking)
1603                                 return;
1604
1605                         if (is_define){
1606                                 if (defines == null)
1607                                         defines = new Hashtable ();
1608                                 define (arg);
1609                         } else {
1610                                 if (defines == null)
1611                                         return;
1612                                 if (defines.Contains (arg))
1613                                         defines.Remove (arg);
1614                         }
1615                 }
1616
1617                 /// <summary>
1618                 /// Handles #pragma directive
1619                 /// </summary>
1620                 void PreProcessPragma (string arg)
1621                 {
1622                         const string warning = "warning";
1623                         const string w_disable = "warning disable";
1624                         const string w_restore = "warning restore";
1625
1626                         if (arg == w_disable) {
1627                                 Report.RegisterWarningRegion (Location).WarningDisable (line);
1628                                 return;
1629                         }
1630
1631                         if (arg == w_restore) {
1632                                 Report.RegisterWarningRegion (Location).WarningEnable (line);
1633                                 return;
1634                         }
1635
1636                         if (arg.StartsWith (w_disable)) {
1637                                 int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
1638                                 foreach (int code in codes) {
1639                                         if (code != 0)
1640                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1641                                 }
1642                                 return;
1643                         }
1644
1645                         if (arg.StartsWith (w_restore)) {
1646                                 int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
1647                                 Hashtable w_table = Report.warning_ignore_table;
1648                                 foreach (int code in codes) {
1649                                         if (w_table != null && w_table.Contains (code))
1650                                                 Report.Warning (1635, 1, Location, String.Format ("Cannot restore warning `CS{0:0000}' because it was disabled globally", code));
1651                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1652                                 }
1653                                 return;
1654                         }
1655
1656                         if (arg.StartsWith (warning)) {
1657                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
1658                                 return;
1659                         }
1660
1661                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
1662                 }
1663
1664                 int[] ParseNumbers (string text)
1665                 {
1666                         string[] string_array = text.Split (',');
1667                         int[] values = new int [string_array.Length];
1668                         int index = 0;
1669                         foreach (string string_code in string_array) {
1670                                 try {
1671                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1672                                 }
1673                                 catch (FormatException) {
1674                                         Report.Warning (1692, 1, Location, "Invalid number");
1675                                 }
1676                         }
1677                         return values;
1678                 }
1679
1680                 bool eval_val (string s)
1681                 {
1682                         if (s == "true")
1683                                 return true;
1684                         if (s == "false")
1685                                 return false;
1686
1687                         if (defines == null)
1688                                 return false;
1689                         if (defines.Contains (s))
1690                                 return true;
1691
1692                         return false;
1693                 }
1694
1695                 bool pp_primary (ref string s)
1696                 {
1697                         s = s.Trim ();
1698                         int len = s.Length;
1699
1700                         if (len > 0){
1701                                 char c = s [0];
1702
1703                                 if (c == '('){
1704                                         s = s.Substring (1);
1705                                         bool val = pp_expr (ref s, false);
1706                                         if (s.Length > 0 && s [0] == ')'){
1707                                                 s = s.Substring (1);
1708                                                 return val;
1709                                         }
1710                                         Error_InvalidDirective ();
1711                                         return false;
1712                                 }
1713
1714                                 if (is_identifier_start_character (c)){
1715                                         int j = 1;
1716
1717                                         while (j < len){
1718                                                 c = s [j];
1719
1720                                                 if (is_identifier_part_character (c)){
1721                                                         j++;
1722                                                         continue;
1723                                                 }
1724                                                 bool v = eval_val (s.Substring (0, j));
1725                                                 s = s.Substring (j);
1726                                                 return v;
1727                                         }
1728                                         bool vv = eval_val (s);
1729                                         s = "";
1730                                         return vv;
1731                                 }
1732                         }
1733                         Error_InvalidDirective ();
1734                         return false;
1735                 }
1736
1737                 bool pp_unary (ref string s)
1738                 {
1739                         s = s.Trim ();
1740                         int len = s.Length;
1741
1742                         if (len > 0){
1743                                 if (s [0] == '!'){
1744                                         if (len > 1 && s [1] == '='){
1745                                                 Error_InvalidDirective ();
1746                                                 return false;
1747                                         }
1748                                         s = s.Substring (1);
1749                                         return ! pp_primary (ref s);
1750                                 } else
1751                                         return pp_primary (ref s);
1752                         } else {
1753                                 Error_InvalidDirective ();
1754                                 return false;
1755                         }
1756                 }
1757
1758                 bool pp_eq (ref string s)
1759                 {
1760                         bool va = pp_unary (ref s);
1761
1762                         s = s.Trim ();
1763                         int len = s.Length;
1764                         if (len > 0){
1765                                 if (s [0] == '='){
1766                                         if (len > 2 && s [1] == '='){
1767                                                 s = s.Substring (2);
1768                                                 return va == pp_unary (ref s);
1769                                         } else {
1770                                                 Error_InvalidDirective ();
1771                                                 return false;
1772                                         }
1773                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
1774                                         s = s.Substring (2);
1775
1776                                         return va != pp_unary (ref s);
1777
1778                                 }
1779                         }
1780
1781                         return va;
1782
1783                 }
1784
1785                 bool pp_and (ref string s)
1786                 {
1787                         bool va = pp_eq (ref s);
1788
1789                         s = s.Trim ();
1790                         int len = s.Length;
1791                         if (len > 0){
1792                                 if (s [0] == '&'){
1793                                         if (len > 2 && s [1] == '&'){
1794                                                 s = s.Substring (2);
1795                                                 return (va & pp_and (ref s));
1796                                         } else {
1797                                                 Error_InvalidDirective ();
1798                                                 return false;
1799                                         }
1800                                 }
1801                         }
1802                         return va;
1803                 }
1804
1805                 //
1806                 // Evaluates an expression for `#if' or `#elif'
1807                 //
1808                 bool pp_expr (ref string s, bool isTerm)
1809                 {
1810                         bool va = pp_and (ref s);
1811                         s = s.Trim ();
1812                         int len = s.Length;
1813                         if (len > 0){
1814                                 char c = s [0];
1815
1816                                 if (c == '|'){
1817                                         if (len > 2 && s [1] == '|'){
1818                                                 s = s.Substring (2);
1819                                                 return va | pp_expr (ref s, isTerm);
1820                                         } else {
1821                                                 Error_InvalidDirective ();
1822                                                 return false;
1823                                         }
1824                                 }
1825                                 if (isTerm) {
1826                                         Error_EndLineExpected ();
1827                                         return false;
1828                                 }
1829                         }
1830
1831                         return va;
1832                 }
1833
1834                 bool eval (string s)
1835                 {
1836                         bool v = pp_expr (ref s, true);
1837                         s = s.Trim ();
1838                         if (s.Length != 0){
1839                                 return false;
1840                         }
1841
1842                         return v;
1843                 }
1844
1845                 void Error_NumericConstantTooLong ()
1846                 {
1847                         Report.Error (1021, Location, "Numeric constant too long");
1848                 }
1849
1850                 void Error_InvalidDirective ()
1851                 {
1852                         Report.Error (1517, Location, "Invalid preprocessor directive");
1853                 }
1854
1855                 void Error_UnexpectedDirective (string extra)
1856                 {
1857                         Report.Error (
1858                                 1028, Location,
1859                                 "Unexpected processor directive (" + extra + ")");
1860                 }
1861
1862                 void Error_TokensSeen ()
1863                 {
1864                         Report.Error (1032, Location,
1865                                 "Cannot define or undefine preprocessor symbols after first token in file");
1866                 }
1867
1868                 void Eror_WrongPreprocessorLocation ()
1869                 {
1870                         Report.Error (1040, Location,
1871                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
1872                 }
1873
1874                 void Error_EndLineExpected ()
1875                 {
1876                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
1877                 }
1878
1879                 //
1880                 // if true, then the code continues processing the code
1881                 // if false, the code stays in a loop until another directive is
1882                 // reached.
1883                 // When caller_is_taking is false we ignore all directives except the ones
1884                 // which can help us to identify where the #if block ends
1885                 bool handle_preprocessing_directive (bool caller_is_taking)
1886                 {
1887                         string cmd, arg;
1888                         bool region_directive = false;
1889
1890                         get_cmd_arg (out cmd, out arg);
1891
1892                         // Eat any trailing whitespaces and single-line comments
1893                         if (arg.IndexOf ("//") != -1)
1894                                 arg = arg.Substring (0, arg.IndexOf ("//"));
1895                         arg = arg.Trim (' ', '\t');
1896
1897                         //
1898                         // The first group of pre-processing instructions is always processed
1899                         //
1900                         switch (cmd){
1901                         case "region":
1902                                 region_directive = true;
1903                                 arg = "true";
1904                                 goto case "if";
1905
1906                         case "endregion":
1907                                 if (ifstack == null || ifstack.Count == 0){
1908                                         Error_UnexpectedDirective ("no #region for this #endregion");
1909                                         return true;
1910                                 }
1911                                 int pop = (int) ifstack.Pop ();
1912
1913                                 if ((pop & REGION) == 0)
1914                                         Report.Error (1027, Location, "Expected `#endif' directive");
1915
1916                                 return caller_is_taking;
1917
1918                         case "if":
1919                                 if (ifstack == null)
1920                                         ifstack = new Stack (2);
1921
1922                                 int flags = region_directive ? REGION : 0;
1923                                 if (ifstack.Count == 0){
1924                                         flags |= PARENT_TAKING;
1925                                 } else {
1926                                         int state = (int) ifstack.Peek ();
1927                                         if ((state & TAKING) != 0) {
1928                                                 flags |= PARENT_TAKING;
1929                                         }
1930                                 }
1931
1932                                 if (caller_is_taking && eval (arg)) {
1933                                         ifstack.Push (flags | TAKING);
1934                                         return true;
1935                                 }
1936                                 ifstack.Push (flags);
1937                                 return false;
1938
1939                         case "endif":
1940                                 if (ifstack == null || ifstack.Count == 0){
1941                                         Error_UnexpectedDirective ("no #if for this #endif");
1942                                         return true;
1943                                 } else {
1944                                         pop = (int) ifstack.Pop ();
1945
1946                                         if ((pop & REGION) != 0)
1947                                                 Report.Error (1038, Location, "#endregion directive expected");
1948
1949                                         if (arg.Length != 0) {
1950                                                 Error_EndLineExpected ();
1951                                         }
1952
1953                                         if (ifstack.Count == 0)
1954                                                 return true;
1955
1956                                         int state = (int) ifstack.Peek ();
1957                                         return (state & TAKING) != 0;
1958                                 }
1959
1960                         case "elif":
1961                                 if (ifstack == null || ifstack.Count == 0){
1962                                         Error_UnexpectedDirective ("no #if for this #elif");
1963                                         return true;
1964                                 } else {
1965                                         int state = (int) ifstack.Pop ();
1966
1967                                         if ((state & REGION) != 0) {
1968                                                 Report.Error (1038, Location, "#endregion directive expected");
1969                                                 return true;
1970                                         }
1971
1972                                         if ((state & ELSE_SEEN) != 0){
1973                                                 Error_UnexpectedDirective ("#elif not valid after #else");
1974                                                 return true;
1975                                         }
1976
1977                                         if ((state & TAKING) != 0) {
1978                                                 ifstack.Push (0);
1979                                                 return false;
1980                                         }
1981
1982                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
1983                                                 ifstack.Push (state | TAKING);
1984                                                 return true;
1985                                         }
1986
1987                                         ifstack.Push (state);
1988                                         return false;
1989                                 }
1990
1991                         case "else":
1992                                 if (ifstack == null || ifstack.Count == 0){
1993                                         Error_UnexpectedDirective ("no #if for this #else");
1994                                         return true;
1995                                 } else {
1996                                         int state = (int) ifstack.Peek ();
1997
1998                                         if ((state & REGION) != 0) {
1999                                                 Report.Error (1038, Location, "#endregion directive expected");
2000                                                 return true;
2001                                         }
2002
2003                                         if ((state & ELSE_SEEN) != 0){
2004                                                 Error_UnexpectedDirective ("#else within #else");
2005                                                 return true;
2006                                         }
2007
2008                                         ifstack.Pop ();
2009
2010                                         if (arg.Length != 0) {
2011                                                 Error_EndLineExpected ();
2012                                                 return true;
2013                                         }
2014
2015                                         bool ret = false;
2016                                         if ((state & PARENT_TAKING) != 0) {
2017                                                 ret = (state & TAKING) == 0;
2018
2019                                                 if (ret)
2020                                                         state |= TAKING;
2021                                                 else
2022                                                         state &= ~TAKING;
2023                                         }
2024
2025                                         ifstack.Push (state | ELSE_SEEN);
2026
2027                                         return ret;
2028                                 }
2029                                 case "define":
2030                                         if (any_token_seen){
2031                                                 Error_TokensSeen ();
2032                                                 return caller_is_taking;
2033                                         }
2034                                         PreProcessDefinition (true, arg, caller_is_taking);
2035                                         return caller_is_taking;
2036
2037                                 case "undef":
2038                                         if (any_token_seen){
2039                                                 Error_TokensSeen ();
2040                                                 return caller_is_taking;
2041                                         }
2042                                         PreProcessDefinition (false, arg, caller_is_taking);
2043                                         return caller_is_taking;
2044                         }
2045
2046                         //
2047                         // These are only processed if we are in a `taking' block
2048                         //
2049                         if (!caller_is_taking)
2050                                 return false;
2051
2052                         switch (cmd){
2053                         case "error":
2054                                 Report.Error (1029, Location, "#error: '" + arg + "'");
2055                                 return true;
2056
2057                         case "warning":
2058                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2059                                 return true;
2060
2061                         case "pragma":
2062                                 if (RootContext.Version == LanguageVersion.ISO_1) {
2063                                         Report.FeatureIsNotStandardized (Location, "#pragma");
2064                                         return true;
2065                                 }
2066
2067                                 PreProcessPragma (arg);
2068                                 return true;
2069
2070                         case "line":
2071                                 if (!PreProcessLine (arg))
2072                                         Report.Error (
2073                                                 1576, Location,
2074                                                 "The line number specified for #line directive is missing or invalid");
2075                                 return caller_is_taking;
2076                         }
2077
2078                         Report.Error (1024, Location, "Wrong preprocessor directive");
2079                         return true;
2080
2081                 }
2082
2083                 private int consume_string (bool quoted)
2084                 {
2085                         int c;
2086                         string_builder.Length = 0;
2087
2088                         while ((c = getChar ()) != -1){
2089                                 if (c == '"'){
2090                                         if (quoted && peekChar () == '"'){
2091                                                 string_builder.Append ((char) c);
2092                                                 getChar ();
2093                                                 continue;
2094                                         } else {
2095                                                 val = string_builder.ToString ();
2096                                                 return Token.LITERAL_STRING;
2097                                         }
2098                                 }
2099
2100                                 if (c == '\n'){
2101                                         if (!quoted)
2102                                                 Report.Error (1010, Location, "Newline in constant");
2103                                 }
2104
2105                                 if (!quoted){
2106                                         c = escape (c);
2107                                         if (c == -1)
2108                                                 return Token.ERROR;
2109                                 }
2110                                 string_builder.Append ((char) c);
2111                         }
2112
2113                         Report.Error (1039, Location, "Unterminated string literal");
2114                         return Token.EOF;
2115                 }
2116
2117                 private int consume_identifier (int s)
2118                 {
2119                         int res = consume_identifier (s, false);
2120
2121                         if (doc_state == XmlCommentState.Allowed)
2122                                 doc_state = XmlCommentState.NotAllowed;
2123                         switch (res) {
2124                         case Token.USING:
2125                         case Token.NAMESPACE:
2126                                 check_incorrect_doc_comment ();
2127                                 break;
2128                         }
2129
2130                         if (res == Token.PARTIAL) {
2131                                 // Save current position and parse next token.
2132                                 PushPosition ();
2133
2134                                 int next_token = token ();
2135                                 bool ok = (next_token == Token.CLASS) ||
2136                                         (next_token == Token.STRUCT) ||
2137                                         (next_token == Token.INTERFACE);
2138
2139                                 PopPosition ();
2140
2141                                 if (ok)
2142                                         return res;
2143
2144                                 if (next_token < Token.LAST_KEYWORD)
2145                                         Report.Error (267, Location, "The `partial' modifier can be used only immediately before keyword `class', `struct', or `interface'");
2146
2147                                 val = new LocatedToken (Location, "partial");
2148                                 return Token.IDENTIFIER;
2149                         }
2150
2151                         return res;
2152                 }
2153
2154                 private int consume_identifier (int s, bool quoted)
2155                 {
2156                         int pos = 1;
2157                         int c = -1;
2158
2159                         id_builder [0] = (char) s;
2160
2161                         current_location = new Location (ref_line, Col);
2162
2163                         while ((c = getChar ()) != -1) {
2164                         loop:
2165                                 if (is_identifier_part_character ((char) c)){
2166                                         if (pos == max_id_size){
2167                                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2168                                                 return Token.ERROR;
2169                                         }
2170
2171                                         id_builder [pos++] = (char) c;
2172 //                                      putback_char = -1;
2173                                 } else if (c == '\\') {
2174                                         c = escape (c);
2175                                         goto loop;
2176                                 } else {
2177 //                                      putback_char = c;
2178                                         putback (c);
2179                                         break;
2180                                 }
2181                         }
2182
2183                         //
2184                         // Optimization: avoids doing the keyword lookup
2185                         // on uppercase letters and _
2186                         //
2187                         if (!quoted && (s >= 'a' || s == '_')){
2188                                 int keyword = GetKeyword (id_builder, pos);
2189                                 if (keyword != -1) {
2190                                         val = Location;
2191                                         return keyword;
2192                                 }
2193                         }
2194
2195                         //
2196                         // Keep identifiers in an array of hashtables to avoid needless
2197                         // allocations
2198                         //
2199
2200                         if (identifiers [pos] != null) {
2201                                 val = identifiers [pos][id_builder];
2202                                 if (val != null) {
2203                                         val = new LocatedToken (Location, (string) val);
2204                                         if (quoted)
2205                                                 escapedIdentifiers.Add (val);
2206                                         return Token.IDENTIFIER;
2207                                 }
2208                         }
2209                         else
2210                                 identifiers [pos] = new CharArrayHashtable (pos);
2211
2212                         val = new String (id_builder, 0, pos);
2213                         if (RootContext.Version == LanguageVersion.ISO_1) {
2214                                 for (int i = 1; i < id_builder.Length; i += 3) {
2215                                         if (id_builder [i] == '_' && (id_builder [i - 1] == '_' || id_builder [i + 1] == '_')) {
2216                                                 Report.Error (1638, Location,
2217                                                         "`{0}': Any identifier with double underscores cannot be used when ISO language version mode is specified", val.ToString ());
2218                                                 break;
2219                                         }
2220                                 }
2221                         }
2222
2223                         char [] chars = new char [pos];
2224                         Array.Copy (id_builder, chars, pos);
2225
2226                         identifiers [pos] [chars] = val;
2227
2228                         val = new LocatedToken (Location, (string) val);
2229                         if (quoted)
2230                                 escapedIdentifiers.Add (val);
2231                         return Token.IDENTIFIER;
2232                 }
2233
2234                 public int xtoken ()
2235                 {
2236                         int t;
2237                         bool doread = false;
2238                         int c;
2239
2240                         // Whether we have seen comments on the current line
2241                         bool comments_seen = false;
2242
2243                         val = null;
2244                         for (;(c = getChar ()) != -1;) {
2245                                 if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == 0xa0)
2246                                         continue;
2247
2248                                 if (c == '\r') {
2249                                         if (peekChar () == '\n')
2250                                                 getChar ();
2251
2252                                         any_token_seen |= tokens_seen;
2253                                         tokens_seen = false;
2254                                         comments_seen = false;
2255                                         continue;
2256                                 }
2257
2258                                 // Handle double-slash comments.
2259                                 if (c == '/'){
2260                                         int d = peekChar ();
2261
2262                                         if (d == '/'){
2263                                                 getChar ();
2264                                                 if (RootContext.Documentation != null && peekChar () == '/') {
2265                                                         getChar ();
2266                                                         // Don't allow ////.
2267                                                         if ((d = peekChar ()) != '/') {
2268                                                                 update_comment_location ();
2269                                                                 if (doc_state == XmlCommentState.Allowed)
2270                                                                         handle_one_line_xml_comment ();
2271                                                                 else if (doc_state == XmlCommentState.NotAllowed)
2272                                                                         warn_incorrect_doc_comment ();
2273                                                         }
2274                                                 }
2275                                                 while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
2276                                                 if (d == '\n'){
2277                                                 }
2278                                                 any_token_seen |= tokens_seen;
2279                                                 tokens_seen = false;
2280                                                 comments_seen = false;
2281                                                 continue;
2282                                         } else if (d == '*'){
2283                                                 getChar ();
2284                                                 bool docAppend = false;
2285                                                 if (RootContext.Documentation != null && peekChar () == '*') {
2286                                                         getChar ();
2287                                                         update_comment_location ();
2288                                                         // But when it is /**/, just do nothing.
2289                                                         if (peekChar () == '/') {
2290                                                                 getChar ();
2291                                                                 continue;
2292                                                         }
2293                                                         if (doc_state == XmlCommentState.Allowed)
2294                                                                 docAppend = true;
2295                                                         else if (doc_state == XmlCommentState.NotAllowed)
2296                                                                 warn_incorrect_doc_comment ();
2297                                                 }
2298
2299                                                 int current_comment_start = 0;
2300                                                 if (docAppend) {
2301                                                         current_comment_start = xml_comment_buffer.Length;
2302                                                         xml_comment_buffer.Append (Environment.NewLine);
2303                                                 }
2304
2305                                                 Location start_location = Location;
2306
2307                                                 while ((d = getChar ()) != -1){
2308                                                         if (d == '*' && peekChar () == '/'){
2309                                                                 getChar ();
2310                                                                 comments_seen = true;
2311                                                                 break;
2312                                                         }
2313                                                         if (docAppend)
2314                                                                 xml_comment_buffer.Append ((char) d);
2315
2316                                                         if (d == '\n'){
2317                                                                 any_token_seen |= tokens_seen;
2318                                                                 tokens_seen = false;
2319                                                                 //
2320                                                                 // Reset 'comments_seen' just to be consistent.
2321                                                                 // It doesn't matter either way, here.
2322                                                                 //
2323                                                                 comments_seen = false;
2324                                                         }
2325                                                 }
2326                                                 if (!comments_seen)
2327                                                         Report.Error (1035, start_location, "End-of-file found, '*/' expected");
2328
2329                                                 if (docAppend)
2330                                                         update_formatted_doc_comment (current_comment_start);
2331                                                 continue;
2332                                         }
2333                                         goto is_punct_label;
2334                                 }
2335
2336
2337                                 if (c == '\\' || is_identifier_start_character ((char)c)){
2338                                         tokens_seen = true;
2339                                         return consume_identifier (c);
2340                                 }
2341
2342                         is_punct_label:
2343                                 current_location = new Location (ref_line, Col);
2344                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
2345                                         tokens_seen = true;
2346                                         if (doread){
2347                                                 getChar ();
2348                                         }
2349                                         return t;
2350                                 }
2351
2352                                 // white space
2353                                 if (c == '\n'){
2354                                         any_token_seen |= tokens_seen;
2355                                         tokens_seen = false;
2356                                         comments_seen = false;
2357                                         continue;
2358                                 }
2359
2360                                 if (c >= '0' && c <= '9'){
2361                                         tokens_seen = true;
2362                                         return is_number (c);
2363                                 }
2364
2365                                 if (c == '.'){
2366                                         tokens_seen = true;
2367                                         int peek = peekChar ();
2368                                         if (peek >= '0' && peek <= '9')
2369                                                 return is_number (c);
2370                                         return Token.DOT;
2371                                 }
2372
2373                                 if (c == '#') {
2374                                         if (tokens_seen || comments_seen) {
2375                                                 Eror_WrongPreprocessorLocation ();
2376                                                 return Token.ERROR;
2377                                         }
2378
2379                                         if (handle_preprocessing_directive (true))
2380                                                 continue;
2381
2382                                         bool directive_expected = false;
2383                                         while ((c = getChar ()) != -1) {
2384                                                 if (col == 1) {
2385                                                         directive_expected = true;
2386                                                 } else if (!directive_expected) {
2387                                                         // TODO: Implement comment support for disabled code and uncomment this code
2388 //                                                      if (c == '#') {
2389 //                                                              Eror_WrongPreprocessorLocation ();
2390 //                                                              return Token.ERROR;
2391 //                                                      }
2392                                                         continue;
2393                                                 }
2394
2395                                                 if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
2396                                                         continue;
2397
2398                                                 if (c == '#') {
2399                                                         if (handle_preprocessing_directive (false))
2400                                                                 break;
2401                                                 }
2402                                                 directive_expected = false;
2403                                         }
2404
2405                                         if (c != -1) {
2406                                                 tokens_seen = false;
2407                                                 continue;
2408                                         }
2409
2410                                         return Token.EOF;
2411                                 }
2412
2413                                 if (c == '"')
2414                                         return consume_string (false);
2415
2416                                 if (c == '\''){
2417                                         c = getChar ();
2418                                         tokens_seen = true;
2419                                         if (c == '\''){
2420                                                 error_details = "Empty character literal";
2421                                                 Report.Error (1011, Location, error_details);
2422                                                 return Token.ERROR;
2423                                         }
2424                                         if (c == '\r' || c == '\n') {
2425                                                 Report.Error (1010, Location, "Newline in constant");
2426                                                 return Token.ERROR;
2427                                         }
2428                                         c = escape (c);
2429                                         if (c == -1)
2430                                                 return Token.ERROR;
2431                                         val = new System.Char ();
2432                                         val = (char) c;
2433                                         c = getChar ();
2434
2435                                         if (c != '\''){
2436                                                 error_details = "Too many characters in character literal";
2437                                                 Report.Error (1012, Location, error_details);
2438
2439                                                 // Try to recover, read until newline or next "'"
2440                                                 while ((c = getChar ()) != -1){
2441                                                         if (c == '\n'){
2442                                                                 break;
2443                                                         }
2444                                                         else if (c == '\'')
2445                                                                 break;
2446                                                 }
2447                                                 return Token.ERROR;
2448                                         }
2449                                         return Token.LITERAL_CHARACTER;
2450                                 }
2451
2452                                 if (c == '@') {
2453                                         c = getChar ();
2454                                         if (c == '"') {
2455                                                 tokens_seen = true;
2456                                                 return consume_string (true);
2457                                         } else if (is_identifier_start_character ((char) c)){
2458                                                 return consume_identifier (c, true);
2459                                         } else {
2460                                                 Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
2461                                         }
2462                                 }
2463
2464                                 error_details = ((char)c).ToString ();
2465
2466                                 return Token.ERROR;
2467                         }
2468
2469                         return Token.EOF;
2470                 }
2471
2472                 //
2473                 // Handles one line xml comment
2474                 //
2475                 private void handle_one_line_xml_comment ()
2476                 {
2477                         int c;
2478                         while ((c = peekChar ()) == ' ')
2479                                 getChar (); // skip heading whitespaces.
2480                         while ((c = peekChar ()) != -1 && c != '\n' && c != '\r') {
2481                                 xml_comment_buffer.Append ((char) getChar ());
2482                         }
2483                         if (c == '\r' || c == '\n')
2484                                 xml_comment_buffer.Append (Environment.NewLine);
2485                 }
2486
2487                 //
2488                 // Remove heading "*" in Javadoc-like xml documentation.
2489                 //
2490                 private void update_formatted_doc_comment (int current_comment_start)
2491                 {
2492                         int length = xml_comment_buffer.Length - current_comment_start;
2493                         string [] lines = xml_comment_buffer.ToString (
2494                                 current_comment_start,
2495                                 length).Replace ("\r", "").Split ('\n');
2496
2497                         // The first line starts with /**, thus it is not target
2498                         // for the format check.
2499                         for (int i = 1; i < lines.Length; i++) {
2500                                 string s = lines [i];
2501                                 int idx = s.IndexOf ('*');
2502                                 string head = null;
2503                                 if (idx < 0) {
2504                                         if (i < lines.Length - 1)
2505                                                 return;
2506                                         head = s;
2507                                 } else
2508                                         head = s.Substring (0, idx);
2509                                 foreach (char c in head)
2510                                         if (c != ' ')
2511                                                 return;
2512                                 lines [i] = s.Substring (idx + 1);
2513                         }
2514                         xml_comment_buffer.Remove (current_comment_start, length);
2515                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
2516                 }
2517
2518                 //
2519                 // Updates current comment location.
2520                 //
2521                 private void update_comment_location ()
2522                 {
2523                         if (current_comment_location.IsNull) {
2524                                 // "-2" is for heading "//" or "/*"
2525                                 current_comment_location =
2526                                         new Location (ref_line, col - 2);
2527                         }
2528                 }
2529
2530                 //
2531                 // Checks if there was incorrect doc comments and raise
2532                 // warnings.
2533                 //
2534                 public void check_incorrect_doc_comment ()
2535                 {
2536                         if (xml_comment_buffer.Length > 0)
2537                                 warn_incorrect_doc_comment ();
2538                 }
2539
2540                 //
2541                 // Raises a warning when tokenizer found incorrect doccomment
2542                 // markup.
2543                 //
2544                 private void warn_incorrect_doc_comment ()
2545                 {
2546                         if (doc_state != XmlCommentState.Error) {
2547                                 doc_state = XmlCommentState.Error;
2548                                 // in csc, it is 'XML comment is not placed on
2549                                 // a valid language element'. But that does not
2550                                 // make sense.
2551                                 Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
2552                         }
2553                 }
2554
2555                 //
2556                 // Consumes the saved xml comment lines (if any)
2557                 // as for current target member or type.
2558                 //
2559                 public string consume_doc_comment ()
2560                 {
2561                         if (xml_comment_buffer.Length > 0) {
2562                                 string ret = xml_comment_buffer.ToString ();
2563                                 reset_doc_comment ();
2564                                 return ret;
2565                         }
2566                         return null;
2567                 }
2568
2569                 void reset_doc_comment ()
2570                 {
2571                         xml_comment_buffer.Length = 0;
2572                         current_comment_location = Location.Null;
2573                 }
2574
2575                 public void cleanup ()
2576                 {
2577                         if (ifstack != null && ifstack.Count >= 1) {
2578                                 current_location = new Location (ref_line, Col);
2579                                 int state = (int) ifstack.Pop ();
2580                                 if ((state & REGION) != 0)
2581                                         Report.Error (1038, Location, "#endregion directive expected");
2582                                 else
2583                                         Report.Error (1027, Location, "Expected `#endif' directive");
2584                         }
2585                 }
2586         }
2587
2588         //
2589         // Indicates whether it accepts XML documentation or not.
2590         //
2591         public enum XmlCommentState {
2592                 // comment is allowed in this state.
2593                 Allowed,
2594                 // comment is not allowed in this state.
2595                 NotAllowed,
2596                 // once comments appeared when it is NotAllowed, then the
2597                 // state is changed to it, until the state is changed to
2598                 // .Allowed.
2599                 Error
2600         }
2601 }
2602