mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Licensed under the terms of the GNU GPL
   9 //
  10 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // (C) 2004 Novell, Inc
  12 //
  13
  14 /*
  15  * TODO:
  16  *   Make sure we accept the proper Unicode ranges, per the spec.
  17  *   Report error 1032
  18 */
  19
  20 using System;
  21 using System.Text;
  22 using System.Collections;
  23 using System.IO;
  24 using System.Globalization;
  25 using System.Reflection;
  26
  27 namespace Mono.CSharp
  28 {
  29         /// <summary>
  30         ///    Tokenizer for C# source code.
  31         /// </summary>
  32
  33         public class Tokenizer : yyParser.yyInput
  34         {
  35                 SeekableStreamReader reader;
  36                 SourceFile ref_name;
  37                 SourceFile file_name;
  38                 int ref_line = 1;
  39                 int line = 1;
  40                 int col = 0;
  41                 int previous_col;
  42                 int current_token;
  43                 bool handle_get_set = false;
  44                 bool handle_remove_add = false;
  45                 bool handle_assembly = false;
  46                 bool handle_constraints = false;
  47                 bool handle_typeof = false;
  48                 Location current_location;
  49                 Location current_comment_location = Location.Null;
  50                 ArrayList escapedIdentifiers = new ArrayList ();
  51
  52                 //
  53                 // XML documentation buffer. The save point is used to divide
  54                 // comments on types and comments on members.
  55                 //
  56                 StringBuilder xml_comment_buffer;
  57
  58                 //
  59                 // See comment on XmlCommentState enumeration.
  60                 //
  61                 XmlCommentState xmlDocState = XmlCommentState.Allowed;
  62
  63                 //
  64                 // Whether tokens have been seen on this line
  65                 //
  66                 bool tokens_seen = false;
  67
  68                 //
  69                 // Whether a token has been seen on the file
  70                 // This is needed because `define' is not allowed to be used
  71                 // after a token has been seen.
  72                 //
  73                 bool any_token_seen = false;
  74
  75                 static Hashtable tokenValues;
  76
  77                 private static Hashtable TokenValueName
  78                 {
  79                         get {
  80                                 if (tokenValues == null)
  81                                         tokenValues = GetTokenValueNameHash ();
  82
  83                                 return tokenValues;
  84                         }
  85                 }
  86
  87                 private static Hashtable GetTokenValueNameHash ()
  88                 {
  89                         Type t = typeof (Token);
  90                         FieldInfo [] fields = t.GetFields ();
  91                         Hashtable hash = new Hashtable ();
  92                         foreach (FieldInfo field in fields) {
  93                                 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))
  94                                         hash.Add (field.GetValue (null), field.Name);
  95                         }
  96                         return hash;
  97                 }
  98
  99                 //
 100                 // Returns a verbose representation of the current location
 101                 //
 102                 public string location {
 103                         get {
 104                                 string det;
 105
 106                                 if (current_token == Token.ERROR)
 107                                         det = "detail: " + error_details;
 108                                 else
 109                                         det = "";
 110
 111                                 // return "Line:     "+line+" Col: "+col + "\n" +
 112                                 //       "VirtLine: "+ref_line +
 113                                 //       " Token: "+current_token + " " + det;
 114                                 string current_token_name = TokenValueName [current_token] as string;
 115                                 if (current_token_name == null)
 116                                         current_token_name = current_token.ToString ();
 117
 118                                 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,
 119                                                                                        ref_line,
 120                                                                                        col,
 121                                                                                        current_token_name,
 122                                                                                        det);
 123                         }
 124                 }
 125
 126                 public bool PropertyParsing {
 127                         get {
 128                                 return handle_get_set;
 129                         }
 130
 131                         set {
 132                                 handle_get_set = value;
 133                         }
 134                 }
 135
 136                 public bool AssemblyTargetParsing {
 137                         get {
 138                                 return handle_assembly;
 139                         }
 140
 141                         set {
 142                                 handle_assembly = value;
 143                         }
 144                 }
 145
 146                 public bool EventParsing {
 147                         get {
 148                                 return handle_remove_add;
 149                         }
 150
 151                         set {
 152                                 handle_remove_add = value;
 153                         }
 154                 }
 155
 156                 public bool ConstraintsParsing {
 157                         get {
 158                                 return handle_constraints;
 159                         }
 160
 161                         set {
 162                                 handle_constraints = value;
 163                         }
 164                 }
 165
 166                 public bool TypeOfParsing {
 167                         get {
 168                                 return handle_typeof;
 169                         }
 170
 171                         set {
 172                                 handle_typeof = value;
 173                         }
 174                 }
 175
 176                 public XmlCommentState doc_state {
 177                         get { return xmlDocState; }
 178                         set {
 179                                 if (value == XmlCommentState.Allowed) {
 180                                         check_incorrect_doc_comment ();
 181                                         reset_doc_comment ();
 182                                 }
 183                                 xmlDocState = value;
 184                         }
 185                 }
 186
 187                 public bool IsEscapedIdentifier (Location loc)
 188                 {
 189                         foreach (LocatedToken lt in escapedIdentifiers)
 190                                 if (lt.Location.Equals (loc))
 191                                         return true;
 192                         return false;
 193                 }
 194
 195                 //
 196                 // Class variables
 197                 //
 198                 static CharArrayHashtable[] keywords;
 199                 static Hashtable keywordStrings = new Hashtable ();
 200                 static NumberStyles styles;
 201                 static NumberFormatInfo csharp_format_info;
 202
 203                 //
 204                 // Values for the associated token returned
 205                 //
 206                 int putback_char;
 207                 Object val;
 208
 209                 //
 210                 // Pre-processor
 211                 //
 212                 Hashtable defines;
 213
 214                 const int TAKING        = 1;
 215                 const int TAKEN_BEFORE  = 2;
 216                 const int ELSE_SEEN     = 4;
 217                 const int PARENT_TAKING = 8;
 218                 const int REGION        = 16;
 219
 220                 //
 221                 // pre-processor if stack state:
 222                 //
 223                 Stack ifstack;
 224
 225                 static System.Text.StringBuilder string_builder;
 226
 227                 const int max_id_size = 512;
 228                 static char [] id_builder = new char [max_id_size];
 229
 230                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 231
 232                 const int max_number_size = 512;
 233                 static char [] number_builder = new char [max_number_size];
 234                 static int number_pos;
 235
 236                 //
 237                 // Details about the error encoutered by the tokenizer
 238                 //
 239                 string error_details;
 240
 241                 public string error {
 242                         get {
 243                                 return error_details;
 244                         }
 245                 }
 246
 247                 public int Line {
 248                         get {
 249                                 return ref_line;
 250                         }
 251                 }
 252
 253                 public int Col {
 254                         get {
 255                                 return col;
 256                         }
 257                 }
 258
 259                 //
 260                 // This is used when the tokenizer needs to save
 261                 // the current position as it needs to do some parsing
 262                 // on its own to deamiguate a token in behalf of the
 263                 // parser.
 264                 //
 265                 Stack position_stack = new Stack (2);
 266                 class Position {
 267                         public int position;
 268                         public int ref_line;
 269                         public int col;
 270                         public int putback_char;
 271                         public int previous_col;
 272 #if GMCS_SOURCES
 273                         public int parsing_generic_less_than;
 274 #endif
 275                         public Position (Tokenizer t)
 276                         {
 277                                 position = t.reader.Position;
 278                                 ref_line = t.ref_line;
 279                                 col = t.col;
 280                                 putback_char = t.putback_char;
 281                                 previous_col = t.previous_col;
 282 #if GMCS_SOURCES
 283                                 parsing_generic_less_than = t.parsing_generic_less_than;
 284 #endif
 285                         }
 286                 }
 287
 288                 public void PushPosition ()
 289                 {
 290                         position_stack.Push (new Position (this));
 291                 }
 292
 293                 public void PopPosition ()
 294                 {
 295                         Position p = (Position) position_stack.Pop ();
 296
 297                         reader.Position = p.position;
 298                         ref_line = p.ref_line;
 299                         col = p.col;
 300                         putback_char = p.putback_char;
 301                         previous_col = p.previous_col;
 302
 303                 }
 304
 305                 // Do not reset the position, ignore it.
 306                 public void DiscardPosition ()
 307                 {
 308                         position_stack.Pop ();
 309                 }
 310
 311                 static void AddKeyword (string kw, int token) {
 312                         keywordStrings.Add (kw, kw);
 313                         if (keywords [kw.Length] == null) {
 314                                 keywords [kw.Length] = new CharArrayHashtable (kw.Length);
 315                         }
 316                         keywords [kw.Length] [kw.ToCharArray ()] = token;
 317                 }
 318
 319                 static void InitTokens ()
 320                 {
 321                         keywords = new CharArrayHashtable [64];
 322
 323                         AddKeyword ("__arglist", Token.ARGLIST);
 324                         AddKeyword ("abstract", Token.ABSTRACT);
 325                         AddKeyword ("as", Token.AS);
 326                         AddKeyword ("add", Token.ADD);
 327                         AddKeyword ("assembly", Token.ASSEMBLY);
 328                         AddKeyword ("base", Token.BASE);
 329                         AddKeyword ("bool", Token.BOOL);
 330                         AddKeyword ("break", Token.BREAK);
 331                         AddKeyword ("byte", Token.BYTE);
 332                         AddKeyword ("case", Token.CASE);
 333                         AddKeyword ("catch", Token.CATCH);
 334                         AddKeyword ("char", Token.CHAR);
 335                         AddKeyword ("checked", Token.CHECKED);
 336                         AddKeyword ("class", Token.CLASS);
 337                         AddKeyword ("const", Token.CONST);
 338                         AddKeyword ("continue", Token.CONTINUE);
 339                         AddKeyword ("decimal", Token.DECIMAL);
 340                         AddKeyword ("default", Token.DEFAULT);
 341                         AddKeyword ("delegate", Token.DELEGATE);
 342                         AddKeyword ("do", Token.DO);
 343                         AddKeyword ("double", Token.DOUBLE);
 344                         AddKeyword ("else", Token.ELSE);
 345                         AddKeyword ("enum", Token.ENUM);
 346                         AddKeyword ("event", Token.EVENT);
 347                         AddKeyword ("explicit", Token.EXPLICIT);
 348                         AddKeyword ("extern", Token.EXTERN);
 349                         AddKeyword ("false", Token.FALSE);
 350                         AddKeyword ("finally", Token.FINALLY);
 351                         AddKeyword ("fixed", Token.FIXED);
 352                         AddKeyword ("float", Token.FLOAT);
 353                         AddKeyword ("for", Token.FOR);
 354                         AddKeyword ("foreach", Token.FOREACH);
 355                         AddKeyword ("goto", Token.GOTO);
 356                         AddKeyword ("get", Token.GET);
 357                         AddKeyword ("if", Token.IF);
 358                         AddKeyword ("implicit", Token.IMPLICIT);
 359                         AddKeyword ("in", Token.IN);
 360                         AddKeyword ("int", Token.INT);
 361                         AddKeyword ("interface", Token.INTERFACE);
 362                         AddKeyword ("internal", Token.INTERNAL);
 363                         AddKeyword ("is", Token.IS);
 364                         AddKeyword ("lock", Token.LOCK);
 365                         AddKeyword ("long", Token.LONG);
 366                         AddKeyword ("namespace", Token.NAMESPACE);
 367                         AddKeyword ("new", Token.NEW);
 368                         AddKeyword ("null", Token.NULL);
 369                         AddKeyword ("object", Token.OBJECT);
 370                         AddKeyword ("operator", Token.OPERATOR);
 371                         AddKeyword ("out", Token.OUT);
 372                         AddKeyword ("override", Token.OVERRIDE);
 373                         AddKeyword ("params", Token.PARAMS);
 374                         AddKeyword ("private", Token.PRIVATE);
 375                         AddKeyword ("protected", Token.PROTECTED);
 376                         AddKeyword ("public", Token.PUBLIC);
 377                         AddKeyword ("readonly", Token.READONLY);
 378                         AddKeyword ("ref", Token.REF);
 379                         AddKeyword ("remove", Token.REMOVE);
 380                         AddKeyword ("return", Token.RETURN);
 381                         AddKeyword ("sbyte", Token.SBYTE);
 382                         AddKeyword ("sealed", Token.SEALED);
 383                         AddKeyword ("set", Token.SET);
 384                         AddKeyword ("short", Token.SHORT);
 385                         AddKeyword ("sizeof", Token.SIZEOF);
 386                         AddKeyword ("stackalloc", Token.STACKALLOC);
 387                         AddKeyword ("static", Token.STATIC);
 388                         AddKeyword ("string", Token.STRING);
 389                         AddKeyword ("struct", Token.STRUCT);
 390                         AddKeyword ("switch", Token.SWITCH);
 391                         AddKeyword ("this", Token.THIS);
 392                         AddKeyword ("throw", Token.THROW);
 393                         AddKeyword ("true", Token.TRUE);
 394                         AddKeyword ("try", Token.TRY);
 395                         AddKeyword ("typeof", Token.TYPEOF);
 396                         AddKeyword ("uint", Token.UINT);
 397                         AddKeyword ("ulong", Token.ULONG);
 398                         AddKeyword ("unchecked", Token.UNCHECKED);
 399                         AddKeyword ("unsafe", Token.UNSAFE);
 400                         AddKeyword ("ushort", Token.USHORT);
 401                         AddKeyword ("using", Token.USING);
 402                         AddKeyword ("virtual", Token.VIRTUAL);
 403                         AddKeyword ("void", Token.VOID);
 404                         AddKeyword ("volatile", Token.VOLATILE);
 405                         AddKeyword ("while", Token.WHILE);
 406                         AddKeyword ("partial", Token.PARTIAL);
 407 #if GMCS_SOURCE
 408                         AddKeyword ("where", Token.WHERE);
 409 #endif
 410                 }
 411
 412                 //
 413                 // Class initializer
 414                 //
 415                 static Tokenizer ()
 416                 {
 417                         InitTokens ();
 418                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 419                         styles = NumberStyles.Float;
 420
 421                         string_builder = new System.Text.StringBuilder ();
 422                 }
 423
 424                 int GetKeyword (char[] id, int id_len)
 425                 {
 426                         /*
 427                          * Keywords are stored in an array of hashtables grouped by their
 428                          * length.
 429                          */
 430
 431                         if ((id_len >= keywords.Length) || (keywords [id_len] == null))
 432                                 return -1;
 433                         object o = keywords [id_len] [id];
 434
 435                         if (o == null)
 436                                 return -1;
 437
 438                         int res = (int) o;
 439
 440                         if (handle_get_set == false && (res == Token.GET || res == Token.SET))
 441                                 return -1;
 442                         if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
 443                                 return -1;
 444                         if (handle_assembly == false && res == Token.ASSEMBLY)
 445                                 return -1;
 446 #if GMCS_SOURCE
 447                         if (handle_constraints == false && res == Token.WHERE)
 448                                 return -1;
 449 #endif
 450                         return res;
 451
 452                 }
 453
 454                 public Location Location {
 455                         get { return current_location; }
 456                 }
 457
 458                 void define (string def)
 459                 {
 460                         if (!RootContext.AllDefines.Contains (def)){
 461                                 RootContext.AllDefines [def] = true;
 462                         }
 463                         if (defines.Contains (def))
 464                                 return;
 465                         defines [def] = true;
 466                 }
 467
 468                 public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)
 469                 {
 470                         this.ref_name = file;
 471                         this.file_name = file;
 472                         reader = input;
 473
 474                         putback_char = -1;
 475
 476                         if (defs != null){
 477                                 defines = new Hashtable ();
 478                                 foreach (string def in defs)
 479                                         define (def);
 480                         }
 481
 482                         xml_comment_buffer = new StringBuilder ();
 483
 484                         //
 485                         // FIXME: This could be `Location.Push' but we have to
 486                         // find out why the MS compiler allows this
 487                         //
 488                         Mono.CSharp.Location.Push (file);
 489                 }
 490
 491                 static bool is_identifier_start_character (char c)
 492                 {
 493                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter (c);
 494                 }
 495
 496                 static bool is_identifier_part_character (char c)
 497                 {
 498                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
 499                 }
 500
 501                 public static bool IsKeyword (string s)
 502                 {
 503                         return keywordStrings [s] != null;
 504                 }
 505
 506                 public static bool IsValidIdentifier (string s)
 507                 {
 508                         if (s == null || s.Length == 0)
 509                                 return false;
 510
 511                         if (!is_identifier_start_character (s [0]))
 512                                 return false;
 513
 514                         for (int i = 1; i < s.Length; i ++)
 515                                 if (! is_identifier_part_character (s [i]))
 516                                         return false;
 517
 518                         return true;
 519                 }
 520
 521 #if GMCS_SOURCE
 522                 bool parse_generic_dimension (out int dimension)
 523                 {
 524                         dimension = 1;
 525
 526                 again:
 527                         int the_token = token ();
 528                         if (the_token == Token.OP_GENERICS_GT)
 529                                 return true;
 530                         else if (the_token == Token.COMMA) {
 531                                 dimension++;
 532                                 goto again;
 533                         }
 534
 535                         return false;
 536                 }
 537
 538                 bool parse_less_than ()
 539                 {
 540                 start:
 541                         int the_token = token ();
 542                         if (the_token == Token.OPEN_BRACKET) {
 543                                 do {
 544                                         the_token = token ();
 545                                 } while (the_token != Token.CLOSE_BRACKET);
 546                                 the_token = token ();
 547                         }
 548                         switch (the_token) {
 549                         case Token.IDENTIFIER:
 550                         case Token.OBJECT:
 551                         case Token.STRING:
 552                         case Token.BOOL:
 553                         case Token.DECIMAL:
 554                         case Token.FLOAT:
 555                         case Token.DOUBLE:
 556                         case Token.SBYTE:
 557                         case Token.BYTE:
 558                         case Token.SHORT:
 559                         case Token.USHORT:
 560                         case Token.INT:
 561                         case Token.UINT:
 562                         case Token.LONG:
 563                         case Token.ULONG:
 564                         case Token.CHAR:
 565                         case Token.VOID:
 566                                 break;
 567
 568                         default:
 569                                 return false;
 570                         }
 571                 again:
 572                         the_token = token ();
 573
 574                         if (the_token == Token.OP_GENERICS_GT)
 575                                 return true;
 576                         else if ((the_token == Token.COMMA) || (the_token == Token.DOT))
 577                                 goto start;
 578                         else if (the_token == Token.INTERR || the_token == Token.STAR)
 579                                 goto again;
 580                         else if (the_token == Token.OP_GENERICS_LT) {
 581                                 if (!parse_less_than ())
 582                                         return false;
 583                                 goto again;
 584                         } else if (the_token == Token.OPEN_BRACKET) {
 585                         rank_specifiers:
 586                                 the_token = token ();
 587                                 if (the_token == Token.CLOSE_BRACKET)
 588                                         goto again;
 589                                 else if (the_token == Token.COMMA)
 590                                         goto rank_specifiers;
 591                                 return false;
 592                         }
 593
 594                         return false;
 595                 }
 596
 597                 int parsing_generic_less_than = 0;
 598
 599                 public void PutbackNullable ()
 600                 {
 601                         if (nullable_pos < 0)
 602                                 throw new Exception ();
 603
 604                         current_token = -1;
 605                         val = null;
 606                         reader.Position = nullable_pos;
 607
 608                         putback_char = '?';
 609                 }
 610
 611                 public void PutbackCloseParens ()
 612                 {
 613                         putback_char = ')';
 614                 }
 615
 616
 617                 int nullable_pos = -1;
 618
 619                 public void CheckNullable (bool is_nullable)
 620                 {
 621                         if (is_nullable)
 622                                 nullable_pos = reader.Position;
 623                         else
 624                                 nullable_pos = -1;
 625                 }
 626 #endif
 627                 int is_punct (char c, ref bool doread)
 628                 {
 629                         int d;
 630                         int t;
 631
 632                         doread = false;
 633
 634                         switch (c){
 635                         case '{':
 636                                 val = Location;
 637                                 return Token.OPEN_BRACE;
 638                         case '}':
 639                                 val = Location;
 640                                 return Token.CLOSE_BRACE;
 641                         case '[':
 642                                 // To block doccomment inside attribute declaration.
 643                                 if (doc_state == XmlCommentState.Allowed)
 644                                         doc_state = XmlCommentState.NotAllowed;
 645                                 return Token.OPEN_BRACKET;
 646                         case ']':
 647                                 return Token.CLOSE_BRACKET;
 648                         case '(':
 649                                 return Token.OPEN_PARENS;
 650                         case ')': {
 651                                 if (deambiguate_close_parens == 0)
 652                                         return Token.CLOSE_PARENS;
 653
 654                                 --deambiguate_close_parens;
 655
 656                                 PushPosition ();
 657
 658                                 // disable preprocessing directives when peeking
 659                                 process_directives = false;
 660                                 int new_token = xtoken ();
 661                                 process_directives = true;
 662
 663                                 PopPosition ();
 664
 665                                 if (new_token == Token.OPEN_PARENS)
 666                                         return Token.CLOSE_PARENS_OPEN_PARENS;
 667                                 else if (new_token == Token.MINUS)
 668                                         return Token.CLOSE_PARENS_MINUS;
 669                                 else if (IsCastToken (new_token))
 670                                         return Token.CLOSE_PARENS_CAST;
 671                                 else
 672                                         return Token.CLOSE_PARENS_NO_CAST;
 673                         }
 674
 675                         case ',':
 676                                 return Token.COMMA;
 677                         case ';':
 678                                 val = Location;
 679                                 return Token.SEMICOLON;
 680                         case '~':
 681                                 val = Location;
 682                                 return Token.TILDE;
 683                         case '?':
 684                                 return Token.INTERR;
 685                         }
 686 #if GMCS_SOURCE
 687                         if (c == '<') {
 688                                 if (parsing_generic_less_than++ > 0)
 689                                         return Token.OP_GENERICS_LT;
 690
 691                                 if (handle_typeof) {
 692                                         int dimension;
 693                                         PushPosition ();
 694                                         if (parse_generic_dimension (out dimension)) {
 695                                                 val = dimension;
 696                                                 DiscardPosition ();
 697                                                 return Token.GENERIC_DIMENSION;
 698                                         }
 699                                         PopPosition ();
 700                                 }
 701
 702                                 // Save current position and parse next token.
 703                                 PushPosition ();
 704                                 bool is_generic_lt = parse_less_than ();
 705                                 PopPosition ();
 706
 707                                 if (is_generic_lt) {
 708                                         parsing_generic_less_than++;
 709                                         return Token.OP_GENERICS_LT;
 710                                 } else
 711                                         parsing_generic_less_than = 0;
 712
 713                                 d = peekChar ();
 714                                 if (d == '<'){
 715                                         getChar ();
 716                                         d = peekChar ();
 717
 718                                         if (d == '='){
 719                                                 doread = true;
 720                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 721                                         }
 722                                         return Token.OP_SHIFT_LEFT;
 723                                 } else if (d == '='){
 724                                         doread = true;
 725                                         return Token.OP_LE;
 726                                 }
 727                                 return Token.OP_LT;
 728                         } else if (c == '>') {
 729                                 if (parsing_generic_less_than > 0) {
 730                                         parsing_generic_less_than--;
 731                                         return Token.OP_GENERICS_GT;
 732                                 }
 733
 734                                 d = peekChar ();
 735                                 if (d == '>'){
 736                                         getChar ();
 737                                         d = peekChar ();
 738
 739                                         if (d == '='){
 740                                                 doread = true;
 741                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 742                                         }
 743                                         return Token.OP_SHIFT_RIGHT;
 744                                 } else if (d == '='){
 745                                         doread = true;
 746                                         return Token.OP_GE;
 747                                 }
 748                                 return Token.OP_GT;
 749                         }
 750 #endif
 751                         d = peekChar ();
 752                         if (c == '+'){
 753
 754                                 if (d == '+') {
 755                                         val = Location;
 756                                         t = Token.OP_INC;
 757                                 }
 758                                 else if (d == '=')
 759                                         t = Token.OP_ADD_ASSIGN;
 760                                 else {
 761                                         val = Location;
 762                                         return Token.PLUS;
 763                                 }
 764                                 doread = true;
 765                                 return t;
 766                         }
 767                         if (c == '-'){
 768                                 if (d == '-') {
 769                                         val = Location;
 770                                         t = Token.OP_DEC;
 771                                 }
 772                                 else if (d == '=')
 773                                         t = Token.OP_SUB_ASSIGN;
 774                                 else if (d == '>')
 775                                         t = Token.OP_PTR;
 776                                 else {
 777                                         val = Location;
 778                                         return Token.MINUS;
 779                                 }
 780                                 doread = true;
 781                                 return t;
 782                         }
 783
 784                         if (c == '!'){
 785                                 if (d == '='){
 786                                         doread = true;
 787                                         return Token.OP_NE;
 788                                 }
 789                                 val = Location;
 790                                 return Token.BANG;
 791                         }
 792
 793                         if (c == '='){
 794                                 if (d == '='){
 795                                         doread = true;
 796                                         return Token.OP_EQ;
 797                                 }
 798                                 return Token.ASSIGN;
 799                         }
 800
 801                         if (c == '&'){
 802                                 if (d == '&'){
 803                                         doread = true;
 804                                         return Token.OP_AND;
 805                                 } else if (d == '='){
 806                                         doread = true;
 807                                         return Token.OP_AND_ASSIGN;
 808                                 }
 809                                 val = Location;
 810                                 return Token.BITWISE_AND;
 811                         }
 812
 813                         if (c == '|'){
 814                                 if (d == '|'){
 815                                         doread = true;
 816                                         return Token.OP_OR;
 817                                 } else if (d == '='){
 818                                         doread = true;
 819                                         return Token.OP_OR_ASSIGN;
 820                                 }
 821                                 return Token.BITWISE_OR;
 822                         }
 823
 824                         if (c == '*'){
 825                                 if (d == '='){
 826                                         doread = true;
 827                                         return Token.OP_MULT_ASSIGN;
 828                                 }
 829                                 val = Location;
 830                                 return Token.STAR;
 831                         }
 832
 833                         if (c == '/'){
 834                                 if (d == '='){
 835                                         doread = true;
 836                                         return Token.OP_DIV_ASSIGN;
 837                                 }
 838                                 return Token.DIV;
 839                         }
 840
 841                         if (c == '%'){
 842                                 if (d == '='){
 843                                         doread = true;
 844                                         return Token.OP_MOD_ASSIGN;
 845                                 }
 846                                 return Token.PERCENT;
 847                         }
 848
 849                         if (c == '^'){
 850                                 if (d == '='){
 851                                         doread = true;
 852                                         return Token.OP_XOR_ASSIGN;
 853                                 }
 854                                 return Token.CARRET;
 855                         }
 856
 857 #if !GMCS_SOURCE
 858                         if (c == '<'){
 859                                 if (d == '<'){
 860                                         getChar ();
 861                                         d = peekChar ();
 862
 863                                         if (d == '='){
 864                                                 doread = true;
 865                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 866                                         }
 867                                         return Token.OP_SHIFT_LEFT;
 868                                 } else if (d == '='){
 869                                         doread = true;
 870                                         return Token.OP_LE;
 871                                 }
 872                                 return Token.OP_LT;
 873                         }
 874
 875                         if (c == '>'){
 876                                 if (d == '>'){
 877                                         getChar ();
 878                                         d = peekChar ();
 879
 880                                         if (d == '='){
 881                                                 doread = true;
 882                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 883                                         }
 884                                         return Token.OP_SHIFT_RIGHT;
 885                                 } else if (d == '='){
 886                                         doread = true;
 887                                         return Token.OP_GE;
 888                                 }
 889                                 return Token.OP_GT;
 890                         }
 891 #endif
 892                         if (c == ':'){
 893                                 if (d == ':'){
 894                                         doread = true;
 895                                         return Token.DOUBLE_COLON;
 896                                 }
 897                                 val = Location;
 898                                 return Token.COLON;
 899                         }
 900
 901                         return Token.ERROR;
 902                 }
 903
 904                 int deambiguate_close_parens = 0;
 905
 906                 public void Deambiguate_CloseParens (object expression)
 907                 {
 908                         putback (')');
 909
 910                         // When any binary operation is used we are sure it is not a cast
 911                         if (expression is Binary)
 912                                 return;
 913
 914                         deambiguate_close_parens++;
 915                 }
 916
 917                 bool decimal_digits (int c)
 918                 {
 919                         int d;
 920                         bool seen_digits = false;
 921
 922                         if (c != -1){
 923                                 if (number_pos == max_number_size)
 924                                         Error_NumericConstantTooLong ();
 925                                 number_builder [number_pos++] = (char) c;
 926                         }
 927
 928                         //
 929                         // We use peekChar2, because decimal_digits needs to do a
 930                         // 2-character look-ahead (5.ToString for example).
 931                         //
 932                         while ((d = peekChar2 ()) != -1){
 933                                 if (d >= '0' && d <= '9'){
 934                                         if (number_pos == max_number_size)
 935                                                 Error_NumericConstantTooLong ();
 936                                         number_builder [number_pos++] = (char) d;
 937                                         getChar ();
 938                                         seen_digits = true;
 939                                 } else
 940                                         break;
 941                         }
 942
 943                         return seen_digits;
 944                 }
 945
 946                 static bool is_hex (int e)
 947                 {
 948                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
 949                 }
 950
 951                 static int real_type_suffix (int c)
 952                 {
 953                         int t;
 954
 955                         switch (c){
 956                         case 'F': case 'f':
 957                                 t =  Token.LITERAL_FLOAT;
 958                                 break;
 959                         case 'D': case 'd':
 960                                 t = Token.LITERAL_DOUBLE;
 961                                 break;
 962                         case 'M': case 'm':
 963                                  t= Token.LITERAL_DECIMAL;
 964                                 break;
 965                         default:
 966                                 return Token.NONE;
 967                         }
 968                         return t;
 969                 }
 970
 971                 int integer_type_suffix (ulong ul, int c)
 972                 {
 973                         bool is_unsigned = false;
 974                         bool is_long = false;
 975
 976                         if (c != -1){
 977                                 bool scanning = true;
 978                                 do {
 979                                         switch (c){
 980                                         case 'U': case 'u':
 981                                                 if (is_unsigned)
 982                                                         scanning = false;
 983                                                 is_unsigned = true;
 984                                                 getChar ();
 985                                                 break;
 986
 987                                         case 'l':
 988                                                 if (!is_unsigned && (RootContext.WarningLevel >= 4)){
 989                                                         //
 990                                                         // if we have not seen anything in between
 991                                                         // report this error
 992                                                         //
 993                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
 994                                                 }
 995                                                 //
 996                                                 // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
 997                                                 // work around that.
 998                                                 //
 999                                                 //goto case 'L';
1000                                                 if (is_long)
1001                                                         scanning = false;
1002                                                 is_long = true;
1003                                                 getChar ();
1004                                                 break;
1005
1006                                         case 'L':
1007                                                 if (is_long)
1008                                                         scanning = false;
1009                                                 is_long = true;
1010                                                 getChar ();
1011                                                 break;
1012
1013                                         default:
1014                                                 scanning = false;
1015                                                 break;
1016                                         }
1017                                         c = peekChar ();
1018                                 } while (scanning);
1019                         }
1020
1021                         if (is_long && is_unsigned){
1022                                 val = ul;
1023                                 return Token.LITERAL_INTEGER;
1024                         } else if (is_unsigned){
1025                                 // uint if possible, or ulong else.
1026
1027                                 if ((ul & 0xffffffff00000000) == 0)
1028                                         val = (uint) ul;
1029                                 else
1030                                         val = ul;
1031                         } else if (is_long){
1032                                 // long if possible, ulong otherwise
1033                                 if ((ul & 0x8000000000000000) != 0)
1034                                         val = ul;
1035                                 else
1036                                         val = (long) ul;
1037                         } else {
1038                                 // int, uint, long or ulong in that order
1039                                 if ((ul & 0xffffffff00000000) == 0){
1040                                         uint ui = (uint) ul;
1041
1042                                         if ((ui & 0x80000000) != 0)
1043                                                 val = ui;
1044                                         else
1045                                                 val = (int) ui;
1046                                 } else {
1047                                         if ((ul & 0x8000000000000000) != 0)
1048                                                 val = ul;
1049                                         else
1050                                                 val = (long) ul;
1051                                 }
1052                         }
1053                         return Token.LITERAL_INTEGER;
1054                 }
1055
1056                 //
1057                 // given `c' as the next char in the input decide whether
1058                 // we need to convert to a special type, and then choose
1059                 // the best representation for the integer
1060                 //
1061                 int adjust_int (int c)
1062                 {
1063                         try {
1064                                 if (number_pos > 9){
1065                                         ulong ul = (uint) (number_builder [0] - '0');
1066
1067                                         for (int i = 1; i < number_pos; i++){
1068                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1069                                         }
1070                                         return integer_type_suffix (ul, c);
1071                                 } else {
1072                                         uint ui = (uint) (number_builder [0] - '0');
1073
1074                                         for (int i = 1; i < number_pos; i++){
1075                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1076                                         }
1077                                         return integer_type_suffix (ui, c);
1078                                 }
1079                         } catch (OverflowException) {
1080                                 error_details = "Integral constant is too large";
1081                                 Report.Error (1021, Location, error_details);
1082                                 val = 0ul;
1083                                 return Token.LITERAL_INTEGER;
1084                         }
1085                         catch (FormatException) {
1086                                 Report.Error (1013, Location, "Invalid number");
1087                                 val = 0ul;
1088                                 return Token.LITERAL_INTEGER;
1089                         }
1090                 }
1091
1092                 int adjust_real (int t)
1093                 {
1094                         string s = new String (number_builder, 0, number_pos);
1095                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1096
1097                         switch (t){
1098                         case Token.LITERAL_DECIMAL:
1099                                 try {
1100                                         val = System.Decimal.Parse (s, styles, csharp_format_info);
1101                                 } catch (OverflowException) {
1102                                         val = 0m;
1103                                         Report.Error (594, Location, error_details, "decimal");
1104                                 }
1105                                 break;
1106                         case Token.LITERAL_FLOAT:
1107                                 try {
1108                                         val = float.Parse (s, styles, csharp_format_info);
1109                                 } catch (OverflowException) {
1110                                         val = 0.0f;
1111                                         Report.Error (594, Location, error_details, "float");
1112                                 }
1113                                 break;
1114
1115                         case Token.LITERAL_DOUBLE:
1116                         case Token.NONE:
1117                                 t = Token.LITERAL_DOUBLE;
1118                                 try {
1119                                         val = System.Double.Parse (s, styles, csharp_format_info);
1120                                 } catch (OverflowException) {
1121                                         val = 0.0;
1122                                         Report.Error (594, Location, error_details, "double");
1123                                 }
1124                                 break;
1125                         }
1126                         return t;
1127                 }
1128
1129                 int handle_hex ()
1130                 {
1131                         int d;
1132                         ulong ul;
1133
1134                         getChar ();
1135                         while ((d = peekChar ()) != -1){
1136                                 if (is_hex (d)){
1137                                         number_builder [number_pos++] = (char) d;
1138                                         getChar ();
1139                                 } else
1140                                         break;
1141                         }
1142
1143                         string s = new String (number_builder, 0, number_pos);
1144                         try {
1145                                 if (number_pos <= 8)
1146                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1147                                 else
1148                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1149                         } catch (OverflowException){
1150                                 error_details = "Integral constant is too large";
1151                                 Report.Error (1021, Location, error_details);
1152                                 val = 0ul;
1153                                 return Token.LITERAL_INTEGER;
1154                         }
1155                         catch (FormatException) {
1156                                 Report.Error (1013, Location, "Invalid number");
1157                                 val = 0ul;
1158                                 return Token.LITERAL_INTEGER;
1159                         }
1160
1161                         return integer_type_suffix (ul, peekChar ());
1162                 }
1163
1164                 //
1165                 // Invoked if we know we have .digits or digits
1166                 //
1167                 int is_number (int c)
1168                 {
1169                         bool is_real = false;
1170                         int type;
1171
1172                         number_pos = 0;
1173
1174                         if (c >= '0' && c <= '9'){
1175                                 if (c == '0'){
1176                                         int peek = peekChar ();
1177
1178                                         if (peek == 'x' || peek == 'X')
1179                                                 return handle_hex ();
1180                                 }
1181                                 decimal_digits (c);
1182                                 c = getChar ();
1183                         }
1184
1185                         //
1186                         // We need to handle the case of
1187                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1188                         //
1189                         if (c == '.'){
1190                                 if (decimal_digits ('.')){
1191                                         is_real = true;
1192                                         c = getChar ();
1193                                 } else {
1194                                         putback ('.');
1195                                         number_pos--;
1196                                         return adjust_int (-1);
1197                                 }
1198                         }
1199
1200                         if (c == 'e' || c == 'E'){
1201                                 is_real = true;
1202                                 if (number_pos == max_number_size)
1203                                         Error_NumericConstantTooLong ();
1204                                 number_builder [number_pos++] = 'e';
1205                                 c = getChar ();
1206
1207                                 if (c == '+'){
1208                                         if (number_pos == max_number_size)
1209                                                 Error_NumericConstantTooLong ();
1210                                         number_builder [number_pos++] = '+';
1211                                         c = -1;
1212                                 } else if (c == '-') {
1213                                         if (number_pos == max_number_size)
1214                                                 Error_NumericConstantTooLong ();
1215                                         number_builder [number_pos++] = '-';
1216                                         c = -1;
1217                                 } else {
1218                                         if (number_pos == max_number_size)
1219                                                 Error_NumericConstantTooLong ();
1220                                         number_builder [number_pos++] = '+';
1221                                 }
1222
1223                                 decimal_digits (c);
1224                                 c = getChar ();
1225                         }
1226
1227                         type = real_type_suffix (c);
1228                         if (type == Token.NONE && !is_real){
1229                                 putback (c);
1230                                 return adjust_int (c);
1231                         } else
1232                                 is_real = true;
1233
1234                         if (type == Token.NONE){
1235                                 putback (c);
1236                         }
1237
1238                         if (is_real)
1239                                 return adjust_real (type);
1240
1241                         Console.WriteLine ("This should not be reached");
1242                         throw new Exception ("Is Number should never reach this point");
1243                 }
1244
1245                 //
1246                 // Accepts exactly count (4 or 8) hex, no more no less
1247                 //
1248                 int getHex (int count, out bool error)
1249                 {
1250                         int i;
1251                         int total = 0;
1252                         int c;
1253                         int top = count != -1 ? count : 4;
1254
1255                         getChar ();
1256                         error = false;
1257                         for (i = 0; i < top; i++){
1258                                 c = getChar ();
1259
1260                                 if (c >= '0' && c <= '9')
1261                                         c = (int) c - (int) '0';
1262                                 else if (c >= 'A' && c <= 'F')
1263                                         c = (int) c - (int) 'A' + 10;
1264                                 else if (c >= 'a' && c <= 'f')
1265                                         c = (int) c - (int) 'a' + 10;
1266                                 else {
1267                                         error = true;
1268                                         return 0;
1269                                 }
1270
1271                                 total = (total * 16) + c;
1272                                 if (count == -1){
1273                                         int p = peekChar ();
1274                                         if (p == -1)
1275                                                 break;
1276                                         if (!is_hex ((char)p))
1277                                                 break;
1278                                 }
1279                         }
1280                         return total;
1281                 }
1282
1283                 int escape (int c)
1284                 {
1285                         bool error;
1286                         int d;
1287                         int v;
1288
1289                         d = peekChar ();
1290                         if (c != '\\')
1291                                 return c;
1292
1293                         switch (d){
1294                         case 'a':
1295                                 v = '\a'; break;
1296                         case 'b':
1297                                 v = '\b'; break;
1298                         case 'n':
1299                                 v = '\n'; break;
1300                         case 't':
1301                                 v = '\t'; break;
1302                         case 'v':
1303                                 v = '\v'; break;
1304                         case 'r':
1305                                 v = '\r'; break;
1306                         case '\\':
1307                                 v = '\\'; break;
1308                         case 'f':
1309                                 v = '\f'; break;
1310                         case '0':
1311                                 v = 0; break;
1312                         case '"':
1313                                 v = '"'; break;
1314                         case '\'':
1315                                 v = '\''; break;
1316                         case 'x':
1317                                 v = getHex (-1, out error);
1318                                 if (error)
1319                                         goto default;
1320                                 return v;
1321                         case 'u':
1322                                 v = getHex (4, out error);
1323                                 if (error)
1324                                         goto default;
1325                                 return v;
1326                         case 'U':
1327                                 v = getHex (8, out error);
1328                                 if (error)
1329                                         goto default;
1330                                 return v;
1331                         default:
1332                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1333                                 return d;
1334                         }
1335                         getChar ();
1336                         return v;
1337                 }
1338
1339                 int getChar ()
1340                 {
1341                         int x;
1342                         if (putback_char != -1) {
1343                                 x = putback_char;
1344                                 putback_char = -1;
1345                         } else
1346                                 x = reader.Read ();
1347                         if (x == '\n') {
1348                                 line++;
1349                                 ref_line++;
1350                                 previous_col = col;
1351                                 col = 0;
1352                         }
1353                         else
1354                                 col++;
1355                         return x;
1356                 }
1357
1358                 int peekChar ()
1359                 {
1360                         if (putback_char != -1)
1361                                 return putback_char;
1362                         putback_char = reader.Read ();
1363                         return putback_char;
1364                 }
1365
1366                 int peekChar2 ()
1367                 {
1368                         if (putback_char != -1)
1369                                 return putback_char;
1370                         return reader.Peek ();
1371                 }
1372
1373                 void putback (int c)
1374                 {
1375                         if (putback_char != -1){
1376                                 Console.WriteLine ("Col: " + col);
1377                                 Console.WriteLine ("Row: " + line);
1378                                 Console.WriteLine ("Name: " + ref_name.Name);
1379                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1380                                 throw new Exception ("This should not happen putback on putback");
1381                         }
1382                         if (c == '\n' || col == 0) {
1383                                 // It won't happen though.
1384                                 line--;
1385                                 ref_line--;
1386                                 col = previous_col;
1387                         }
1388                         else
1389                                 col--;
1390                         putback_char = c;
1391                 }
1392
1393                 public bool advance ()
1394                 {
1395                         return peekChar () != -1;
1396                 }
1397
1398                 public Object Value {
1399                         get {
1400                                 return val;
1401                         }
1402                 }
1403
1404                 public Object value ()
1405                 {
1406                         return val;
1407                 }
1408
1409                 static bool IsCastToken (int token)
1410                 {
1411                         switch (token) {
1412                         case Token.BANG:
1413                         case Token.TILDE:
1414                         case Token.IDENTIFIER:
1415                         case Token.LITERAL_INTEGER:
1416                         case Token.LITERAL_FLOAT:
1417                         case Token.LITERAL_DOUBLE:
1418                         case Token.LITERAL_DECIMAL:
1419                         case Token.LITERAL_CHARACTER:
1420                         case Token.LITERAL_STRING:
1421                         case Token.BASE:
1422                         case Token.CHECKED:
1423                         case Token.DELEGATE:
1424                         case Token.FALSE:
1425                         case Token.FIXED:
1426                         case Token.NEW:
1427                         case Token.NULL:
1428                         case Token.SIZEOF:
1429                         case Token.THIS:
1430                         case Token.THROW:
1431                         case Token.TRUE:
1432                         case Token.TYPEOF:
1433                         case Token.UNCHECKED:
1434                         case Token.UNSAFE:
1435 #if GMCS_SOURCE
1436                         case Token.DEFAULT:
1437 #endif
1438
1439                                 //
1440                                 // These can be part of a member access
1441                                 //
1442                         case Token.INT:
1443                         case Token.UINT:
1444                         case Token.SHORT:
1445                         case Token.USHORT:
1446                         case Token.LONG:
1447                         case Token.ULONG:
1448                         case Token.DOUBLE:
1449                         case Token.FLOAT:
1450                         case Token.CHAR:
1451                                 return true;
1452
1453                         default:
1454                                 return false;
1455                         }
1456                 }
1457
1458                 public int token ()
1459                 {
1460                         current_token = xtoken ();
1461
1462 #if GMCS_SOURCE
1463                         if (current_token != Token.DEFAULT)
1464                                 return current_token;
1465
1466                         PushPosition();
1467                         int c = xtoken();
1468                         if (c == -1)
1469                                 current_token = Token.ERROR;
1470                         else if (c == Token.OPEN_PARENS)
1471                                 current_token = Token.DEFAULT_OPEN_PARENS;
1472                         else if (c == Token.COLON)
1473                                 current_token = Token.DEFAULT_COLON;
1474                         else
1475                                 PopPosition();
1476 #endif
1477                         return current_token;
1478                 }
1479
1480                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1481
1482                 void get_cmd_arg (out string cmd, out string arg)
1483                 {
1484                         int c;
1485
1486                         tokens_seen = false;
1487                         arg = "";
1488                         static_cmd_arg.Length = 0;
1489
1490                         // skip over white space
1491                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1492                                 ;
1493
1494                         while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
1495                                 if (is_identifier_part_character ((char) c)){
1496                                         static_cmd_arg.Append ((char) c);
1497                                         c = getChar ();
1498                                 } else {
1499                                         putback (c);
1500                                         break;
1501                                 }
1502                         }
1503
1504                         cmd = static_cmd_arg.ToString ();
1505
1506                         if (c == '\n'){
1507                                 return;
1508                         }
1509
1510                         // skip over white space
1511                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1512                                 ;
1513
1514                         if (c == '\n'){
1515                                 return;
1516                         } else if (c == '\r'){
1517                                 return;
1518                         } else if (c == -1){
1519                                 arg = "";
1520                                 return;
1521                         }
1522
1523                         static_cmd_arg.Length = 0;
1524                         static_cmd_arg.Append ((char) c);
1525
1526                         while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
1527                                 static_cmd_arg.Append ((char) c);
1528                         }
1529
1530                         arg = static_cmd_arg.ToString ().Trim ();
1531                 }
1532
1533                 //
1534                 // Handles the #line directive
1535                 //
1536                 bool PreProcessLine (string arg)
1537                 {
1538                         if (arg.Length == 0)
1539                                 return false;
1540
1541                         if (arg == "default"){
1542                                 ref_line = line;
1543                                 ref_name = file_name;
1544                                 Location.Push (ref_name);
1545                                 return true;
1546                         } else if (arg == "hidden"){
1547                                 //
1548                                 // We ignore #line hidden
1549                                 //
1550                                 return true;
1551                         }
1552
1553                         try {
1554                                 int pos;
1555
1556                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1557                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1558                                         pos++;
1559
1560                                         char [] quotes = { '\"' };
1561
1562                                         string name = arg.Substring (pos). Trim (quotes);
1563                                         ref_name = Location.LookupFile (name);
1564                                         file_name.HasLineDirective = true;
1565                                         ref_name.HasLineDirective = true;
1566                                         Location.Push (ref_name);
1567                                 } else {
1568                                         ref_line = System.Int32.Parse (arg);
1569                                 }
1570                         } catch {
1571                                 return false;
1572                         }
1573
1574                         return true;
1575                 }
1576
1577                 //
1578                 // Handles #define and #undef
1579                 //
1580                 void PreProcessDefinition (bool is_define, string arg)
1581                 {
1582                         if (arg.Length == 0 || arg == "true" || arg == "false"){
1583                                 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
1584                                 return;
1585                         }
1586
1587                         char[] whitespace = { ' ', '\t' };
1588                         if (arg.IndexOfAny (whitespace) != -1){
1589                                 Report.Error (1025, Location, "Single-line comment or end-of-line expected");
1590                                 return;
1591                         }
1592
1593                         if (!is_identifier_start_character (arg [0]))
1594                                 Report.Error (1001, Location, "Identifier expected: " + arg);
1595
1596                         foreach (char c in arg.Substring (1)){
1597                                 if (!is_identifier_part_character (c)){
1598                                         Report.Error (1001, Location, "Identifier expected: " + arg);
1599                                         return;
1600                                 }
1601                         }
1602
1603                         if (is_define){
1604                                 if (defines == null)
1605                                         defines = new Hashtable ();
1606                                 define (arg);
1607                         } else {
1608                                 if (defines == null)
1609                                         return;
1610                                 if (defines.Contains (arg))
1611                                         defines.Remove (arg);
1612                         }
1613                 }
1614
1615                 /// <summary>
1616                 /// Handles #pragma directive
1617                 /// </summary>
1618                 void PreProcessPragma (string arg)
1619                 {
1620                         const string warning = "warning";
1621                         const string w_disable = "warning disable";
1622                         const string w_restore = "warning restore";
1623
1624                         if (arg == w_disable) {
1625                                 Report.RegisterWarningRegion (Location).WarningDisable (line);
1626                                 return;
1627                         }
1628
1629                         if (arg == w_restore) {
1630                                 Report.RegisterWarningRegion (Location).WarningEnable (line);
1631                                 return;
1632                         }
1633
1634                         if (arg.StartsWith (w_disable)) {
1635                                 int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
1636                                 foreach (int code in codes) {
1637                                         if (code != 0)
1638                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1639                                 }
1640                                 return;
1641                         }
1642
1643                         if (arg.StartsWith (w_restore)) {
1644                                 int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
1645                                 Hashtable w_table = Report.warning_ignore_table;
1646                                 foreach (int code in codes) {
1647                                         if (w_table != null && w_table.Contains (code))
1648                                                 Report.Warning (1635, 1, Location, String.Format ("Cannot restore warning `CS{0:0000}' because it was disabled globally", code));
1649                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1650                                 }
1651                                 return;
1652                         }
1653
1654                         if (arg.StartsWith (warning)) {
1655                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
1656                                 return;
1657                         }
1658
1659                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
1660                 }
1661
1662                 int[] ParseNumbers (string text)
1663                 {
1664                         string[] string_array = text.Split (',');
1665                         int[] values = new int [string_array.Length];
1666                         int index = 0;
1667                         foreach (string string_code in string_array) {
1668                                 try {
1669                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1670                                 }
1671                                 catch (FormatException) {
1672                                         Report.Warning (1692, 1, Location, "Invalid number");
1673                                 }
1674                         }
1675                         return values;
1676                 }
1677
1678                 bool eval_val (string s)
1679                 {
1680                         if (s == "true")
1681                                 return true;
1682                         if (s == "false")
1683                                 return false;
1684
1685                         if (defines == null)
1686                                 return false;
1687                         if (defines.Contains (s))
1688                                 return true;
1689
1690                         return false;
1691                 }
1692
1693                 bool pp_primary (ref string s)
1694                 {
1695                         s = s.Trim ();
1696                         int len = s.Length;
1697
1698                         if (len > 0){
1699                                 char c = s [0];
1700
1701                                 if (c == '('){
1702                                         s = s.Substring (1);
1703                                         bool val = pp_expr (ref s);
1704                                         if (s.Length > 0 && s [0] == ')'){
1705                                                 s = s.Substring (1);
1706                                                 return val;
1707                                         }
1708                                         Error_InvalidDirective ();
1709                                         return false;
1710                                 }
1711
1712                                 if (is_identifier_start_character (c)){
1713                                         int j = 1;
1714
1715                                         while (j < len){
1716                                                 c = s [j];
1717
1718                                                 if (is_identifier_part_character (c)){
1719                                                         j++;
1720                                                         continue;
1721                                                 }
1722                                                 bool v = eval_val (s.Substring (0, j));
1723                                                 s = s.Substring (j);
1724                                                 return v;
1725                                         }
1726                                         bool vv = eval_val (s);
1727                                         s = "";
1728                                         return vv;
1729                                 }
1730                         }
1731                         Error_InvalidDirective ();
1732                         return false;
1733                 }
1734
1735                 bool pp_unary (ref string s)
1736                 {
1737                         s = s.Trim ();
1738                         int len = s.Length;
1739
1740                         if (len > 0){
1741                                 if (s [0] == '!'){
1742                                         if (len > 1 && s [1] == '='){
1743                                                 Error_InvalidDirective ();
1744                                                 return false;
1745                                         }
1746                                         s = s.Substring (1);
1747                                         return ! pp_primary (ref s);
1748                                 } else
1749                                         return pp_primary (ref s);
1750                         } else {
1751                                 Error_InvalidDirective ();
1752                                 return false;
1753                         }
1754                 }
1755
1756                 bool pp_eq (ref string s)
1757                 {
1758                         bool va = pp_unary (ref s);
1759
1760                         s = s.Trim ();
1761                         int len = s.Length;
1762                         if (len > 0){
1763                                 if (s [0] == '='){
1764                                         if (len > 2 && s [1] == '='){
1765                                                 s = s.Substring (2);
1766                                                 return va == pp_unary (ref s);
1767                                         } else {
1768                                                 Error_InvalidDirective ();
1769                                                 return false;
1770                                         }
1771                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
1772                                         s = s.Substring (2);
1773
1774                                         return va != pp_unary (ref s);
1775
1776                                 }
1777                         }
1778
1779                         return va;
1780
1781                 }
1782
1783                 bool pp_and (ref string s)
1784                 {
1785                         bool va = pp_eq (ref s);
1786
1787                         s = s.Trim ();
1788                         int len = s.Length;
1789                         if (len > 0){
1790                                 if (s [0] == '&'){
1791                                         if (len > 2 && s [1] == '&'){
1792                                                 s = s.Substring (2);
1793                                                 return (va & pp_and (ref s));
1794                                         } else {
1795                                                 Error_InvalidDirective ();
1796                                                 return false;
1797                                         }
1798                                 }
1799                         }
1800                         return va;
1801                 }
1802
1803                 //
1804                 // Evaluates an expression for `#if' or `#elif'
1805                 //
1806                 bool pp_expr (ref string s)
1807                 {
1808                         bool va = pp_and (ref s);
1809                         s = s.Trim ();
1810                         int len = s.Length;
1811                         if (len > 0){
1812                                 char c = s [0];
1813
1814                                 if (c == '|'){
1815                                         if (len > 2 && s [1] == '|'){
1816                                                 s = s.Substring (2);
1817                                                 return va | pp_expr (ref s);
1818                                         } else {
1819                                                 Error_InvalidDirective ();
1820                                                 return false;
1821                                         }
1822                                 }
1823                         }
1824
1825                         return va;
1826                 }
1827
1828                 bool eval (string s)
1829                 {
1830                         bool v = pp_expr (ref s);
1831                         s = s.Trim ();
1832                         if (s.Length != 0){
1833                                 return false;
1834                         }
1835
1836                         return v;
1837                 }
1838
1839                 void Error_NumericConstantTooLong ()
1840                 {
1841                         Report.Error (1021, Location, "Numeric constant too long");
1842                 }
1843
1844                 void Error_InvalidDirective ()
1845                 {
1846                         Report.Error (1517, Location, "Invalid preprocessor directive");
1847                 }
1848
1849                 void Error_UnexpectedDirective (string extra)
1850                 {
1851                         Report.Error (
1852                                 1028, Location,
1853                                 "Unexpected processor directive (" + extra + ")");
1854                 }
1855
1856                 void Error_TokensSeen ()
1857                 {
1858                         Report.Error (1032, Location,
1859                                 "Cannot define or undefine preprocessor symbols after first token in file");
1860                 }
1861
1862                 //
1863                 // Set to false to stop handling preprocesser directives
1864                 //
1865                 bool process_directives = true;
1866
1867                 //
1868                 // if true, then the code continues processing the code
1869                 // if false, the code stays in a loop until another directive is
1870                 // reached.
1871                 //
1872                 bool handle_preprocessing_directive (bool caller_is_taking)
1873                 {
1874                         string cmd, arg;
1875                         bool region_directive = false;
1876
1877                         get_cmd_arg (out cmd, out arg);
1878
1879                         // Eat any trailing whitespaces and single-line comments
1880                         if (arg.IndexOf ("//") != -1)
1881                                 arg = arg.Substring (0, arg.IndexOf ("//"));
1882                         arg = arg.TrimEnd (' ', '\t');
1883
1884                         //
1885                         // The first group of pre-processing instructions is always processed
1886                         //
1887                         switch (cmd){
1888                         case "pragma":
1889                                 if (RootContext.Version == LanguageVersion.ISO_1) {
1890                                         Report.FeatureIsNotStandardized (Location, "#pragma");
1891                                         return caller_is_taking;
1892                                 }
1893
1894                                 PreProcessPragma (arg);
1895                                 return caller_is_taking;
1896
1897                         case "line":
1898                                 if (!PreProcessLine (arg))
1899                                         Report.Error (
1900                                                 1576, Location,
1901                                                 "The line number specified for #line directive is missing or invalid");
1902                                 return caller_is_taking;
1903
1904                         case "region":
1905                                 region_directive = true;
1906                                 arg = "true";
1907                                 goto case "if";
1908
1909                         case "endregion":
1910                                 region_directive = true;
1911                                 goto case "endif";
1912
1913                         case "if":
1914                                 if (arg.Length == 0){
1915                                         Error_InvalidDirective ();
1916                                         return true;
1917                                 }
1918                                 bool taking = false;
1919                                 if (ifstack == null)
1920                                         ifstack = new Stack (2);
1921
1922                                 if (ifstack.Count == 0){
1923                                         taking = true;
1924                                 } else {
1925                                         int state = (int) ifstack.Peek ();
1926                                         if ((state & TAKING) != 0)
1927                                                 taking = true;
1928                                 }
1929
1930                                 if (eval (arg) && taking){
1931                                         int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;
1932                                         if (region_directive)
1933                                                 push |= REGION;
1934                                         ifstack.Push (push);
1935                                         return true;
1936                                 } else {
1937                                         int push = (taking ? PARENT_TAKING : 0);
1938                                         if (region_directive)
1939                                                 push |= REGION;
1940                                         ifstack.Push (push);
1941                                         return false;
1942                                 }
1943
1944                         case "endif":
1945                                 if (ifstack == null || ifstack.Count == 0){
1946                                         Error_UnexpectedDirective ("no #if for this #endif");
1947                                         return true;
1948                                 } else {
1949                                         int pop = (int) ifstack.Pop ();
1950
1951                                         if (region_directive && ((pop & REGION) == 0))
1952                                                 Report.Error (1027, Location, "Expected `#endif' directive");
1953                                         else if (!region_directive && ((pop & REGION) != 0))
1954                                                 Report.Error (1038, Location, "#endregion directive expected");
1955
1956                                         if (!region_directive && arg.Length != 0) {
1957                                                 Report.Error (1025, Location, "Single-line comment or end-of-line expected");
1958                                         }
1959
1960                                         if (ifstack.Count == 0)
1961                                                 return true;
1962                                         else {
1963                                                 int state = (int) ifstack.Peek ();
1964
1965                                                 if ((state & TAKING) != 0)
1966                                                         return true;
1967                                                 else
1968                                                         return false;
1969                                         }
1970                                 }
1971
1972                         case "elif":
1973                                 if (ifstack == null || ifstack.Count == 0){
1974                                         Error_UnexpectedDirective ("no #if for this #elif");
1975                                         return true;
1976                                 } else {
1977                                         int state = (int) ifstack.Peek ();
1978
1979                                         if ((state & REGION) != 0) {
1980                                                 Report.Error (1038, Location, "#endregion directive expected");
1981                                                 return true;
1982                                         }
1983
1984                                         if ((state & ELSE_SEEN) != 0){
1985                                                 Error_UnexpectedDirective ("#elif not valid after #else");
1986                                                 return true;
1987                                         }
1988
1989                                         if ((state & (TAKEN_BEFORE | TAKING)) != 0)
1990                                                 return false;
1991
1992                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
1993                                                 state = (int) ifstack.Pop ();
1994                                                 ifstack.Push (state | TAKING | TAKEN_BEFORE);
1995                                                 return true;
1996                                         } else
1997                                                 return false;
1998                                 }
1999
2000                         case "else":
2001                                 if (ifstack == null || ifstack.Count == 0){
2002                                         Error_UnexpectedDirective ("no #if for this #else");
2003                                         return true;
2004                                 } else {
2005                                         int state = (int) ifstack.Peek ();
2006
2007                                         if ((state & REGION) != 0) {
2008                                                 Report.Error (1038, Location, "#endregion directive expected");
2009                                                 return true;
2010                                         }
2011
2012                                         if ((state & ELSE_SEEN) != 0){
2013                                                 Error_UnexpectedDirective ("#else within #else");
2014                                                 return true;
2015                                         }
2016
2017                                         ifstack.Pop ();
2018
2019                                         bool ret;
2020                                         if ((state & TAKEN_BEFORE) == 0){
2021                                                 ret = ((state & PARENT_TAKING) != 0);
2022                                         } else
2023                                                 ret = false;
2024
2025                                         if (ret)
2026                                                 state |= TAKING;
2027                                         else
2028                                                 state &= ~TAKING;
2029
2030                                         ifstack.Push (state | ELSE_SEEN);
2031
2032                                         return ret;
2033                                 }
2034                         }
2035
2036                         //
2037                         // These are only processed if we are in a `taking' block
2038                         //
2039                         if (!caller_is_taking)
2040                                 return false;
2041
2042                         switch (cmd){
2043                         case "define":
2044                                 if (any_token_seen){
2045                                         Error_TokensSeen ();
2046                                         return true;
2047                                 }
2048                                 PreProcessDefinition (true, arg);
2049                                 return true;
2050
2051                         case "undef":
2052                                 if (any_token_seen){
2053                                         Error_TokensSeen ();
2054                                         return true;
2055                                 }
2056                                 PreProcessDefinition (false, arg);
2057                                 return true;
2058
2059                         case "error":
2060                                 Report.Error (1029, Location, "#error: '" + arg + "'");
2061                                 return true;
2062
2063                         case "warning":
2064                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2065                                 return true;
2066                         }
2067
2068                         Report.Error (1024, Location, "Wrong preprocessor directive");
2069                         return true;
2070
2071                 }
2072
2073                 private int consume_string (bool quoted)
2074                 {
2075                         int c;
2076                         string_builder.Length = 0;
2077
2078                         while ((c = getChar ()) != -1){
2079                                 if (c == '"'){
2080                                         if (quoted && peekChar () == '"'){
2081                                                 string_builder.Append ((char) c);
2082                                                 getChar ();
2083                                                 continue;
2084                                         } else {
2085                                                 val = string_builder.ToString ();
2086                                                 return Token.LITERAL_STRING;
2087                                         }
2088                                 }
2089
2090                                 if (c == '\n'){
2091                                         if (!quoted)
2092                                                 Report.Error (1010, Location, "Newline in constant");
2093                                 }
2094
2095                                 if (!quoted){
2096                                         c = escape (c);
2097                                         if (c == -1)
2098                                                 return Token.ERROR;
2099                                 }
2100                                 string_builder.Append ((char) c);
2101                         }
2102
2103                         Report.Error (1039, Location, "Unterminated string literal");
2104                         return Token.EOF;
2105                 }
2106
2107                 private int consume_identifier (int s)
2108                 {
2109                         int res = consume_identifier (s, false);
2110
2111                         if (doc_state == XmlCommentState.Allowed)
2112                                 doc_state = XmlCommentState.NotAllowed;
2113                         switch (res) {
2114                         case Token.USING:
2115                         case Token.NAMESPACE:
2116                                 check_incorrect_doc_comment ();
2117                                 break;
2118                         }
2119
2120                         if (res == Token.PARTIAL) {
2121                                 // Save current position and parse next token.
2122                                 PushPosition ();
2123
2124                                 int next_token = token ();
2125                                 bool ok = (next_token == Token.CLASS) ||
2126                                         (next_token == Token.STRUCT) ||
2127                                         (next_token == Token.INTERFACE);
2128
2129                                 PopPosition ();
2130
2131                                 if (ok)
2132                                         return res;
2133
2134 //                              Report.Error (267, Location, "The `partial' modifier can be used only immediately before keyword `class', `struct', or `interface'");
2135                                 val = new LocatedToken (Location, "partial");
2136                                 return Token.IDENTIFIER;
2137                         }
2138
2139                         return res;
2140                 }
2141
2142                 private int consume_identifier (int s, bool quoted)
2143                 {
2144                         int pos = 1;
2145                         int c = -1;
2146
2147                         id_builder [0] = (char) s;
2148
2149                         current_location = new Location (ref_line, Col);
2150
2151                         while ((c = getChar ()) != -1) {
2152                         loop:
2153                                 if (is_identifier_part_character ((char) c)){
2154                                         if (pos == max_id_size){
2155                                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2156                                                 return Token.ERROR;
2157                                         }
2158
2159                                         id_builder [pos++] = (char) c;
2160 //                                      putback_char = -1;
2161                                 } else if (c == '\\') {
2162                                         c = escape (c);
2163                                         goto loop;
2164                                 } else {
2165 //                                      putback_char = c;
2166                                         putback (c);
2167                                         break;
2168                                 }
2169                         }
2170
2171                         //
2172                         // Optimization: avoids doing the keyword lookup
2173                         // on uppercase letters and _
2174                         //
2175                         if (!quoted && (s >= 'a' || s == '_')){
2176                                 int keyword = GetKeyword (id_builder, pos);
2177                                 if (keyword != -1) {
2178                                         val = Location;
2179                                         return keyword;
2180                                 }
2181                         }
2182
2183                         //
2184                         // Keep identifiers in an array of hashtables to avoid needless
2185                         // allocations
2186                         //
2187
2188                         if (identifiers [pos] != null) {
2189                                 val = identifiers [pos][id_builder];
2190                                 if (val != null) {
2191                                         val = new LocatedToken (Location, (string) val);
2192                                         if (quoted)
2193                                                 escapedIdentifiers.Add (val);
2194                                         return Token.IDENTIFIER;
2195                                 }
2196                         }
2197                         else
2198                                 identifiers [pos] = new CharArrayHashtable (pos);
2199
2200                         val = new String (id_builder, 0, pos);
2201                         if (RootContext.Version == LanguageVersion.ISO_1) {
2202                                 for (int i = 1; i < id_builder.Length; i += 3) {
2203                                         if (id_builder [i] == '_' && (id_builder [i - 1] == '_' || id_builder [i + 1] == '_')) {
2204                                                 Report.Error (1638, Location,
2205                                                         "`{0}': Any identifier with double underscores cannot be used when ISO language version mode is specified", val.ToString ());
2206                                                 break;
2207                                         }
2208                                 }
2209                         }
2210
2211                         char [] chars = new char [pos];
2212                         Array.Copy (id_builder, chars, pos);
2213
2214                         identifiers [pos] [chars] = val;
2215
2216                         val = new LocatedToken (Location, (string) val);
2217                         if (quoted)
2218                                 escapedIdentifiers.Add (val);
2219                         return Token.IDENTIFIER;
2220                 }
2221
2222                 public int xtoken ()
2223                 {
2224                         int t;
2225                         bool doread = false;
2226                         int c;
2227
2228                         // Whether we have seen comments on the current line
2229                         bool comments_seen = false;
2230
2231                         val = null;
2232                         // optimization: eliminate col and implement #directive semantic correctly.
2233                         for (;(c = getChar ()) != -1;) {
2234                                 if (c == ' ')
2235                                         continue;
2236
2237                                 if (c == '\t') {
2238                                         continue;
2239                                 }
2240
2241                                 if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0)
2242                                         continue;
2243
2244                                 if (c == '\r') {
2245                                         if (peekChar () == '\n')
2246                                                 getChar ();
2247
2248                                         any_token_seen |= tokens_seen;
2249                                         tokens_seen = false;
2250                                         comments_seen = false;
2251                                         continue;
2252                                 }
2253
2254                                 // Handle double-slash comments.
2255                                 if (c == '/'){
2256                                         int d = peekChar ();
2257
2258                                         if (d == '/'){
2259                                                 getChar ();
2260                                                 if (RootContext.Documentation != null && peekChar () == '/') {
2261                                                         getChar ();
2262                                                         // Don't allow ////.
2263                                                         if ((d = peekChar ()) != '/') {
2264                                                                 update_comment_location ();
2265                                                                 if (doc_state == XmlCommentState.Allowed)
2266                                                                         handle_one_line_xml_comment ();
2267                                                                 else if (doc_state == XmlCommentState.NotAllowed)
2268                                                                         warn_incorrect_doc_comment ();
2269                                                         }
2270                                                 }
2271                                                 while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
2272                                                 if (d == '\n'){
2273                                                 }
2274                                                 any_token_seen |= tokens_seen;
2275                                                 tokens_seen = false;
2276                                                 comments_seen = false;
2277                                                 continue;
2278                                         } else if (d == '*'){
2279                                                 getChar ();
2280                                                 bool docAppend = false;
2281                                                 if (RootContext.Documentation != null && peekChar () == '*') {
2282                                                         getChar ();
2283                                                         update_comment_location ();
2284                                                         // But when it is /**/, just do nothing.
2285                                                         if (peekChar () == '/') {
2286                                                                 getChar ();
2287                                                                 continue;
2288                                                         }
2289                                                         if (doc_state == XmlCommentState.Allowed)
2290                                                                 docAppend = true;
2291                                                         else if (doc_state == XmlCommentState.NotAllowed)
2292                                                                 warn_incorrect_doc_comment ();
2293                                                 }
2294
2295                                                 int current_comment_start = 0;
2296                                                 if (docAppend) {
2297                                                         current_comment_start = xml_comment_buffer.Length;
2298                                                         xml_comment_buffer.Append (Environment.NewLine);
2299                                                 }
2300
2301                                                 Location start_location = Location;
2302
2303                                                 while ((d = getChar ()) != -1){
2304                                                         if (d == '*' && peekChar () == '/'){
2305                                                                 getChar ();
2306                                                                 comments_seen = true;
2307                                                                 break;
2308                                                         }
2309                                                         if (docAppend)
2310                                                                 xml_comment_buffer.Append ((char) d);
2311
2312                                                         if (d == '\n'){
2313                                                                 any_token_seen |= tokens_seen;
2314                                                                 tokens_seen = false;
2315                                                                 //
2316                                                                 // Reset 'comments_seen' just to be consistent.
2317                                                                 // It doesn't matter either way, here.
2318                                                                 //
2319                                                                 comments_seen = false;
2320                                                         }
2321                                                 }
2322                                                 if (!comments_seen)
2323                                                         Report.Error (1035, start_location, "End-of-file found, '*/' expected");
2324
2325                                                 if (docAppend)
2326                                                         update_formatted_doc_comment (current_comment_start);
2327                                                 continue;
2328                                         }
2329                                         goto is_punct_label;
2330                                 }
2331
2332
2333                                 if (c == '\\' || is_identifier_start_character ((char)c)){
2334                                         tokens_seen = true;
2335                                         return consume_identifier (c);
2336                                 }
2337
2338                         is_punct_label:
2339                                 current_location = new Location (ref_line, Col);
2340                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
2341                                         tokens_seen = true;
2342                                         if (doread){
2343                                                 getChar ();
2344                                         }
2345                                         return t;
2346                                 }
2347
2348                                 // white space
2349                                 if (c == '\n'){
2350                                         any_token_seen |= tokens_seen;
2351                                         tokens_seen = false;
2352                                         comments_seen = false;
2353                                         continue;
2354                                 }
2355
2356                                 if (c >= '0' && c <= '9'){
2357                                         tokens_seen = true;
2358                                         return is_number (c);
2359                                 }
2360
2361                                 if (c == '.'){
2362                                         tokens_seen = true;
2363                                         int peek = peekChar ();
2364                                         if (peek >= '0' && peek <= '9')
2365                                                 return is_number (c);
2366                                         return Token.DOT;
2367                                 }
2368
2369                                 /* For now, ignore pre-processor commands */
2370                                 // FIXME: In C# the '#' is not limited to appear
2371                                 // on the first column.
2372                                 if (c == '#') {
2373                                         // return NONE if we're not processing directives (during token peeks)
2374                                         if (!process_directives)
2375                                                 return Token.NONE;
2376
2377                                         bool cont = true;
2378                                         if (tokens_seen || comments_seen) {
2379                                                error_details = "Preprocessor directives must appear as the first" +
2380                                                " non-whitespace character on a line.";
2381
2382                                                Report.Error (1040, Location, error_details);
2383
2384                                                return Token.ERROR;
2385                                        }
2386
2387                                 start_again:
2388
2389                                         cont = handle_preprocessing_directive (cont);
2390
2391                                         if (cont){
2392                                                 continue;
2393                                         }
2394
2395                                         bool skipping = false;
2396                                         for (;(c = getChar ()) != -1;){
2397                                                 if (c == '\n'){
2398                                                         skipping = false;
2399                                                 } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
2400                                                         continue;
2401                                                 else if (c != '#')
2402                                                         skipping = true;
2403                                                 if (c == '#' && !skipping)
2404                                                         goto start_again;
2405                                         }
2406                                         any_token_seen |= tokens_seen;
2407                                         tokens_seen = false;
2408                                         if (c == -1)
2409                                                 Report.Error (1027, Location, "Expected `#endif' directive");
2410                                         continue;
2411                                 }
2412
2413                                 if (c == '"')
2414                                         return consume_string (false);
2415
2416                                 if (c == '\''){
2417                                         c = getChar ();
2418                                         tokens_seen = true;
2419                                         if (c == '\''){
2420                                                 error_details = "Empty character literal";
2421                                                 Report.Error (1011, Location, error_details);
2422                                                 return Token.ERROR;
2423                                         }
2424                                         if (c == '\r' || c == '\n') {
2425                                                 Report.Error (1010, Location, "Newline in constant");
2426                                                 return Token.ERROR;
2427                                         }
2428                                         c = escape (c);
2429                                         if (c == -1)
2430                                                 return Token.ERROR;
2431                                         val = new System.Char ();
2432                                         val = (char) c;
2433                                         c = getChar ();
2434
2435                                         if (c != '\''){
2436                                                 error_details = "Too many characters in character literal";
2437                                                 Report.Error (1012, Location, error_details);
2438
2439                                                 // Try to recover, read until newline or next "'"
2440                                                 while ((c = getChar ()) != -1){
2441                                                         if (c == '\n'){
2442                                                                 break;
2443                                                         }
2444                                                         else if (c == '\'')
2445                                                                 break;
2446                                                 }
2447                                                 return Token.ERROR;
2448                                         }
2449                                         return Token.LITERAL_CHARACTER;
2450                                 }
2451
2452                                 if (c == '@') {
2453                                         c = getChar ();
2454                                         if (c == '"') {
2455                                                 tokens_seen = true;
2456                                                 return consume_string (true);
2457                                         } else if (is_identifier_start_character ((char) c)){
2458                                                 return consume_identifier (c, true);
2459                                         } else {
2460                                                 Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
2461                                         }
2462                                 }
2463
2464                                 error_details = ((char)c).ToString ();
2465
2466                                 return Token.ERROR;
2467                         }
2468
2469                         return Token.EOF;
2470                 }
2471
2472                 //
2473                 // Handles one line xml comment
2474                 //
2475                 private void handle_one_line_xml_comment ()
2476                 {
2477                         int c;
2478                         while ((c = peekChar ()) == ' ')
2479                                 getChar (); // skip heading whitespaces.
2480                         while ((c = peekChar ()) != -1 && c != '\n' && c != '\r') {
2481                                 xml_comment_buffer.Append ((char) getChar ());
2482                         }
2483                         if (c == '\r' || c == '\n')
2484                                 xml_comment_buffer.Append (Environment.NewLine);
2485                 }
2486
2487                 //
2488                 // Remove heading "*" in Javadoc-like xml documentation.
2489                 //
2490                 private void update_formatted_doc_comment (int current_comment_start)
2491                 {
2492                         int length = xml_comment_buffer.Length - current_comment_start;
2493                         string [] lines = xml_comment_buffer.ToString (
2494                                 current_comment_start,
2495                                 length).Replace ("\r", "").Split ('\n');
2496
2497                         // The first line starts with /**, thus it is not target
2498                         // for the format check.
2499                         for (int i = 1; i < lines.Length; i++) {
2500                                 string s = lines [i];
2501                                 int idx = s.IndexOf ('*');
2502                                 string head = null;
2503                                 if (idx < 0) {
2504                                         if (i < lines.Length - 1)
2505                                                 return;
2506                                         head = s;
2507                                 } else
2508                                         head = s.Substring (0, idx);
2509                                 foreach (char c in head)
2510                                         if (c != ' ')
2511                                                 return;
2512                                 lines [i] = s.Substring (idx + 1);
2513                         }
2514                         xml_comment_buffer.Remove (current_comment_start, length);
2515                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
2516                 }
2517
2518                 //
2519                 // Updates current comment location.
2520                 //
2521                 private void update_comment_location ()
2522                 {
2523                         if (current_comment_location.IsNull) {
2524                                 // "-2" is for heading "//" or "/*"
2525                                 current_comment_location =
2526                                         new Location (ref_line, col - 2);
2527                         }
2528                 }
2529
2530                 //
2531                 // Checks if there was incorrect doc comments and raise
2532                 // warnings.
2533                 //
2534                 public void check_incorrect_doc_comment ()
2535                 {
2536                         if (xml_comment_buffer.Length > 0)
2537                                 warn_incorrect_doc_comment ();
2538                 }
2539
2540                 //
2541                 // Raises a warning when tokenizer found incorrect doccomment
2542                 // markup.
2543                 //
2544                 private void warn_incorrect_doc_comment ()
2545                 {
2546                         if (doc_state != XmlCommentState.Error) {
2547                                 doc_state = XmlCommentState.Error;
2548                                 // in csc, it is 'XML comment is not placed on
2549                                 // a valid language element'. But that does not
2550                                 // make sense.
2551                                 Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
2552                         }
2553                 }
2554
2555                 //
2556                 // Consumes the saved xml comment lines (if any)
2557                 // as for current target member or type.
2558                 //
2559                 public string consume_doc_comment ()
2560                 {
2561                         if (xml_comment_buffer.Length > 0) {
2562                                 string ret = xml_comment_buffer.ToString ();
2563                                 reset_doc_comment ();
2564                                 return ret;
2565                         }
2566                         return null;
2567                 }
2568
2569                 void reset_doc_comment ()
2570                 {
2571                         xml_comment_buffer.Length = 0;
2572                         current_comment_location = Location.Null;
2573                 }
2574
2575                 public void cleanup ()
2576                 {
2577                         if (ifstack != null && ifstack.Count >= 1) {
2578                                 int state = (int) ifstack.Pop ();
2579                                 if ((state & REGION) != 0)
2580                                         Report.Error (1038, Location, "#endregion directive expected");
2581 //                              else
2582 //                                      Report.Error (1027, Location, "Expected `#endif' directive");
2583                         }
2584                 }
2585         }
2586
2587         //
2588         // Indicates whether it accepts XML documentation or not.
2589         //
2590         public enum XmlCommentState {
2591                 // comment is allowed in this state.
2592                 Allowed,
2593                 // comment is not allowed in this state.
2594                 NotAllowed,
2595                 // once comments appeared when it is NotAllowed, then the
2596                 // state is changed to it, until the state is changed to
2597                 // .Allowed.
2598                 Error
2599         }
2600 }
2601