mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //
   7 // Licensed under the terms of the GNU GPL
   8 //
   9 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
  10 // (C) 2004 Novell, Inc
  11 //
  12
  13 /*
  14  * TODO:
  15  *   Make sure we accept the proper Unicode ranges, per the spec.
  16  *   Report error 1032
  17 */
  18
  19 using System;
  20 using System.Text;
  21 using System.Collections;
  22 using System.IO;
  23 using System.Globalization;
  24 using System.Reflection;
  25
  26 namespace Mono.CSharp
  27 {
  28         /// <summary>
  29         ///    Tokenizer for C# source code.
  30         /// </summary>
  31
  32         public class Tokenizer : yyParser.yyInput
  33         {
  34                 SeekableStreamReader reader;
  35                 public SourceFile ref_name;
  36                 public SourceFile file_name;
  37                 public int ref_line = 1;
  38                 public int line = 1;
  39                 public int col = 1;
  40                 public int current_token;
  41                 bool handle_get_set = false;
  42                 bool handle_remove_add = false;
  43                 bool handle_assembly = false;
  44
  45                 //
  46                 // XML documentation buffer. The save point is used to divide
  47                 // comments on types and comments on members.
  48                 //
  49                 StringBuilder xml_comment_buffer;
  50
  51                 //
  52                 // See comment on XmlCommentState enumeration.
  53                 //
  54                 XmlCommentState xmlDocState = XmlCommentState.Allowed;
  55
  56                 //
  57                 // Whether tokens have been seen on this line
  58                 //
  59                 bool tokens_seen = false;
  60
  61                 //
  62                 // Whether a token has been seen on the file
  63                 // This is needed because `define' is not allowed to be used
  64                 // after a token has been seen.
  65                 //
  66                 bool any_token_seen = false;
  67                 static Hashtable tokenValues;
  68
  69                 private static Hashtable TokenValueName
  70                 {
  71                         get {
  72                                 if (tokenValues == null)
  73                                         tokenValues = GetTokenValueNameHash ();
  74
  75                                 return tokenValues;
  76                         }
  77                 }
  78
  79                 private static Hashtable GetTokenValueNameHash ()
  80                 {
  81                         Type t = typeof (Token);
  82                         FieldInfo [] fields = t.GetFields ();
  83                         Hashtable hash = new Hashtable ();
  84                         foreach (FieldInfo field in fields) {
  85                                 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))
  86                                         hash.Add (field.GetValue (null), field.Name);
  87                         }
  88                         return hash;
  89                 }
  90
  91                 //
  92                 // Returns a verbose representation of the current location
  93                 //
  94                 public string location {
  95                         get {
  96                                 string det;
  97
  98                                 if (current_token == Token.ERROR)
  99                                         det = "detail: " + error_details;
 100                                 else
 101                                         det = "";
 102
 103                                 // return "Line:     "+line+" Col: "+col + "\n" +
 104                                 //       "VirtLine: "+ref_line +
 105                                 //       " Token: "+current_token + " " + det;
 106                                 string current_token_name = TokenValueName [current_token] as string;
 107                                 if (current_token_name == null)
 108                                         current_token_name = current_token.ToString ();
 109
 110                                 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,
 111                                                                                        ref_line,
 112                                                                                        col,
 113                                                                                        current_token_name,
 114                                                                                        det);
 115                         }
 116                 }
 117
 118                 public bool PropertyParsing {
 119                         get {
 120                                 return handle_get_set;
 121                         }
 122
 123                         set {
 124                                 handle_get_set = value;
 125                         }
 126                 }
 127
 128                 public bool AssemblyTargetParsing {
 129                         get {
 130                                 return handle_assembly;
 131                         }
 132
 133                         set {
 134                                 handle_assembly = value;
 135                         }
 136                 }
 137
 138                 public bool EventParsing {
 139                         get {
 140                                 return handle_remove_add;
 141                         }
 142
 143                         set {
 144                                 handle_remove_add = value;
 145                         }
 146                 }
 147
 148                 public XmlCommentState doc_state {
 149                         get { return xmlDocState; }
 150                         set {
 151                                 if (value == XmlCommentState.Allowed) {
 152                                         check_incorrect_doc_comment ();
 153                                         consume_doc_comment ();
 154                                 }
 155                                 xmlDocState = value;
 156                         }
 157                 }
 158
 159
 160                 //
 161                 // Class variables
 162                 //
 163                 static CharArrayHashtable[] keywords;
 164                 static NumberStyles styles;
 165                 static NumberFormatInfo csharp_format_info;
 166
 167                 //
 168                 // Values for the associated token returned
 169                 //
 170                 int putback_char;
 171                 Object val;
 172
 173                 //
 174                 // Pre-processor
 175                 //
 176                 Hashtable defines;
 177
 178                 const int TAKING        = 1;
 179                 const int TAKEN_BEFORE  = 2;
 180                 const int ELSE_SEEN     = 4;
 181                 const int PARENT_TAKING = 8;
 182                 const int REGION        = 16;
 183
 184                 //
 185                 // pre-processor if stack state:
 186                 //
 187                 Stack ifstack;
 188
 189                 static System.Text.StringBuilder string_builder;
 190
 191                 const int max_id_size = 512;
 192                 static char [] id_builder = new char [max_id_size];
 193
 194                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 195
 196                 const int max_number_size = 128;
 197                 static char [] number_builder = new char [max_number_size];
 198                 static int number_pos;
 199
 200                 //
 201                 // Details about the error encoutered by the tokenizer
 202                 //
 203                 string error_details;
 204
 205                 public string error {
 206                         get {
 207                                 return error_details;
 208                         }
 209                 }
 210
 211                 public int Line {
 212                         get {
 213                                 return ref_line;
 214                         }
 215                 }
 216
 217                 public int Col {
 218                         get {
 219                                 return col;
 220                         }
 221                 }
 222
 223                 static void AddKeyword (string kw, int token) {
 224                         if (keywords [kw.Length] == null) {
 225                                 keywords [kw.Length] = new CharArrayHashtable (kw.Length);
 226                         }
 227                         keywords [kw.Length] [kw.ToCharArray ()] = token;
 228                 }
 229
 230                 static void InitTokens ()
 231                 {
 232                         keywords = new CharArrayHashtable [64];
 233
 234                         AddKeyword ("__arglist", Token.ARGLIST);
 235                         AddKeyword ("abstract", Token.ABSTRACT);
 236                         AddKeyword ("as", Token.AS);
 237                         AddKeyword ("add", Token.ADD);
 238                         AddKeyword ("assembly", Token.ASSEMBLY);
 239                         AddKeyword ("base", Token.BASE);
 240                         AddKeyword ("bool", Token.BOOL);
 241                         AddKeyword ("break", Token.BREAK);
 242                         AddKeyword ("byte", Token.BYTE);
 243                         AddKeyword ("case", Token.CASE);
 244                         AddKeyword ("catch", Token.CATCH);
 245                         AddKeyword ("char", Token.CHAR);
 246                         AddKeyword ("checked", Token.CHECKED);
 247                         AddKeyword ("class", Token.CLASS);
 248                         AddKeyword ("const", Token.CONST);
 249                         AddKeyword ("continue", Token.CONTINUE);
 250                         AddKeyword ("decimal", Token.DECIMAL);
 251                         AddKeyword ("default", Token.DEFAULT);
 252                         AddKeyword ("delegate", Token.DELEGATE);
 253                         AddKeyword ("do", Token.DO);
 254                         AddKeyword ("double", Token.DOUBLE);
 255                         AddKeyword ("else", Token.ELSE);
 256                         AddKeyword ("enum", Token.ENUM);
 257                         AddKeyword ("event", Token.EVENT);
 258                         AddKeyword ("explicit", Token.EXPLICIT);
 259                         AddKeyword ("extern", Token.EXTERN);
 260                         AddKeyword ("false", Token.FALSE);
 261                         AddKeyword ("finally", Token.FINALLY);
 262                         AddKeyword ("fixed", Token.FIXED);
 263                         AddKeyword ("float", Token.FLOAT);
 264                         AddKeyword ("for", Token.FOR);
 265                         AddKeyword ("foreach", Token.FOREACH);
 266                         AddKeyword ("goto", Token.GOTO);
 267                         AddKeyword ("get", Token.GET);
 268                         AddKeyword ("if", Token.IF);
 269                         AddKeyword ("implicit", Token.IMPLICIT);
 270                         AddKeyword ("in", Token.IN);
 271                         AddKeyword ("int", Token.INT);
 272                         AddKeyword ("interface", Token.INTERFACE);
 273                         AddKeyword ("internal", Token.INTERNAL);
 274                         AddKeyword ("is", Token.IS);
 275                         AddKeyword ("lock", Token.LOCK);
 276                         AddKeyword ("long", Token.LONG);
 277                         AddKeyword ("namespace", Token.NAMESPACE);
 278                         AddKeyword ("new", Token.NEW);
 279                         AddKeyword ("null", Token.NULL);
 280                         AddKeyword ("object", Token.OBJECT);
 281                         AddKeyword ("operator", Token.OPERATOR);
 282                         AddKeyword ("out", Token.OUT);
 283                         AddKeyword ("override", Token.OVERRIDE);
 284                         AddKeyword ("params", Token.PARAMS);
 285                         AddKeyword ("private", Token.PRIVATE);
 286                         AddKeyword ("protected", Token.PROTECTED);
 287                         AddKeyword ("public", Token.PUBLIC);
 288                         AddKeyword ("readonly", Token.READONLY);
 289                         AddKeyword ("ref", Token.REF);
 290                         AddKeyword ("remove", Token.REMOVE);
 291                         AddKeyword ("return", Token.RETURN);
 292                         AddKeyword ("sbyte", Token.SBYTE);
 293                         AddKeyword ("sealed", Token.SEALED);
 294                         AddKeyword ("set", Token.SET);
 295                         AddKeyword ("short", Token.SHORT);
 296                         AddKeyword ("sizeof", Token.SIZEOF);
 297                         AddKeyword ("stackalloc", Token.STACKALLOC);
 298                         AddKeyword ("static", Token.STATIC);
 299                         AddKeyword ("string", Token.STRING);
 300                         AddKeyword ("struct", Token.STRUCT);
 301                         AddKeyword ("switch", Token.SWITCH);
 302                         AddKeyword ("this", Token.THIS);
 303                         AddKeyword ("throw", Token.THROW);
 304                         AddKeyword ("true", Token.TRUE);
 305                         AddKeyword ("try", Token.TRY);
 306                         AddKeyword ("typeof", Token.TYPEOF);
 307                         AddKeyword ("uint", Token.UINT);
 308                         AddKeyword ("ulong", Token.ULONG);
 309                         AddKeyword ("unchecked", Token.UNCHECKED);
 310                         AddKeyword ("unsafe", Token.UNSAFE);
 311                         AddKeyword ("ushort", Token.USHORT);
 312                         AddKeyword ("using", Token.USING);
 313                         AddKeyword ("virtual", Token.VIRTUAL);
 314                         AddKeyword ("void", Token.VOID);
 315                         AddKeyword ("volatile", Token.VOLATILE);
 316                         AddKeyword ("while", Token.WHILE);
 317                         AddKeyword ("partial", Token.PARTIAL);
 318                 }
 319
 320                 //
 321                 // Class initializer
 322                 //
 323                 static Tokenizer ()
 324                 {
 325                         InitTokens ();
 326                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 327                         styles = NumberStyles.Float;
 328
 329                         string_builder = new System.Text.StringBuilder ();
 330                 }
 331
 332                 int GetKeyword (char[] id, int id_len)
 333                 {
 334                         /*
 335                          * Keywords are stored in an array of hashtables grouped by their
 336                          * length.
 337                          */
 338
 339                         if ((id_len >= keywords.Length) || (keywords [id_len] == null))
 340                                 return -1;
 341                         object o = keywords [id_len] [id];
 342
 343                         if (o == null)
 344                                 return -1;
 345
 346                         int res = (int) o;
 347
 348                         if (handle_get_set == false && (res == Token.GET || res == Token.SET))
 349                                 return -1;
 350                         if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
 351                                 return -1;
 352                         if (handle_assembly == false && res == Token.ASSEMBLY)
 353                                 return -1;
 354
 355                         return res;
 356
 357                 }
 358
 359                 public Location Location {
 360                         get {
 361                                 return new Location (ref_line);
 362                         }
 363                 }
 364
 365                 void define (string def)
 366                 {
 367                         if (!RootContext.AllDefines.Contains (def)){
 368                                 RootContext.AllDefines [def] = true;
 369                         }
 370                         if (defines.Contains (def))
 371                                 return;
 372                         defines [def] = true;
 373                 }
 374
 375                 public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)
 376                 {
 377                         this.ref_name = file;
 378                         this.file_name = file;
 379                         reader = input;
 380
 381                         putback_char = -1;
 382
 383                         if (defs != null){
 384                                 defines = new Hashtable ();
 385                                 foreach (string def in defs)
 386                                         define (def);
 387                         }
 388
 389                         xml_comment_buffer = new StringBuilder ();
 390
 391                         //
 392                         // FIXME: This could be `Location.Push' but we have to
 393                         // find out why the MS compiler allows this
 394                         //
 395                         Mono.CSharp.Location.Push (file);
 396                 }
 397
 398                 public static void Cleanup () {
 399                         identifiers = null;
 400                 }
 401
 402                 static bool is_identifier_start_character (char c)
 403                 {
 404                         return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c);
 405                 }
 406
 407                 static bool is_identifier_part_character (char c)
 408                 {
 409                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
 410                 }
 411
 412                 public static bool IsValidIdentifier (string s)
 413                 {
 414                         if (s == null || s.Length == 0)
 415                                 return false;
 416
 417                         if (!is_identifier_start_character (s [0]))
 418                                 return false;
 419
 420                         for (int i = 1; i < s.Length; i ++)
 421                                 if (! is_identifier_part_character (s [i]))
 422                                         return false;
 423
 424                         return true;
 425                 }
 426
 427                 int is_punct (char c, ref bool doread)
 428                 {
 429                         int d;
 430                         int t;
 431
 432                         doread = false;
 433
 434                         switch (c){
 435                         case '{':
 436                                 return Token.OPEN_BRACE;
 437                         case '}':
 438                                 return Token.CLOSE_BRACE;
 439                         case '[':
 440                                 // To block doccomment inside attribute declaration.
 441                                 if (doc_state == XmlCommentState.Allowed)
 442                                         doc_state = XmlCommentState.NotAllowed;
 443                                 return Token.OPEN_BRACKET;
 444                         case ']':
 445                                 return Token.CLOSE_BRACKET;
 446                         case '(':
 447                                 return Token.OPEN_PARENS;
 448                         case ')': {
 449                                 if (deambiguate_close_parens == 0)
 450                                         return Token.CLOSE_PARENS;
 451
 452                                 --deambiguate_close_parens;
 453
 454                                 // Save current position and parse next token.
 455                                 int old = reader.Position;
 456                                 int new_token = token ();
 457                                 reader.Position = old;
 458                                 putback_char = -1;
 459
 460                                 if (new_token == Token.OPEN_PARENS)
 461                                         return Token.CLOSE_PARENS_OPEN_PARENS;
 462                                 else if (new_token == Token.MINUS)
 463                                         return Token.CLOSE_PARENS_MINUS;
 464                                 else if (IsCastToken (new_token))
 465                                         return Token.CLOSE_PARENS_CAST;
 466                                 else
 467                                         return Token.CLOSE_PARENS_NO_CAST;
 468                         }
 469
 470                         case ',':
 471                                 return Token.COMMA;
 472                         case ':':
 473                                 return Token.COLON;
 474                         case ';':
 475                                 return Token.SEMICOLON;
 476                         case '~':
 477                                 return Token.TILDE;
 478                         case '?':
 479                                 return Token.INTERR;
 480                         }
 481
 482                         d = peekChar ();
 483                         if (c == '+'){
 484
 485                                 if (d == '+')
 486                                         t = Token.OP_INC;
 487                                 else if (d == '=')
 488                                         t = Token.OP_ADD_ASSIGN;
 489                                 else
 490                                         return Token.PLUS;
 491                                 doread = true;
 492                                 return t;
 493                         }
 494                         if (c == '-'){
 495                                 if (d == '-')
 496                                         t = Token.OP_DEC;
 497                                 else if (d == '=')
 498                                         t = Token.OP_SUB_ASSIGN;
 499                                 else if (d == '>')
 500                                         t = Token.OP_PTR;
 501                                 else
 502                                         return Token.MINUS;
 503                                 doread = true;
 504                                 return t;
 505                         }
 506
 507                         if (c == '!'){
 508                                 if (d == '='){
 509                                         doread = true;
 510                                         return Token.OP_NE;
 511                                 }
 512                                 return Token.BANG;
 513                         }
 514
 515                         if (c == '='){
 516                                 if (d == '='){
 517                                         doread = true;
 518                                         return Token.OP_EQ;
 519                                 }
 520                                 return Token.ASSIGN;
 521                         }
 522
 523                         if (c == '&'){
 524                                 if (d == '&'){
 525                                         doread = true;
 526                                         return Token.OP_AND;
 527                                 } else if (d == '='){
 528                                         doread = true;
 529                                         return Token.OP_AND_ASSIGN;
 530                                 }
 531                                 return Token.BITWISE_AND;
 532                         }
 533
 534                         if (c == '|'){
 535                                 if (d == '|'){
 536                                         doread = true;
 537                                         return Token.OP_OR;
 538                                 } else if (d == '='){
 539                                         doread = true;
 540                                         return Token.OP_OR_ASSIGN;
 541                                 }
 542                                 return Token.BITWISE_OR;
 543                         }
 544
 545                         if (c == '*'){
 546                                 if (d == '='){
 547                                         doread = true;
 548                                         return Token.OP_MULT_ASSIGN;
 549                                 }
 550                                 return Token.STAR;
 551                         }
 552
 553                         if (c == '/'){
 554                                 if (d == '='){
 555                                         doread = true;
 556                                         return Token.OP_DIV_ASSIGN;
 557                                 }
 558                                 return Token.DIV;
 559                         }
 560
 561                         if (c == '%'){
 562                                 if (d == '='){
 563                                         doread = true;
 564                                         return Token.OP_MOD_ASSIGN;
 565                                 }
 566                                 return Token.PERCENT;
 567                         }
 568
 569                         if (c == '^'){
 570                                 if (d == '='){
 571                                         doread = true;
 572                                         return Token.OP_XOR_ASSIGN;
 573                                 }
 574                                 return Token.CARRET;
 575                         }
 576
 577                         if (c == '<'){
 578                                 if (d == '<'){
 579                                         getChar ();
 580                                         d = peekChar ();
 581
 582                                         if (d == '='){
 583                                                 doread = true;
 584                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 585                                         }
 586                                         return Token.OP_SHIFT_LEFT;
 587                                 } else if (d == '='){
 588                                         doread = true;
 589                                         return Token.OP_LE;
 590                                 }
 591                                 return Token.OP_LT;
 592                         }
 593
 594                         if (c == '>'){
 595                                 if (d == '>'){
 596                                         getChar ();
 597                                         d = peekChar ();
 598
 599                                         if (d == '='){
 600                                                 doread = true;
 601                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 602                                         }
 603                                         return Token.OP_SHIFT_RIGHT;
 604                                 } else if (d == '='){
 605                                         doread = true;
 606                                         return Token.OP_GE;
 607                                 }
 608                                 return Token.OP_GT;
 609                         }
 610                         return Token.ERROR;
 611                 }
 612
 613                 int deambiguate_close_parens = 0;
 614
 615                 public void Deambiguate_CloseParens ()
 616                 {
 617                         putback (')');
 618                         deambiguate_close_parens++;
 619                 }
 620
 621                 void Error_NumericConstantTooLong ()
 622                 {
 623                         Report.Error (1021, Location, "Numeric constant too long");
 624                 }
 625
 626                 bool decimal_digits (int c)
 627                 {
 628                         int d;
 629                         bool seen_digits = false;
 630
 631                         if (c != -1){
 632                                 if (number_pos == max_number_size)
 633                                         Error_NumericConstantTooLong ();
 634                                 number_builder [number_pos++] = (char) c;
 635                         }
 636
 637                         //
 638                         // We use peekChar2, because decimal_digits needs to do a
 639                         // 2-character look-ahead (5.ToString for example).
 640                         //
 641                         while ((d = peekChar2 ()) != -1){
 642                                 if (d >= '0' && d <= '9'){
 643                                         if (number_pos == max_number_size)
 644                                                 Error_NumericConstantTooLong ();
 645                                         number_builder [number_pos++] = (char) d;
 646                                         getChar ();
 647                                         seen_digits = true;
 648                                 } else
 649                                         break;
 650                         }
 651
 652                         return seen_digits;
 653                 }
 654
 655                 bool is_hex (int e)
 656                 {
 657                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
 658                 }
 659
 660                 void hex_digits (int c)
 661                 {
 662                         if (c != -1)
 663                                 number_builder [number_pos++] = (char) c;
 664
 665                 }
 666
 667                 int real_type_suffix (int c)
 668                 {
 669                         int t;
 670
 671                         switch (c){
 672                         case 'F': case 'f':
 673                                 t =  Token.LITERAL_FLOAT;
 674                                 break;
 675                         case 'D': case 'd':
 676                                 t = Token.LITERAL_DOUBLE;
 677                                 break;
 678                         case 'M': case 'm':
 679                                  t= Token.LITERAL_DECIMAL;
 680                                 break;
 681                         default:
 682                                 return Token.NONE;
 683                         }
 684                         return t;
 685                 }
 686
 687                 int integer_type_suffix (ulong ul, int c)
 688                 {
 689                         bool is_unsigned = false;
 690                         bool is_long = false;
 691
 692                         if (c != -1){
 693                                 bool scanning = true;
 694                                 do {
 695                                         switch (c){
 696                                         case 'U': case 'u':
 697                                                 if (is_unsigned)
 698                                                         scanning = false;
 699                                                 is_unsigned = true;
 700                                                 getChar ();
 701                                                 break;
 702
 703                                         case 'l':
 704                                                 if (!is_unsigned && (RootContext.WarningLevel >= 4)){
 705                                                         //
 706                                                         // if we have not seen anything in between
 707                                                         // report this error
 708                                                         //
 709                                                         Report.Warning (78, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
 710                                                 }
 711                                                 goto case 'L';
 712
 713                                         case 'L':
 714                                                 if (is_long)
 715                                                         scanning = false;
 716                                                 is_long = true;
 717                                                 getChar ();
 718                                                 break;
 719
 720                                         default:
 721                                                 scanning = false;
 722                                                 break;
 723                                         }
 724                                         c = peekChar ();
 725                                 } while (scanning);
 726                         }
 727
 728                         if (is_long && is_unsigned){
 729                                 val = ul;
 730                                 return Token.LITERAL_INTEGER;
 731                         } else if (is_unsigned){
 732                                 // uint if possible, or ulong else.
 733
 734                                 if ((ul & 0xffffffff00000000) == 0)
 735                                         val = (uint) ul;
 736                                 else
 737                                         val = ul;
 738                         } else if (is_long){
 739                                 // long if possible, ulong otherwise
 740                                 if ((ul & 0x8000000000000000) != 0)
 741                                         val = ul;
 742                                 else
 743                                         val = (long) ul;
 744                         } else {
 745                                 // int, uint, long or ulong in that order
 746                                 if ((ul & 0xffffffff00000000) == 0){
 747                                         uint ui = (uint) ul;
 748
 749                                         if ((ui & 0x80000000) != 0)
 750                                                 val = ui;
 751                                         else
 752                                                 val = (int) ui;
 753                                 } else {
 754                                         if ((ul & 0x8000000000000000) != 0)
 755                                                 val = ul;
 756                                         else
 757                                                 val = (long) ul;
 758                                 }
 759                         }
 760                         return Token.LITERAL_INTEGER;
 761                 }
 762
 763                 //
 764                 // given `c' as the next char in the input decide whether
 765                 // we need to convert to a special type, and then choose
 766                 // the best representation for the integer
 767                 //
 768                 int adjust_int (int c)
 769                 {
 770                         try {
 771                                 if (number_pos > 9){
 772                                         ulong ul = (uint) (number_builder [0] - '0');
 773
 774                                         for (int i = 1; i < number_pos; i++){
 775                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
 776                                         }
 777                                         return integer_type_suffix (ul, c);
 778                                 } else {
 779                                         uint ui = (uint) (number_builder [0] - '0');
 780
 781                                         for (int i = 1; i < number_pos; i++){
 782                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
 783                                         }
 784                                         return integer_type_suffix (ui, c);
 785                                 }
 786                         } catch (OverflowException) {
 787                                 error_details = "Integral constant is too large";
 788                                 Report.Error (1021, Location, error_details);
 789                                 val = 0ul;
 790                                 return Token.LITERAL_INTEGER;
 791                         }
 792                 }
 793
 794                 int adjust_real (int t)
 795                 {
 796                         string s = new String (number_builder, 0, number_pos);
 797
 798                         switch (t){
 799                         case Token.LITERAL_DECIMAL:
 800                                 try {
 801                                         val = System.Decimal.Parse (s, styles, csharp_format_info);
 802                                 } catch (OverflowException) {
 803                                         val = 0m;
 804                                         error_details = "Floating-point constant is outside the range of the type 'decimal'";
 805                                         Report.Error (594, Location, error_details);
 806                                 }
 807                                 break;
 808                         case Token.LITERAL_FLOAT:
 809                                 try {
 810                                         val = (float) System.Double.Parse (s, styles, csharp_format_info);
 811                                 } catch (OverflowException) {
 812                                         val = 0.0f;
 813                                         error_details = "Floating-point constant is outside the range of the type 'float'";
 814                                         Report.Error (594, Location, error_details);
 815                                 }
 816                                 break;
 817
 818                         case Token.LITERAL_DOUBLE:
 819                         case Token.NONE:
 820                                 t = Token.LITERAL_DOUBLE;
 821                                 try {
 822                                         val = System.Double.Parse (s, styles, csharp_format_info);
 823                                 } catch (OverflowException) {
 824                                         val = 0.0;
 825                                         error_details = "Floating-point constant is outside the range of the type 'double'";
 826                                         Report.Error (594, Location, error_details);
 827                                 }
 828                                 break;
 829                         }
 830                         return t;
 831                 }
 832
 833                 int handle_hex ()
 834                 {
 835                         int d;
 836                         ulong ul;
 837
 838                         getChar ();
 839                         while ((d = peekChar ()) != -1){
 840                                 if (is_hex (d)){
 841                                         number_builder [number_pos++] = (char) d;
 842                                         getChar ();
 843                                 } else
 844                                         break;
 845                         }
 846
 847                         string s = new String (number_builder, 0, number_pos);
 848                         try {
 849                                 if (number_pos <= 8)
 850                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
 851                                 else
 852                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
 853                         } catch (OverflowException){
 854                                 error_details = "Integral constant is too large";
 855                                 Report.Error (1021, Location, error_details);
 856                                 val = 0ul;
 857                                 return Token.LITERAL_INTEGER;
 858                         }
 859                         catch (FormatException) {
 860                                 Report.Error (1013, Location, "Invalid number");
 861                                 val = 0ul;
 862                                 return Token.LITERAL_INTEGER;
 863                         }
 864
 865                         return integer_type_suffix (ul, peekChar ());
 866                 }
 867
 868                 //
 869                 // Invoked if we know we have .digits or digits
 870                 //
 871                 int is_number (int c)
 872                 {
 873                         bool is_real = false;
 874                         int type;
 875
 876                         number_pos = 0;
 877
 878                         if (c >= '0' && c <= '9'){
 879                                 if (c == '0'){
 880                                         int peek = peekChar ();
 881
 882                                         if (peek == 'x' || peek == 'X')
 883                                                 return handle_hex ();
 884                                 }
 885                                 decimal_digits (c);
 886                                 c = getChar ();
 887                         }
 888
 889                         //
 890                         // We need to handle the case of
 891                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
 892                         //
 893                         if (c == '.'){
 894                                 if (decimal_digits ('.')){
 895                                         is_real = true;
 896                                         c = getChar ();
 897                                 } else {
 898                                         putback ('.');
 899                                         number_pos--;
 900                                         return adjust_int (-1);
 901                                 }
 902                         }
 903
 904                         if (c == 'e' || c == 'E'){
 905                                 is_real = true;
 906                                 if (number_pos == max_number_size)
 907                                         Error_NumericConstantTooLong ();
 908                                 number_builder [number_pos++] = 'e';
 909                                 c = getChar ();
 910
 911                                 if (c == '+'){
 912                                         if (number_pos == max_number_size)
 913                                                 Error_NumericConstantTooLong ();
 914                                         number_builder [number_pos++] = '+';
 915                                         c = -1;
 916                                 } else if (c == '-') {
 917                                         if (number_pos == max_number_size)
 918                                                 Error_NumericConstantTooLong ();
 919                                         number_builder [number_pos++] = '-';
 920                                         c = -1;
 921                                 } else {
 922                                         if (number_pos == max_number_size)
 923                                                 Error_NumericConstantTooLong ();
 924                                         number_builder [number_pos++] = '+';
 925                                 }
 926
 927                                 decimal_digits (c);
 928                                 c = getChar ();
 929                         }
 930
 931                         type = real_type_suffix (c);
 932                         if (type == Token.NONE && !is_real){
 933                                 putback (c);
 934                                 return adjust_int (c);
 935                         } else
 936                                 is_real = true;
 937
 938                         if (type == Token.NONE){
 939                                 putback (c);
 940                         }
 941
 942                         if (is_real)
 943                                 return adjust_real (type);
 944
 945                         Console.WriteLine ("This should not be reached");
 946                         throw new Exception ("Is Number should never reach this point");
 947                 }
 948
 949                 //
 950                 // Accepts exactly count (4 or 8) hex, no more no less
 951                 //
 952                 int getHex (int count, out bool error)
 953                 {
 954                         int i;
 955                         int total = 0;
 956                         int c;
 957                         int top = count != -1 ? count : 4;
 958
 959                         getChar ();
 960                         error = false;
 961                         for (i = 0; i < top; i++){
 962                                 c = getChar ();
 963
 964                                 if (c >= '0' && c <= '9')
 965                                         c = (int) c - (int) '0';
 966                                 else if (c >= 'A' && c <= 'F')
 967                                         c = (int) c - (int) 'A' + 10;
 968                                 else if (c >= 'a' && c <= 'f')
 969                                         c = (int) c - (int) 'a' + 10;
 970                                 else {
 971                                         error = true;
 972                                         return 0;
 973                                 }
 974
 975                                 total = (total * 16) + c;
 976                                 if (count == -1){
 977                                         int p = peekChar ();
 978                                         if (p == -1)
 979                                                 break;
 980                                         if (!is_hex ((char)p))
 981                                                 break;
 982                                 }
 983                         }
 984                         return total;
 985                 }
 986
 987                 int escape (int c)
 988                 {
 989                         bool error;
 990                         int d;
 991                         int v;
 992
 993                         d = peekChar ();
 994                         if (c != '\\')
 995                                 return c;
 996
 997                         switch (d){
 998                         case 'a':
 999                                 v = '\a'; break;
1000                         case 'b':
1001                                 v = '\b'; break;
1002                         case 'n':
1003                                 v = '\n'; break;
1004                         case 't':
1005                                 v = '\t'; break;
1006                         case 'v':
1007                                 v = '\v'; break;
1008                         case 'r':
1009                                 v = '\r'; break;
1010                         case '\\':
1011                                 v = '\\'; break;
1012                         case 'f':
1013                                 v = '\f'; break;
1014                         case '0':
1015                                 v = 0; break;
1016                         case '"':
1017                                 v = '"'; break;
1018                         case '\'':
1019                                 v = '\''; break;
1020                         case 'x':
1021                                 v = getHex (-1, out error);
1022                                 if (error)
1023                                         goto default;
1024                                 return v;
1025                         case 'u':
1026                                 v = getHex (4, out error);
1027                                 if (error)
1028                                         goto default;
1029                                 return v;
1030                         case 'U':
1031                                 v = getHex (8, out error);
1032                                 if (error)
1033                                         goto default;
1034                                 return v;
1035                         default:
1036                                 Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);
1037                                 return d;
1038                         }
1039                         getChar ();
1040                         return v;
1041                 }
1042
1043                 int getChar ()
1044                 {
1045                         if (putback_char != -1){
1046                                 int x = putback_char;
1047                                 putback_char = -1;
1048
1049                                 return x;
1050                         }
1051                         return reader.Read ();
1052                 }
1053
1054                 int peekChar ()
1055                 {
1056                         if (putback_char != -1)
1057                                 return putback_char;
1058                         putback_char = reader.Read ();
1059                         return putback_char;
1060                 }
1061
1062                 int peekChar2 ()
1063                 {
1064                         if (putback_char != -1)
1065                                 return putback_char;
1066                         return reader.Peek ();
1067                 }
1068
1069                 void putback (int c)
1070                 {
1071                         if (putback_char != -1){
1072                                 Console.WriteLine ("Col: " + col);
1073                                 Console.WriteLine ("Row: " + line);
1074                                 Console.WriteLine ("Name: " + ref_name.Name);
1075                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1076                                 throw new Exception ("This should not happen putback on putback");
1077                         }
1078                         putback_char = c;
1079                 }
1080
1081                 public bool advance ()
1082                 {
1083                         return peekChar () != -1;
1084                 }
1085
1086                 public Object Value {
1087                         get {
1088                                 return val;
1089                         }
1090                 }
1091
1092                 public Object value ()
1093                 {
1094                         return val;
1095                 }
1096
1097                 bool IsCastToken (int token)
1098                 {
1099                         switch (token) {
1100                         case Token.BANG:
1101                         case Token.TILDE:
1102                         case Token.IDENTIFIER:
1103                         case Token.LITERAL_INTEGER:
1104                         case Token.LITERAL_FLOAT:
1105                         case Token.LITERAL_DOUBLE:
1106                         case Token.LITERAL_DECIMAL:
1107                         case Token.LITERAL_CHARACTER:
1108                         case Token.LITERAL_STRING:
1109                         case Token.BASE:
1110                         case Token.CHECKED:
1111                         case Token.FALSE:
1112                         case Token.FIXED:
1113                         case Token.NEW:
1114                         case Token.NULL:
1115                         case Token.SIZEOF:
1116                         case Token.THIS:
1117                         case Token.THROW:
1118                         case Token.TRUE:
1119                         case Token.TYPEOF:
1120                         case Token.UNCHECKED:
1121                         case Token.UNSAFE:
1122
1123                                 //
1124                                 // These can be part of a member access
1125                                 //
1126                         case Token.INT:
1127                         case Token.UINT:
1128                         case Token.SHORT:
1129                         case Token.USHORT:
1130                         case Token.LONG:
1131                         case Token.ULONG:
1132                         case Token.DOUBLE:
1133                         case Token.FLOAT:
1134                         case Token.CHAR:
1135                                 return true;
1136
1137                         default:
1138                                 return false;
1139                         }
1140                 }
1141
1142                 public int token ()
1143                 {
1144                         current_token = xtoken ();
1145                         return current_token;
1146                 }
1147
1148                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1149
1150                 void get_cmd_arg (out string cmd, out string arg)
1151                 {
1152                         int c;
1153
1154                         tokens_seen = false;
1155                         arg = "";
1156                         static_cmd_arg.Length = 0;
1157
1158                         // skip over white space
1159                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1160                                 ;
1161
1162
1163                         while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
1164                                 if (is_identifier_part_character ((char) c)){
1165                                         static_cmd_arg.Append ((char) c);
1166                                         c = getChar ();
1167                                 } else {
1168                                         putback (c);
1169                                         break;
1170                                 }
1171                         }
1172
1173                         cmd = static_cmd_arg.ToString ();
1174
1175                         if (c == '\n'){
1176                                 line++;
1177                                 ref_line++;
1178                                 return;
1179                         } else if (c == '\r')
1180                                 col = 0;
1181
1182                         // skip over white space
1183                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1184                                 ;
1185
1186                         if (c == '\n'){
1187                                 line++;
1188                                 ref_line++;
1189                                 return;
1190                         } else if (c == '\r'){
1191                                 col = 0;
1192                                 return;
1193                         }
1194
1195                         static_cmd_arg.Length = 0;
1196                         static_cmd_arg.Append ((char) c);
1197
1198                         while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
1199                                 static_cmd_arg.Append ((char) c);
1200                         }
1201
1202                         if (c == '\n'){
1203                                 line++;
1204                                 ref_line++;
1205                         } else if (c == '\r')
1206                                 col = 0;
1207                         arg = static_cmd_arg.ToString ().Trim ();
1208                 }
1209
1210                 //
1211                 // Handles the #line directive
1212                 //
1213                 bool PreProcessLine (string arg)
1214                 {
1215                         if (arg == "")
1216                                 return false;
1217
1218                         if (arg == "default"){
1219                                 ref_line = line;
1220                                 ref_name = file_name;
1221                                 Location.Push (ref_name);
1222                                 return true;
1223                         } else if (arg == "hidden"){
1224                                 //
1225                                 // We ignore #line hidden
1226                                 //
1227                                 return true;
1228                         }
1229
1230                         try {
1231                                 int pos;
1232
1233                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1234                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1235                                         pos++;
1236
1237                                         char [] quotes = { '\"' };
1238
1239                                         string name = arg.Substring (pos). Trim (quotes);
1240                                         ref_name = Location.LookupFile (name);
1241                                         file_name.HasLineDirective = true;
1242                                         ref_name.HasLineDirective = true;
1243                                         Location.Push (ref_name);
1244                                 } else {
1245                                         ref_line = System.Int32.Parse (arg);
1246                                 }
1247                         } catch {
1248                                 return false;
1249                         }
1250
1251                         return true;
1252                 }
1253
1254                 //
1255                 // Handles #define and #undef
1256                 //
1257                 void PreProcessDefinition (bool is_define, string arg)
1258                 {
1259                         if (arg == "" || arg == "true" || arg == "false"){
1260                                 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
1261                                 return;
1262                         }
1263
1264                         char[] whitespace = { ' ', '\t' };
1265                         if (arg.IndexOfAny (whitespace) != -1){
1266                                 Report.Error (1025, Location, "Single-line comment or end-of-line expected");
1267                                 return;
1268                         }
1269
1270                         if (!is_identifier_start_character (arg [0]))
1271                                 Report.Error (1001, Location, "Identifier expected: " + arg);
1272
1273                         foreach (char c in arg.Substring (1)){
1274                                 if (!is_identifier_part_character (c)){
1275                                         Report.Error (1001, Location, "Identifier expected: " + arg);
1276                                         return;
1277                                 }
1278                         }
1279
1280                         if (is_define){
1281                                 if (defines == null)
1282                                         defines = new Hashtable ();
1283                                 define (arg);
1284                         } else {
1285                                 if (defines == null)
1286                                         return;
1287                                 if (defines.Contains (arg))
1288                                         defines.Remove (arg);
1289                         }
1290                 }
1291
1292                 /// <summary>
1293                 /// Handles #pragma directive
1294                 /// </summary>
1295                 void PreProcessPragma (string arg)
1296                 {
1297                         const string disable = "warning disable";
1298                         const string restore = "warning restore";
1299
1300                         if (arg == disable) {
1301                                 Report.RegisterWarningRegion (Location).WarningDisable (line);
1302                                 return;
1303                         }
1304
1305                         if (arg == restore) {
1306                                 Report.RegisterWarningRegion (Location).WarningEnable (line);
1307                                 return;
1308                         }
1309
1310                         if (arg.StartsWith (disable)) {
1311                                 int[] codes = ParseNumbers (arg.Substring (disable.Length));
1312                                 foreach (int code in codes) {
1313                                         if (code != 0)
1314                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1315                                 }
1316                                 return;
1317                         }
1318
1319                         if (arg.StartsWith (restore)) {
1320                                 int[] codes = ParseNumbers (arg.Substring (restore.Length));
1321                                 foreach (int code in codes) {
1322                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1323                                 }
1324                                 return;
1325                         }
1326
1327                         return;
1328                 }
1329
1330                 int[] ParseNumbers (string text)
1331                 {
1332                         string[] string_array = text.Split (',');
1333                         int[] values = new int [string_array.Length];
1334                         int index = 0;
1335                         foreach (string string_code in string_array) {
1336                                 try {
1337                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1338                                 }
1339                                 catch (FormatException) {
1340                                         Report.Warning (1692, Location, "Invalid number");
1341                                 }
1342                         }
1343                         return values;
1344                 }
1345
1346                 bool eval_val (string s)
1347                 {
1348                         if (s == "true")
1349                                 return true;
1350                         if (s == "false")
1351                                 return false;
1352
1353                         if (defines == null)
1354                                 return false;
1355                         if (defines.Contains (s))
1356                                 return true;
1357
1358                         return false;
1359                 }
1360
1361                 bool pp_primary (ref string s)
1362                 {
1363                         s = s.Trim ();
1364                         int len = s.Length;
1365
1366                         if (len > 0){
1367                                 char c = s [0];
1368
1369                                 if (c == '('){
1370                                         s = s.Substring (1);
1371                                         bool val = pp_expr (ref s);
1372                                         if (s.Length > 0 && s [0] == ')'){
1373                                                 s = s.Substring (1);
1374                                                 return val;
1375                                         }
1376                                         Error_InvalidDirective ();
1377                                         return false;
1378                                 }
1379
1380                                 if (is_identifier_start_character (c)){
1381                                         int j = 1;
1382
1383                                         while (j < len){
1384                                                 c = s [j];
1385
1386                                                 if (is_identifier_part_character (c)){
1387                                                         j++;
1388                                                         continue;
1389                                                 }
1390                                                 bool v = eval_val (s.Substring (0, j));
1391                                                 s = s.Substring (j);
1392                                                 return v;
1393                                         }
1394                                         bool vv = eval_val (s);
1395                                         s = "";
1396                                         return vv;
1397                                 }
1398                         }
1399                         Error_InvalidDirective ();
1400                         return false;
1401                 }
1402
1403                 bool pp_unary (ref string s)
1404                 {
1405                         s = s.Trim ();
1406                         int len = s.Length;
1407
1408                         if (len > 0){
1409                                 if (s [0] == '!'){
1410                                         if (len > 1 && s [1] == '='){
1411                                                 Error_InvalidDirective ();
1412                                                 return false;
1413                                         }
1414                                         s = s.Substring (1);
1415                                         return ! pp_primary (ref s);
1416                                 } else
1417                                         return pp_primary (ref s);
1418                         } else {
1419                                 Error_InvalidDirective ();
1420                                 return false;
1421                         }
1422                 }
1423
1424                 bool pp_eq (ref string s)
1425                 {
1426                         bool va = pp_unary (ref s);
1427
1428                         s = s.Trim ();
1429                         int len = s.Length;
1430                         if (len > 0){
1431                                 if (s [0] == '='){
1432                                         if (len > 2 && s [1] == '='){
1433                                                 s = s.Substring (2);
1434                                                 return va == pp_unary (ref s);
1435                                         } else {
1436                                                 Error_InvalidDirective ();
1437                                                 return false;
1438                                         }
1439                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
1440                                         s = s.Substring (2);
1441
1442                                         return va != pp_unary (ref s);
1443
1444                                 }
1445                         }
1446
1447                         return va;
1448
1449                 }
1450
1451                 bool pp_and (ref string s)
1452                 {
1453                         bool va = pp_eq (ref s);
1454
1455                         s = s.Trim ();
1456                         int len = s.Length;
1457                         if (len > 0){
1458                                 if (s [0] == '&'){
1459                                         if (len > 2 && s [1] == '&'){
1460                                                 s = s.Substring (2);
1461                                                 return (va & pp_eq (ref s));
1462                                         } else {
1463                                                 Error_InvalidDirective ();
1464                                                 return false;
1465                                         }
1466                                 }
1467                         }
1468                         return va;
1469                 }
1470
1471                 //
1472                 // Evaluates an expression for `#if' or `#elif'
1473                 //
1474                 bool pp_expr (ref string s)
1475                 {
1476                         bool va = pp_and (ref s);
1477                         s = s.Trim ();
1478                         int len = s.Length;
1479                         if (len > 0){
1480                                 char c = s [0];
1481
1482                                 if (c == '|'){
1483                                         if (len > 2 && s [1] == '|'){
1484                                                 s = s.Substring (2);
1485                                                 return va | pp_expr (ref s);
1486                                         } else {
1487                                                 Error_InvalidDirective ();
1488                                                 return false;
1489                                         }
1490                                 }
1491                         }
1492
1493                         return va;
1494                 }
1495
1496                 bool eval (string s)
1497                 {
1498                         bool v = pp_expr (ref s);
1499                         s = s.Trim ();
1500                         if (s.Length != 0){
1501                                 Error_InvalidDirective ();
1502                                 return false;
1503                         }
1504
1505                         return v;
1506                 }
1507
1508                 void Error_InvalidDirective ()
1509                 {
1510                         Report.Error (1517, Location, "Invalid pre-processor directive");
1511                 }
1512
1513                 void Error_UnexpectedDirective (string extra)
1514                 {
1515                         Report.Error (
1516                                 1028, Location,
1517                                 "Unexpected processor directive (" + extra + ")");
1518                 }
1519
1520                 void Error_TokensSeen ()
1521                 {
1522                         Report.Error (
1523                                 1032, Location,
1524                                 "Cannot define or undefine pre-processor symbols after a token in the file");
1525                 }
1526
1527                 //
1528                 // if true, then the code continues processing the code
1529                 // if false, the code stays in a loop until another directive is
1530                 // reached.
1531                 //
1532                 bool handle_preprocessing_directive (bool caller_is_taking)
1533                 {
1534                         string cmd, arg;
1535                         bool region_directive = false;
1536
1537                         get_cmd_arg (out cmd, out arg);
1538
1539                         // Eat any trailing whitespaces and single-line comments
1540                         if (arg.IndexOf ("//") != -1)
1541                                 arg = arg.Substring (0, arg.IndexOf ("//"));
1542                         arg = arg.TrimEnd (' ', '\t');
1543
1544                         //
1545                         // The first group of pre-processing instructions is always processed
1546                         //
1547                         switch (cmd){
1548                         case "pragma":
1549                                 if (RootContext.Version == LanguageVersion.ISO_1) {
1550                                         Report.FeatureIsNotStandardized (Location, "#pragma");
1551                                         return caller_is_taking;
1552                                 }
1553
1554                                 PreProcessPragma (arg);
1555                                 return caller_is_taking;
1556
1557                         case "line":
1558                                 if (!PreProcessLine (arg))
1559                                         Report.Error (
1560                                                 1576, Location,
1561                                                 "Argument to #line directive is missing or invalid");
1562                                 return caller_is_taking;
1563
1564                         case "region":
1565                                 region_directive = true;
1566                                 arg = "true";
1567                                 goto case "if";
1568
1569                         case "endregion":
1570                                 region_directive = true;
1571                                 goto case "endif";
1572
1573                         case "if":
1574                                 if (arg == ""){
1575                                         Error_InvalidDirective ();
1576                                         return true;
1577                                 }
1578                                 bool taking = false;
1579                                 if (ifstack == null)
1580                                         ifstack = new Stack ();
1581
1582                                 if (ifstack.Count == 0){
1583                                         taking = true;
1584                                 } else {
1585                                         int state = (int) ifstack.Peek ();
1586                                         if ((state & TAKING) != 0)
1587                                                 taking = true;
1588                                 }
1589
1590                                 if (eval (arg) && taking){
1591                                         int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;
1592                                         if (region_directive)
1593                                                 push |= REGION;
1594                                         ifstack.Push (push);
1595                                         return true;
1596                                 } else {
1597                                         int push = (taking ? PARENT_TAKING : 0);
1598                                         if (region_directive)
1599                                                 push |= REGION;
1600                                         ifstack.Push (push);
1601                                         return false;
1602                                 }
1603
1604                         case "endif":
1605                                 if (ifstack == null || ifstack.Count == 0){
1606                                         Error_UnexpectedDirective ("no #if for this #endif");
1607                                         return true;
1608                                 } else {
1609                                         int pop = (int) ifstack.Pop ();
1610
1611                                         if (region_directive && ((pop & REGION) == 0))
1612                                                 Report.Error (1027, Location, "#endif directive expected");
1613                                         else if (!region_directive && ((pop & REGION) != 0))
1614                                                 Report.Error (1038, Location, "#endregion directive expected");
1615
1616                                         if (ifstack.Count == 0)
1617                                                 return true;
1618                                         else {
1619                                                 int state = (int) ifstack.Peek ();
1620
1621                                                 if ((state & TAKING) != 0)
1622                                                         return true;
1623                                                 else
1624                                                         return false;
1625                                         }
1626                                 }
1627
1628                         case "elif":
1629                                 if (ifstack == null || ifstack.Count == 0){
1630                                         Error_UnexpectedDirective ("no #if for this #elif");
1631                                         return true;
1632                                 } else {
1633                                         int state = (int) ifstack.Peek ();
1634
1635                                         if ((state & REGION) != 0) {
1636                                                 Report.Error (1038, Location, "#endregion directive expected");
1637                                                 return true;
1638                                         }
1639
1640                                         if ((state & ELSE_SEEN) != 0){
1641                                                 Error_UnexpectedDirective ("#elif not valid after #else");
1642                                                 return true;
1643                                         }
1644
1645                                         if ((state & (TAKEN_BEFORE | TAKING)) != 0)
1646                                                 return false;
1647
1648                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
1649                                                 state = (int) ifstack.Pop ();
1650                                                 ifstack.Push (state | TAKING | TAKEN_BEFORE);
1651                                                 return true;
1652                                         } else
1653                                                 return false;
1654                                 }
1655
1656                         case "else":
1657                                 if (ifstack == null || ifstack.Count == 0){
1658                                         Report.Error (
1659                                                 1028, Location,
1660                                                 "Unexpected processor directive (no #if for this #else)");
1661                                         return true;
1662                                 } else {
1663                                         int state = (int) ifstack.Peek ();
1664
1665                                         if ((state & REGION) != 0) {
1666                                                 Report.Error (1038, Location, "#endregion directive expected");
1667                                                 return true;
1668                                         }
1669
1670                                         if ((state & ELSE_SEEN) != 0){
1671                                                 Error_UnexpectedDirective ("#else within #else");
1672                                                 return true;
1673                                         }
1674
1675                                         ifstack.Pop ();
1676
1677                                         bool ret;
1678                                         if ((state & TAKEN_BEFORE) == 0){
1679                                                 ret = ((state & PARENT_TAKING) != 0);
1680                                         } else
1681                                                 ret = false;
1682
1683                                         if (ret)
1684                                                 state |= TAKING;
1685                                         else
1686                                                 state &= ~TAKING;
1687
1688                                         ifstack.Push (state | ELSE_SEEN);
1689
1690                                         return ret;
1691                                 }
1692                         }
1693
1694                         //
1695                         // These are only processed if we are in a `taking' block
1696                         //
1697                         if (!caller_is_taking)
1698                                 return false;
1699
1700                         switch (cmd){
1701                         case "define":
1702                                 if (any_token_seen){
1703                                         Error_TokensSeen ();
1704                                         return true;
1705                                 }
1706                                 PreProcessDefinition (true, arg);
1707                                 return true;
1708
1709                         case "undef":
1710                                 if (any_token_seen){
1711                                         Error_TokensSeen ();
1712                                         return true;
1713                                 }
1714                                 PreProcessDefinition (false, arg);
1715                                 return true;
1716
1717                         case "error":
1718                                 Report.Error (1029, Location, "#error: '" + arg + "'");
1719                                 return true;
1720
1721                         case "warning":
1722                                 Report.Warning (1030, Location, "#warning: '{0}'", arg);
1723                                 return true;
1724                         }
1725
1726                         Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");
1727                         return true;
1728
1729                 }
1730
1731                 private int consume_string (bool quoted)
1732                 {
1733                         int c;
1734                         string_builder.Length = 0;
1735
1736                         while ((c = getChar ()) != -1){
1737                                 if (c == '"'){
1738                                         if (quoted && peekChar () == '"'){
1739                                                 string_builder.Append ((char) c);
1740                                                 getChar ();
1741                                                 continue;
1742                                         } else {
1743                                                 val = string_builder.ToString ();
1744                                                 return Token.LITERAL_STRING;
1745                                         }
1746                                 }
1747
1748                                 if (c == '\n'){
1749                                         if (!quoted)
1750                                                 Report.Error (1010, Location, "Newline in constant");
1751                                         line++;
1752                                         ref_line++;
1753                                         col = 0;
1754                                 } else
1755                                         col++;
1756
1757                                 if (!quoted){
1758                                         c = escape (c);
1759                                         if (c == -1)
1760                                                 return Token.ERROR;
1761                                 }
1762                                 string_builder.Append ((char) c);
1763                         }
1764
1765                         Report.Error (1039, Location, "Unterminated string literal");
1766                         return Token.EOF;
1767                 }
1768
1769                 private int consume_identifier (int s)
1770                 {
1771                         int res = consume_identifier (s, false);
1772
1773                         if (doc_state == XmlCommentState.Allowed)
1774                                 doc_state = XmlCommentState.NotAllowed;
1775                         switch (res) {
1776                         case Token.USING:
1777                         case Token.NAMESPACE:
1778                                 check_incorrect_doc_comment ();
1779                                 break;
1780                         }
1781
1782                         if (res == Token.PARTIAL) {
1783                                 // Save current position and parse next token.
1784                                 int old = reader.Position;
1785                                 int old_putback = putback_char;
1786
1787                                 putback_char = -1;
1788
1789                                 int next_token = token ();
1790                                 bool ok = (next_token == Token.CLASS) ||
1791                                         (next_token == Token.STRUCT) ||
1792                                         (next_token == Token.INTERFACE);
1793
1794                                 reader.Position = old;
1795                                 putback_char = old_putback;
1796
1797                                 if (ok)
1798                                         return res;
1799                                 else {
1800                                         val = "partial";
1801                                         return Token.IDENTIFIER;
1802                                 }
1803                         }
1804
1805                         return res;
1806                 }
1807
1808                 private int consume_identifier (int s, bool quoted)
1809                 {
1810                         int pos = 1;
1811                         int c;
1812
1813                         id_builder [0] = (char) s;
1814
1815                         while ((c = reader.Read ()) != -1) {
1816                                 if (is_identifier_part_character ((char) c)){
1817                                         if (pos == max_id_size){
1818                                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
1819                                                 return Token.ERROR;
1820                                         }
1821
1822                                         id_builder [pos++] = (char) c;
1823                                         putback_char = -1;
1824                                         col++;
1825                                 } else {
1826                                         putback_char = c;
1827                                         break;
1828                                 }
1829                         }
1830
1831                         //
1832                         // Optimization: avoids doing the keyword lookup
1833                         // on uppercase letters and _
1834                         //
1835                         if (!quoted && (s >= 'a' || s == '_')){
1836                                 int keyword = GetKeyword (id_builder, pos);
1837                                 if (keyword != -1)
1838                                         return keyword;
1839                         }
1840
1841                         //
1842                         // Keep identifiers in an array of hashtables to avoid needless
1843                         // allocations
1844                         //
1845
1846                         if (identifiers [pos] != null) {
1847                                 val = identifiers [pos][id_builder];
1848                                 if (val != null) {
1849                                         return Token.IDENTIFIER;
1850                                 }
1851                         }
1852                         else
1853                                 identifiers [pos] = new CharArrayHashtable (pos);
1854
1855                         val = new String (id_builder, 0, pos);
1856
1857                         char [] chars = new char [pos];
1858                         Array.Copy (id_builder, chars, pos);
1859
1860                         identifiers [pos] [chars] = val;
1861
1862                         return Token.IDENTIFIER;
1863                 }
1864
1865                 public int xtoken ()
1866                 {
1867                         int t;
1868                         bool doread = false;
1869                         int c;
1870
1871                         val = null;
1872                         // optimization: eliminate col and implement #directive semantic correctly.
1873                         for (;(c = getChar ()) != -1; col++) {
1874                                 if (c == ' ')
1875                                         continue;
1876
1877                                 if (c == '\t') {
1878                                         col = (((col + 8) / 8) * 8) - 1;
1879                                         continue;
1880                                 }
1881
1882                                 if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0)
1883                                         continue;
1884
1885                                 if (c == '\r') {
1886                                         if (peekChar () == '\n')
1887                                                 getChar ();
1888
1889                                         line++;
1890                                         ref_line++;
1891                                         col = 0;
1892                                         any_token_seen |= tokens_seen;
1893                                         tokens_seen = false;
1894                                         continue;
1895                                 }
1896
1897                                 // Handle double-slash comments.
1898                                 if (c == '/'){
1899                                         int d = peekChar ();
1900
1901                                         if (d == '/'){
1902                                                 getChar ();
1903                                                 if (RootContext.Documentation != null && peekChar () == '/') {
1904                                                         getChar ();
1905                                                         // Don't allow ////.
1906                                                         if ((d = peekChar ()) != '/') {
1907                                                                 if (doc_state == XmlCommentState.Allowed)
1908                                                                         handle_one_line_xml_comment ();
1909                                                                 else if (doc_state == XmlCommentState.NotAllowed)
1910                                                                         warn_incorrect_doc_comment ();
1911                                                         }
1912                                                 }
1913                                                 while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
1914                                                         col++;
1915                                                 if (d == '\n'){
1916                                                         line++;
1917                                                         ref_line++;
1918                                                         col = 0;
1919                                                 }
1920                                                 any_token_seen |= tokens_seen;
1921                                                 tokens_seen = false;
1922                                                 continue;
1923                                         } else if (d == '*'){
1924                                                 getChar ();
1925                                                 bool docAppend = false;
1926                                                 if (RootContext.Documentation != null && peekChar () == '*') {
1927                                                         getChar ();
1928                                                         // But when it is /**/, just do nothing.
1929                                                         if (peekChar () == '/') {
1930                                                                 getChar ();
1931                                                                 continue;
1932                                                         }
1933                                                         if (doc_state == XmlCommentState.Allowed)
1934                                                                 docAppend = true;
1935                                                         else if (doc_state == XmlCommentState.NotAllowed)
1936                                                                 warn_incorrect_doc_comment ();
1937                                                 }
1938
1939                                                 int currentCommentStart = 0;
1940                                                 if (docAppend) {
1941                                                         currentCommentStart = xml_comment_buffer.Length;
1942                                                         xml_comment_buffer.Append (Environment.NewLine);
1943                                                 }
1944
1945                                                 while ((d = getChar ()) != -1){
1946                                                         if (d == '*' && peekChar () == '/'){
1947                                                                 getChar ();
1948                                                                 col++;
1949                                                                 break;
1950                                                         }
1951                                                         if (docAppend)
1952                                                                 xml_comment_buffer.Append ((char) d);
1953
1954                                                         if (d == '\n'){
1955                                                                 line++;
1956                                                                 ref_line++;
1957                                                                 col = 0;
1958                                                                 any_token_seen |= tokens_seen;
1959                                                                 tokens_seen = false;
1960                                                         }
1961                                                 }
1962                                                 if (docAppend)
1963                                                         update_formatted_doc_comment (currentCommentStart);
1964                                                 continue;
1965                                         }
1966                                         goto is_punct_label;
1967                                 }
1968
1969
1970                                 if (is_identifier_start_character ((char)c)){
1971                                         tokens_seen = true;
1972                                         return consume_identifier (c);
1973                                 }
1974
1975                         is_punct_label:
1976                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
1977                                         tokens_seen = true;
1978                                         if (doread){
1979                                                 getChar ();
1980                                                 col++;
1981                                         }
1982                                         return t;
1983                                 }
1984
1985                                 // white space
1986                                 if (c == '\n'){
1987                                         line++;
1988                                         ref_line++;
1989                                         col = 0;
1990                                         any_token_seen |= tokens_seen;
1991                                         tokens_seen = false;
1992                                         continue;
1993                                 }
1994
1995                                 if (c >= '0' && c <= '9'){
1996                                         tokens_seen = true;
1997                                         return is_number (c);
1998                                 }
1999
2000                                 if (c == '.'){
2001                                         tokens_seen = true;
2002                                         int peek = peekChar ();
2003                                         if (peek >= '0' && peek <= '9')
2004                                                 return is_number (c);
2005                                         return Token.DOT;
2006                                 }
2007
2008                                 /* For now, ignore pre-processor commands */
2009                                 // FIXME: In C# the '#' is not limited to appear
2010                                 // on the first column.
2011                                 if (c == '#' && !tokens_seen){
2012                                         bool cont = true;
2013
2014                                 start_again:
2015
2016                                         cont = handle_preprocessing_directive (cont);
2017
2018                                         if (cont){
2019                                                 col = 0;
2020                                                 continue;
2021                                         }
2022                                         col = 1;
2023
2024                                         bool skipping = false;
2025                                         for (;(c = getChar ()) != -1; col++){
2026                                                 if (c == '\n'){
2027                                                         col = 0;
2028                                                         line++;
2029                                                         ref_line++;
2030                                                         skipping = false;
2031                                                 } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
2032                                                         continue;
2033                                                 else if (c != '#')
2034                                                         skipping = true;
2035                                                 if (c == '#' && !skipping)
2036                                                         goto start_again;
2037                                         }
2038                                         any_token_seen |= tokens_seen;
2039                                         tokens_seen = false;
2040                                         if (c == -1)
2041                                                 Report.Error (1027, Location, "#endif/#endregion expected");
2042                                         continue;
2043                                 }
2044
2045                                 if (c == '"')
2046                                         return consume_string (false);
2047
2048                                 if (c == '\''){
2049                                         c = getChar ();
2050                                         tokens_seen = true;
2051                                         if (c == '\''){
2052                                                 error_details = "Empty character literal";
2053                                                 Report.Error (1011, Location, error_details);
2054                                                 return Token.ERROR;
2055                                         }
2056                                         c = escape (c);
2057                                         if (c == -1)
2058                                                 return Token.ERROR;
2059                                         val = new System.Char ();
2060                                         val = (char) c;
2061                                         c = getChar ();
2062
2063                                         if (c != '\''){
2064                                                 error_details = "Too many characters in character literal";
2065                                                 Report.Error (1012, Location, error_details);
2066
2067                                                 // Try to recover, read until newline or next "'"
2068                                                 while ((c = getChar ()) != -1){
2069                                                         if (c == '\n' || c == '\''){
2070                                                                 line++;
2071                                                                 ref_line++;
2072                                                                 col = 0;
2073                                                                 break;
2074                                                         } else
2075                                                                 col++;
2076
2077                                                 }
2078                                                 return Token.ERROR;
2079                                         }
2080                                         return Token.LITERAL_CHARACTER;
2081                                 }
2082
2083                                 if (c == '@') {
2084                                         c = getChar ();
2085                                         if (c == '"') {
2086                                                 tokens_seen = true;
2087                                                 return consume_string (true);
2088                                         } else if (is_identifier_start_character ((char) c)){
2089                                                 return consume_identifier (c, true);
2090                                         } else {
2091                                                 Report.Error (1033, Location, "'@' must be followed by string constant or identifier");
2092                                         }
2093                                 }
2094
2095                                 if (c == '#') {
2096                                         error_details = "Preprocessor directives must appear as the first non-whitespace " +
2097                                                 "character on a line.";
2098
2099                                         Report.Error (1040, Location, error_details);
2100
2101                                         return Token.ERROR;
2102                                 }
2103
2104                                 error_details = ((char)c).ToString ();
2105
2106                                 return Token.ERROR;
2107                         }
2108
2109                         return Token.EOF;
2110                 }
2111
2112                 //
2113                 // Handles one line xml comment
2114                 //
2115                 private void handle_one_line_xml_comment ()
2116                 {
2117                         int c;
2118                         while ((c = peekChar ()) == ' ')
2119                                 getChar (); // skip heading whitespaces.
2120                         while ((c = peekChar ()) != -1 && c != '\n' && c != '\r') {
2121                                 col++;
2122                                 xml_comment_buffer.Append ((char) getChar ());
2123                         }
2124                         if (c == '\r' || c == '\n')
2125                                 xml_comment_buffer.Append (Environment.NewLine);
2126                 }
2127
2128                 //
2129                 // Remove heading "*" in Javadoc-like xml documentation.
2130                 //
2131                 private void update_formatted_doc_comment (int currentCommentStart)
2132                 {
2133                         int length = xml_comment_buffer.Length - currentCommentStart;
2134                         string [] lines = xml_comment_buffer.ToString (
2135                                 currentCommentStart,
2136                                 length).Replace ("\r", "").Split ('\n');
2137                         // The first line starts with /**, thus it is not target
2138                         // for the format check.
2139                         for (int i = 1; i < lines.Length; i++) {
2140                                 string s = lines [i];
2141                                 int idx = s.IndexOf ('*');
2142                                 string head = null;
2143                                 if (idx < 0) {
2144                                         if (i < lines.Length - 1)
2145                                                 return;
2146                                         head = s;
2147                                 }
2148                                 else
2149                                         head = s.Substring (0, idx);
2150                                 foreach (char c in head)
2151                                         if (c != ' ')
2152                                                 return;
2153                                 lines [i] = s.Substring (idx + 1);
2154                         }
2155                         xml_comment_buffer.Remove (currentCommentStart, length);
2156                         xml_comment_buffer.Insert (
2157                                 currentCommentStart,
2158                                 String.Join (Environment.NewLine, lines));
2159                 }
2160
2161                 //
2162                 // Checks if there was incorrect doc comments and raise
2163                 // warnings.
2164                 //
2165                 public void check_incorrect_doc_comment ()
2166                 {
2167                         if (xml_comment_buffer.Length > 0)
2168                                 warn_incorrect_doc_comment ();
2169                 }
2170
2171                 //
2172                 // Raises a warning when tokenizer found incorrect doccomment
2173                 // markup.
2174                 //
2175                 private void warn_incorrect_doc_comment ()
2176                 {
2177                         doc_state = XmlCommentState.Error;
2178                         // in csc, it is 'XML comment is not placed on a valid
2179                         // language element'. But that does not make sense.
2180                         Report.Warning (1587, 2, Location, "XML comment is placed on an invalid language element which can not accept it.");
2181                 }
2182
2183                 //
2184                 // Consumes the saved xml comment lines (if any)
2185                 // as for current target member or type.
2186                 //
2187                 public string consume_doc_comment ()
2188                 {
2189                         if (xml_comment_buffer.Length > 0) {
2190                                 string ret = xml_comment_buffer.ToString ();
2191                                 xml_comment_buffer.Length = 0;
2192                                 return ret;
2193                         }
2194                         return null;
2195                 }
2196
2197                 public void cleanup ()
2198                 {
2199                         if (ifstack != null && ifstack.Count >= 1) {
2200                                 int state = (int) ifstack.Pop ();
2201                                 if ((state & REGION) != 0)
2202                                         Report.Error (1038, Location, "#endregion directive expected");
2203                                 else
2204                                         Report.Error (1027, "#endif directive expected");
2205                         }
2206
2207                 }
2208         }
2209
2210         //
2211         // Indicates whether it accepts XML documentation or not.
2212         //
2213         public enum XmlCommentState {
2214                 // comment is allowed in this state.
2215                 Allowed,
2216                 // comment is not allowed in this state.
2217                 NotAllowed,
2218                 // once comments appeared when it is NotAllowed, then the
2219                 // state is changed to it, until the state is changed to
2220                 // .Allowed.
2221                 Error
2222         }
2223 }