mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //
   7 // Licensed under the terms of the GNU GPL
   8 //
   9 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
  10 //
  11
  12 /*
  13  * TODO:
  14  *   Make sure we accept the proper Unicode ranges, per the spec.
  15  *   Report error 1032
  16 */
  17
  18 using System;
  19 using System.Text;
  20 using System.Collections;
  21 using System.IO;
  22 using System.Globalization;
  23 using System.Reflection;
  24
  25 namespace Mono.CSharp
  26 {
  27         /// <summary>
  28         ///    Tokenizer for C# source code.
  29         /// </summary>
  30
  31         public class Tokenizer : yyParser.yyInput
  32         {
  33                 SeekableStreamReader reader;
  34                 public SourceFile ref_name;
  35                 public SourceFile file_name;
  36                 public int ref_line = 1;
  37                 public int line = 1;
  38                 public int col = 1;
  39                 public int current_token;
  40                 bool handle_get_set = false;
  41                 bool handle_remove_add = false;
  42                 bool handle_assembly = false;
  43
  44                 //
  45                 // Whether tokens have been seen on this line
  46                 //
  47                 bool tokens_seen = false;
  48
  49                 //
  50                 // Whether a token has been seen on the file
  51                 // This is needed because `define' is not allowed to be used
  52                 // after a token has been seen.
  53                 //
  54                 bool any_token_seen = false;
  55                 static Hashtable tokenValues;
  56
  57                 private static Hashtable TokenValueName
  58                 {
  59                         get {
  60                                 if (tokenValues == null)
  61                                         tokenValues = GetTokenValueNameHash ();
  62
  63                                 return tokenValues;
  64                         }
  65                 }
  66
  67                 private static Hashtable GetTokenValueNameHash ()
  68                 {
  69                         Type t = typeof (Token);
  70                         FieldInfo [] fields = t.GetFields ();
  71                         Hashtable hash = new Hashtable ();
  72                         foreach (FieldInfo field in fields) {
  73                                 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))
  74                                         hash.Add (field.GetValue (null), field.Name);
  75                         }
  76                         return hash;
  77                 }
  78
  79                 //
  80                 // Returns a verbose representation of the current location
  81                 //
  82                 public string location {
  83                         get {
  84                                 string det;
  85
  86                                 if (current_token == Token.ERROR)
  87                                         det = "detail: " + error_details;
  88                                 else
  89                                         det = "";
  90
  91                                 // return "Line:     "+line+" Col: "+col + "\n" +
  92                                 //       "VirtLine: "+ref_line +
  93                                 //       " Token: "+current_token + " " + det;
  94                                 string current_token_name = TokenValueName [current_token] as string;
  95                                 if (current_token_name == null)
  96                                         current_token_name = current_token.ToString ();
  97
  98                                 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,
  99                                                                                        ref_line,
 100                                                                                        col,
 101                                                                                        current_token_name,
 102                                                                                        det);
 103                         }
 104                 }
 105
 106                 public bool PropertyParsing {
 107                         get {
 108                                 return handle_get_set;
 109                         }
 110
 111                         set {
 112                                 handle_get_set = value;
 113                         }
 114                 }
 115
 116                 public bool AssemblyTargetParsing {
 117                         get {
 118                                 return handle_assembly;
 119                         }
 120
 121                         set {
 122                                 handle_assembly = value;
 123                         }
 124                 }
 125
 126                 public bool EventParsing {
 127                         get {
 128                                 return handle_remove_add;
 129                         }
 130
 131                         set {
 132                                 handle_remove_add = value;
 133                         }
 134                 }
 135
 136                 //
 137                 // Class variables
 138                 //
 139                 static CharArrayHashtable[] keywords;
 140                 static NumberStyles styles;
 141                 static NumberFormatInfo csharp_format_info;
 142
 143                 //
 144                 // Values for the associated token returned
 145                 //
 146                 int putback_char;
 147                 Object val;
 148
 149                 //
 150                 // Pre-processor
 151                 //
 152                 Hashtable defines;
 153
 154                 const int TAKING        = 1;
 155                 const int TAKEN_BEFORE  = 2;
 156                 const int ELSE_SEEN     = 4;
 157                 const int PARENT_TAKING = 8;
 158                 const int REGION        = 16;
 159
 160                 //
 161                 // pre-processor if stack state:
 162                 //
 163                 Stack ifstack;
 164
 165                 static System.Text.StringBuilder string_builder;
 166
 167                 const int max_id_size = 512;
 168                 static char [] id_builder = new char [max_id_size];
 169
 170                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 171
 172                 const int max_number_size = 128;
 173                 static char [] number_builder = new char [max_number_size];
 174                 static int number_pos;
 175
 176                 //
 177                 // Details about the error encoutered by the tokenizer
 178                 //
 179                 string error_details;
 180
 181                 public string error {
 182                         get {
 183                                 return error_details;
 184                         }
 185                 }
 186
 187                 public int Line {
 188                         get {
 189                                 return ref_line;
 190                         }
 191                 }
 192
 193                 public int Col {
 194                         get {
 195                                 return col;
 196                         }
 197                 }
 198
 199                 static void AddKeyword (string kw, int token) {
 200                         if (keywords [kw.Length] == null) {
 201                                 keywords [kw.Length] = new CharArrayHashtable (kw.Length);
 202                         }
 203                         keywords [kw.Length] [kw.ToCharArray ()] = token;
 204                 }
 205
 206                 static void InitTokens ()
 207                 {
 208                         keywords = new CharArrayHashtable [64];
 209
 210                         AddKeyword ("__arglist", Token.ARGLIST);
 211                         AddKeyword ("abstract", Token.ABSTRACT);
 212                         AddKeyword ("as", Token.AS);
 213                         AddKeyword ("add", Token.ADD);
 214                         AddKeyword ("assembly", Token.ASSEMBLY);
 215                         AddKeyword ("base", Token.BASE);
 216                         AddKeyword ("bool", Token.BOOL);
 217                         AddKeyword ("break", Token.BREAK);
 218                         AddKeyword ("byte", Token.BYTE);
 219                         AddKeyword ("case", Token.CASE);
 220                         AddKeyword ("catch", Token.CATCH);
 221                         AddKeyword ("char", Token.CHAR);
 222                         AddKeyword ("checked", Token.CHECKED);
 223                         AddKeyword ("class", Token.CLASS);
 224                         AddKeyword ("const", Token.CONST);
 225                         AddKeyword ("continue", Token.CONTINUE);
 226                         AddKeyword ("decimal", Token.DECIMAL);
 227                         AddKeyword ("default", Token.DEFAULT);
 228                         AddKeyword ("delegate", Token.DELEGATE);
 229                         AddKeyword ("do", Token.DO);
 230                         AddKeyword ("double", Token.DOUBLE);
 231                         AddKeyword ("else", Token.ELSE);
 232                         AddKeyword ("enum", Token.ENUM);
 233                         AddKeyword ("event", Token.EVENT);
 234                         AddKeyword ("explicit", Token.EXPLICIT);
 235                         AddKeyword ("extern", Token.EXTERN);
 236                         AddKeyword ("false", Token.FALSE);
 237                         AddKeyword ("finally", Token.FINALLY);
 238                         AddKeyword ("fixed", Token.FIXED);
 239                         AddKeyword ("float", Token.FLOAT);
 240                         AddKeyword ("for", Token.FOR);
 241                         AddKeyword ("foreach", Token.FOREACH);
 242                         AddKeyword ("goto", Token.GOTO);
 243                         AddKeyword ("get", Token.GET);
 244                         AddKeyword ("if", Token.IF);
 245                         AddKeyword ("implicit", Token.IMPLICIT);
 246                         AddKeyword ("in", Token.IN);
 247                         AddKeyword ("int", Token.INT);
 248                         AddKeyword ("interface", Token.INTERFACE);
 249                         AddKeyword ("internal", Token.INTERNAL);
 250                         AddKeyword ("is", Token.IS);
 251                         AddKeyword ("lock", Token.LOCK);
 252                         AddKeyword ("long", Token.LONG);
 253                         AddKeyword ("namespace", Token.NAMESPACE);
 254                         AddKeyword ("new", Token.NEW);
 255                         AddKeyword ("null", Token.NULL);
 256                         AddKeyword ("object", Token.OBJECT);
 257                         AddKeyword ("operator", Token.OPERATOR);
 258                         AddKeyword ("out", Token.OUT);
 259                         AddKeyword ("override", Token.OVERRIDE);
 260                         AddKeyword ("params", Token.PARAMS);
 261                         AddKeyword ("private", Token.PRIVATE);
 262                         AddKeyword ("protected", Token.PROTECTED);
 263                         AddKeyword ("public", Token.PUBLIC);
 264                         AddKeyword ("readonly", Token.READONLY);
 265                         AddKeyword ("ref", Token.REF);
 266                         AddKeyword ("remove", Token.REMOVE);
 267                         AddKeyword ("return", Token.RETURN);
 268                         AddKeyword ("sbyte", Token.SBYTE);
 269                         AddKeyword ("sealed", Token.SEALED);
 270                         AddKeyword ("set", Token.SET);
 271                         AddKeyword ("short", Token.SHORT);
 272                         AddKeyword ("sizeof", Token.SIZEOF);
 273                         AddKeyword ("stackalloc", Token.STACKALLOC);
 274                         AddKeyword ("static", Token.STATIC);
 275                         AddKeyword ("string", Token.STRING);
 276                         AddKeyword ("struct", Token.STRUCT);
 277                         AddKeyword ("switch", Token.SWITCH);
 278                         AddKeyword ("this", Token.THIS);
 279                         AddKeyword ("throw", Token.THROW);
 280                         AddKeyword ("true", Token.TRUE);
 281                         AddKeyword ("try", Token.TRY);
 282                         AddKeyword ("typeof", Token.TYPEOF);
 283                         AddKeyword ("uint", Token.UINT);
 284                         AddKeyword ("ulong", Token.ULONG);
 285                         AddKeyword ("unchecked", Token.UNCHECKED);
 286                         AddKeyword ("unsafe", Token.UNSAFE);
 287                         AddKeyword ("ushort", Token.USHORT);
 288                         AddKeyword ("using", Token.USING);
 289                         AddKeyword ("virtual", Token.VIRTUAL);
 290                         AddKeyword ("void", Token.VOID);
 291                         AddKeyword ("volatile", Token.VOLATILE);
 292                         AddKeyword ("while", Token.WHILE);
 293                         AddKeyword ("partial", Token.PARTIAL);
 294                 }
 295
 296                 //
 297                 // Class initializer
 298                 //
 299                 static Tokenizer ()
 300                 {
 301                         InitTokens ();
 302                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 303                         styles = NumberStyles.Float;
 304
 305                         string_builder = new System.Text.StringBuilder ();
 306                 }
 307
 308                 int GetKeyword (char[] id, int id_len)
 309                 {
 310                         /*
 311                          * Keywords are stored in an array of hashtables grouped by their
 312                          * length.
 313                          */
 314
 315                         if ((id_len >= keywords.Length) || (keywords [id_len] == null))
 316                                 return -1;
 317                         object o = keywords [id_len] [id];
 318
 319                         if (o == null)
 320                                 return -1;
 321
 322                         int res = (int) o;
 323
 324                         if (handle_get_set == false && (res == Token.GET || res == Token.SET))
 325                                 return -1;
 326                         if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
 327                                 return -1;
 328                         if (handle_assembly == false && res == Token.ASSEMBLY)
 329                                 return -1;
 330
 331                         return res;
 332
 333                 }
 334
 335                 public Location Location {
 336                         get {
 337                                 return new Location (ref_line);
 338                         }
 339                 }
 340
 341                 void define (string def)
 342                 {
 343                         if (!RootContext.AllDefines.Contains (def)){
 344                                 RootContext.AllDefines [def] = true;
 345                         }
 346                         if (defines.Contains (def))
 347                                 return;
 348                         defines [def] = true;
 349                 }
 350
 351                 public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)
 352                 {
 353                         this.ref_name = file;
 354                         this.file_name = file;
 355                         reader = input;
 356
 357                         putback_char = -1;
 358
 359                         if (defs != null){
 360                                 defines = new Hashtable ();
 361                                 foreach (string def in defs)
 362                                         define (def);
 363                         }
 364
 365                         //
 366                         // FIXME: This could be `Location.Push' but we have to
 367                         // find out why the MS compiler allows this
 368                         //
 369                         Mono.CSharp.Location.Push (file);
 370                 }
 371
 372                 public static void Cleanup () {
 373                         identifiers = null;
 374                 }
 375
 376                 static bool is_identifier_start_character (char c)
 377                 {
 378                         return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c);
 379                 }
 380
 381                 static bool is_identifier_part_character (char c)
 382                 {
 383                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
 384                 }
 385
 386                 public static bool IsValidIdentifier (string s)
 387                 {
 388                         if (s == null || s.Length == 0)
 389                                 return false;
 390
 391                         if (!is_identifier_start_character (s [0]))
 392                                 return false;
 393
 394                         for (int i = 1; i < s.Length; i ++)
 395                                 if (! is_identifier_part_character (s [i]))
 396                                         return false;
 397
 398                         return true;
 399                 }
 400
 401                 int is_punct (char c, ref bool doread)
 402                 {
 403                         int d;
 404                         int t;
 405
 406                         doread = false;
 407
 408                         switch (c){
 409                         case '{':
 410                                 return Token.OPEN_BRACE;
 411                         case '}':
 412                                 return Token.CLOSE_BRACE;
 413                         case '[':
 414                                 return Token.OPEN_BRACKET;
 415                         case ']':
 416                                 return Token.CLOSE_BRACKET;
 417                         case '(':
 418                                 return Token.OPEN_PARENS;
 419                         case ')': {
 420                                 if (deambiguate_close_parens == 0)
 421                                         return Token.CLOSE_PARENS;
 422
 423                                 --deambiguate_close_parens;
 424
 425                                 // Save current position and parse next token.
 426                                 int old = reader.Position;
 427                                 int new_token = token ();
 428                                 reader.Position = old;
 429                                 putback_char = -1;
 430
 431                                 if (new_token == Token.OPEN_PARENS)
 432                                         return Token.CLOSE_PARENS_OPEN_PARENS;
 433                                 else if (new_token == Token.MINUS)
 434                                         return Token.CLOSE_PARENS_MINUS;
 435                                 else if (IsCastToken (new_token))
 436                                         return Token.CLOSE_PARENS_CAST;
 437                                 else
 438                                         return Token.CLOSE_PARENS_NO_CAST;
 439                         }
 440
 441                         case ',':
 442                                 return Token.COMMA;
 443                         case ':':
 444                                 return Token.COLON;
 445                         case ';':
 446                                 return Token.SEMICOLON;
 447                         case '~':
 448                                 return Token.TILDE;
 449                         case '?':
 450                                 return Token.INTERR;
 451                         }
 452
 453                         d = peekChar ();
 454                         if (c == '+'){
 455
 456                                 if (d == '+')
 457                                         t = Token.OP_INC;
 458                                 else if (d == '=')
 459                                         t = Token.OP_ADD_ASSIGN;
 460                                 else
 461                                         return Token.PLUS;
 462                                 doread = true;
 463                                 return t;
 464                         }
 465                         if (c == '-'){
 466                                 if (d == '-')
 467                                         t = Token.OP_DEC;
 468                                 else if (d == '=')
 469                                         t = Token.OP_SUB_ASSIGN;
 470                                 else if (d == '>')
 471                                         t = Token.OP_PTR;
 472                                 else
 473                                         return Token.MINUS;
 474                                 doread = true;
 475                                 return t;
 476                         }
 477
 478                         if (c == '!'){
 479                                 if (d == '='){
 480                                         doread = true;
 481                                         return Token.OP_NE;
 482                                 }
 483                                 return Token.BANG;
 484                         }
 485
 486                         if (c == '='){
 487                                 if (d == '='){
 488                                         doread = true;
 489                                         return Token.OP_EQ;
 490                                 }
 491                                 return Token.ASSIGN;
 492                         }
 493
 494                         if (c == '&'){
 495                                 if (d == '&'){
 496                                         doread = true;
 497                                         return Token.OP_AND;
 498                                 } else if (d == '='){
 499                                         doread = true;
 500                                         return Token.OP_AND_ASSIGN;
 501                                 }
 502                                 return Token.BITWISE_AND;
 503                         }
 504
 505                         if (c == '|'){
 506                                 if (d == '|'){
 507                                         doread = true;
 508                                         return Token.OP_OR;
 509                                 } else if (d == '='){
 510                                         doread = true;
 511                                         return Token.OP_OR_ASSIGN;
 512                                 }
 513                                 return Token.BITWISE_OR;
 514                         }
 515
 516                         if (c == '*'){
 517                                 if (d == '='){
 518                                         doread = true;
 519                                         return Token.OP_MULT_ASSIGN;
 520                                 }
 521                                 return Token.STAR;
 522                         }
 523
 524                         if (c == '/'){
 525                                 if (d == '='){
 526                                         doread = true;
 527                                         return Token.OP_DIV_ASSIGN;
 528                                 }
 529                                 return Token.DIV;
 530                         }
 531
 532                         if (c == '%'){
 533                                 if (d == '='){
 534                                         doread = true;
 535                                         return Token.OP_MOD_ASSIGN;
 536                                 }
 537                                 return Token.PERCENT;
 538                         }
 539
 540                         if (c == '^'){
 541                                 if (d == '='){
 542                                         doread = true;
 543                                         return Token.OP_XOR_ASSIGN;
 544                                 }
 545                                 return Token.CARRET;
 546                         }
 547
 548                         if (c == '<'){
 549                                 if (d == '<'){
 550                                         getChar ();
 551                                         d = peekChar ();
 552
 553                                         if (d == '='){
 554                                                 doread = true;
 555                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 556                                         }
 557                                         return Token.OP_SHIFT_LEFT;
 558                                 } else if (d == '='){
 559                                         doread = true;
 560                                         return Token.OP_LE;
 561                                 }
 562                                 return Token.OP_LT;
 563                         }
 564
 565                         if (c == '>'){
 566                                 if (d == '>'){
 567                                         getChar ();
 568                                         d = peekChar ();
 569
 570                                         if (d == '='){
 571                                                 doread = true;
 572                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 573                                         }
 574                                         return Token.OP_SHIFT_RIGHT;
 575                                 } else if (d == '='){
 576                                         doread = true;
 577                                         return Token.OP_GE;
 578                                 }
 579                                 return Token.OP_GT;
 580                         }
 581                         return Token.ERROR;
 582                 }
 583
 584                 int deambiguate_close_parens = 0;
 585
 586                 public void Deambiguate_CloseParens ()
 587                 {
 588                         putback (')');
 589                         deambiguate_close_parens++;
 590                 }
 591
 592                 void Error_NumericConstantTooLong ()
 593                 {
 594                         Report.Error (1021, Location, "Numeric constant too long");
 595                 }
 596
 597                 bool decimal_digits (int c)
 598                 {
 599                         int d;
 600                         bool seen_digits = false;
 601
 602                         if (c != -1){
 603                                 if (number_pos == max_number_size)
 604                                         Error_NumericConstantTooLong ();
 605                                 number_builder [number_pos++] = (char) c;
 606                         }
 607
 608                         //
 609                         // We use peekChar2, because decimal_digits needs to do a
 610                         // 2-character look-ahead (5.ToString for example).
 611                         //
 612                         while ((d = peekChar2 ()) != -1){
 613                                 if (d >= '0' && d <= '9'){
 614                                         if (number_pos == max_number_size)
 615                                                 Error_NumericConstantTooLong ();
 616                                         number_builder [number_pos++] = (char) d;
 617                                         getChar ();
 618                                         seen_digits = true;
 619                                 } else
 620                                         break;
 621                         }
 622
 623                         return seen_digits;
 624                 }
 625
 626                 bool is_hex (int e)
 627                 {
 628                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
 629                 }
 630
 631                 void hex_digits (int c)
 632                 {
 633                         if (c != -1)
 634                                 number_builder [number_pos++] = (char) c;
 635
 636                 }
 637
 638                 int real_type_suffix (int c)
 639                 {
 640                         int t;
 641
 642                         switch (c){
 643                         case 'F': case 'f':
 644                                 t =  Token.LITERAL_FLOAT;
 645                                 break;
 646                         case 'D': case 'd':
 647                                 t = Token.LITERAL_DOUBLE;
 648                                 break;
 649                         case 'M': case 'm':
 650                                  t= Token.LITERAL_DECIMAL;
 651                                 break;
 652                         default:
 653                                 return Token.NONE;
 654                         }
 655                         return t;
 656                 }
 657
 658                 int integer_type_suffix (ulong ul, int c)
 659                 {
 660                         bool is_unsigned = false;
 661                         bool is_long = false;
 662
 663                         if (c != -1){
 664                                 bool scanning = true;
 665                                 do {
 666                                         switch (c){
 667                                         case 'U': case 'u':
 668                                                 if (is_unsigned)
 669                                                         scanning = false;
 670                                                 is_unsigned = true;
 671                                                 getChar ();
 672                                                 break;
 673
 674                                         case 'l':
 675                                                 if (!is_unsigned && (RootContext.WarningLevel >= 4)){
 676                                                         //
 677                                                         // if we have not seen anything in between
 678                                                         // report this error
 679                                                         //
 680                                                         Report.Warning (78, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
 681                                                 }
 682                                                 goto case 'L';
 683
 684                                         case 'L':
 685                                                 if (is_long)
 686                                                         scanning = false;
 687                                                 is_long = true;
 688                                                 getChar ();
 689                                                 break;
 690
 691                                         default:
 692                                                 scanning = false;
 693                                                 break;
 694                                         }
 695                                         c = peekChar ();
 696                                 } while (scanning);
 697                         }
 698
 699                         if (is_long && is_unsigned){
 700                                 val = ul;
 701                                 return Token.LITERAL_INTEGER;
 702                         } else if (is_unsigned){
 703                                 // uint if possible, or ulong else.
 704
 705                                 if ((ul & 0xffffffff00000000) == 0)
 706                                         val = (uint) ul;
 707                                 else
 708                                         val = ul;
 709                         } else if (is_long){
 710                                 // long if possible, ulong otherwise
 711                                 if ((ul & 0x8000000000000000) != 0)
 712                                         val = ul;
 713                                 else
 714                                         val = (long) ul;
 715                         } else {
 716                                 // int, uint, long or ulong in that order
 717                                 if ((ul & 0xffffffff00000000) == 0){
 718                                         uint ui = (uint) ul;
 719
 720                                         if ((ui & 0x80000000) != 0)
 721                                                 val = ui;
 722                                         else
 723                                                 val = (int) ui;
 724                                 } else {
 725                                         if ((ul & 0x8000000000000000) != 0)
 726                                                 val = ul;
 727                                         else
 728                                                 val = (long) ul;
 729                                 }
 730                         }
 731                         return Token.LITERAL_INTEGER;
 732                 }
 733
 734                 //
 735                 // given `c' as the next char in the input decide whether
 736                 // we need to convert to a special type, and then choose
 737                 // the best representation for the integer
 738                 //
 739                 int adjust_int (int c)
 740                 {
 741                         try {
 742                                 if (number_pos > 9){
 743                                         ulong ul = (uint) (number_builder [0] - '0');
 744
 745                                         for (int i = 1; i < number_pos; i++){
 746                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
 747                                         }
 748                                         return integer_type_suffix (ul, c);
 749                                 } else {
 750                                         uint ui = (uint) (number_builder [0] - '0');
 751
 752                                         for (int i = 1; i < number_pos; i++){
 753                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
 754                                         }
 755                                         return integer_type_suffix (ui, c);
 756                                 }
 757                         } catch (OverflowException) {
 758                                 error_details = "Integral constant is too large";
 759                                 Report.Error (1021, Location, error_details);
 760                                 val = 0ul;
 761                                 return Token.LITERAL_INTEGER;
 762                         }
 763                 }
 764
 765                 int adjust_real (int t)
 766                 {
 767                         string s = new String (number_builder, 0, number_pos);
 768
 769                         switch (t){
 770                         case Token.LITERAL_DECIMAL:
 771                                 try {
 772                                         val = System.Decimal.Parse (s, styles, csharp_format_info);
 773                                 } catch (OverflowException) {
 774                                         val = 0m;
 775                                         error_details = "Floating-point constant is outside the range of the type 'decimal'";
 776                                         Report.Error (594, Location, error_details);
 777                                 }
 778                                 break;
 779                         case Token.LITERAL_FLOAT:
 780                                 try {
 781                                         val = (float) System.Double.Parse (s, styles, csharp_format_info);
 782                                 } catch (OverflowException) {
 783                                         val = 0.0f;
 784                                         error_details = "Floating-point constant is outside the range of the type 'float'";
 785                                         Report.Error (594, Location, error_details);
 786                                 }
 787                                 break;
 788
 789                         case Token.LITERAL_DOUBLE:
 790                         case Token.NONE:
 791                                 t = Token.LITERAL_DOUBLE;
 792                                 try {
 793                                         val = System.Double.Parse (s, styles, csharp_format_info);
 794                                 } catch (OverflowException) {
 795                                         val = 0.0;
 796                                         error_details = "Floating-point constant is outside the range of the type 'double'";
 797                                         Report.Error (594, Location, error_details);
 798                                 }
 799                                 break;
 800                         }
 801                         return t;
 802                 }
 803
 804                 int handle_hex ()
 805                 {
 806                         int d;
 807                         ulong ul;
 808
 809                         getChar ();
 810                         while ((d = peekChar ()) != -1){
 811                                 if (is_hex (d)){
 812                                         number_builder [number_pos++] = (char) d;
 813                                         getChar ();
 814                                 } else
 815                                         break;
 816                         }
 817
 818                         string s = new String (number_builder, 0, number_pos);
 819                         try {
 820                                 if (number_pos <= 8)
 821                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
 822                                 else
 823                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
 824                         } catch (OverflowException){
 825                                 error_details = "Integral constant is too large";
 826                                 Report.Error (1021, Location, error_details);
 827                                 val = 0ul;
 828                                 return Token.LITERAL_INTEGER;
 829                         }
 830
 831                         return integer_type_suffix (ul, peekChar ());
 832                 }
 833
 834                 //
 835                 // Invoked if we know we have .digits or digits
 836                 //
 837                 int is_number (int c)
 838                 {
 839                         bool is_real = false;
 840                         int type;
 841
 842                         number_pos = 0;
 843
 844                         if (c >= '0' && c <= '9'){
 845                                 if (c == '0'){
 846                                         int peek = peekChar ();
 847
 848                                         if (peek == 'x' || peek == 'X')
 849                                                 return handle_hex ();
 850                                 }
 851                                 decimal_digits (c);
 852                                 c = getChar ();
 853                         }
 854
 855                         //
 856                         // We need to handle the case of
 857                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
 858                         //
 859                         if (c == '.'){
 860                                 if (decimal_digits ('.')){
 861                                         is_real = true;
 862                                         c = getChar ();
 863                                 } else {
 864                                         putback ('.');
 865                                         number_pos--;
 866                                         return adjust_int (-1);
 867                                 }
 868                         }
 869
 870                         if (c == 'e' || c == 'E'){
 871                                 is_real = true;
 872                                 if (number_pos == max_number_size)
 873                                         Error_NumericConstantTooLong ();
 874                                 number_builder [number_pos++] = 'e';
 875                                 c = getChar ();
 876
 877                                 if (c == '+'){
 878                                         if (number_pos == max_number_size)
 879                                                 Error_NumericConstantTooLong ();
 880                                         number_builder [number_pos++] = '+';
 881                                         c = -1;
 882                                 } else if (c == '-') {
 883                                         if (number_pos == max_number_size)
 884                                                 Error_NumericConstantTooLong ();
 885                                         number_builder [number_pos++] = '-';
 886                                         c = -1;
 887                                 } else {
 888                                         if (number_pos == max_number_size)
 889                                                 Error_NumericConstantTooLong ();
 890                                         number_builder [number_pos++] = '+';
 891                                 }
 892
 893                                 decimal_digits (c);
 894                                 c = getChar ();
 895                         }
 896
 897                         type = real_type_suffix (c);
 898                         if (type == Token.NONE && !is_real){
 899                                 putback (c);
 900                                 return adjust_int (c);
 901                         } else
 902                                 is_real = true;
 903
 904                         if (type == Token.NONE){
 905                                 putback (c);
 906                         }
 907
 908                         if (is_real)
 909                                 return adjust_real (type);
 910
 911                         Console.WriteLine ("This should not be reached");
 912                         throw new Exception ("Is Number should never reach this point");
 913                 }
 914
 915                 //
 916                 // Accepts exactly count (4 or 8) hex, no more no less
 917                 //
 918                 int getHex (int count, out bool error)
 919                 {
 920                         int i;
 921                         int total = 0;
 922                         int c;
 923                         int top = count != -1 ? count : 4;
 924
 925                         getChar ();
 926                         error = false;
 927                         for (i = 0; i < top; i++){
 928                                 c = getChar ();
 929
 930                                 if (c >= '0' && c <= '9')
 931                                         c = (int) c - (int) '0';
 932                                 else if (c >= 'A' && c <= 'F')
 933                                         c = (int) c - (int) 'A' + 10;
 934                                 else if (c >= 'a' && c <= 'f')
 935                                         c = (int) c - (int) 'a' + 10;
 936                                 else {
 937                                         error = true;
 938                                         return 0;
 939                                 }
 940
 941                                 total = (total * 16) + c;
 942                                 if (count == -1){
 943                                         int p = peekChar ();
 944                                         if (p == -1)
 945                                                 break;
 946                                         if (!is_hex ((char)p))
 947                                                 break;
 948                                 }
 949                         }
 950                         return total;
 951                 }
 952
 953                 int escape (int c)
 954                 {
 955                         bool error;
 956                         int d;
 957                         int v;
 958
 959                         d = peekChar ();
 960                         if (c != '\\')
 961                                 return c;
 962
 963                         switch (d){
 964                         case 'a':
 965                                 v = '\a'; break;
 966                         case 'b':
 967                                 v = '\b'; break;
 968                         case 'n':
 969                                 v = '\n'; break;
 970                         case 't':
 971                                 v = '\t'; break;
 972                         case 'v':
 973                                 v = '\v'; break;
 974                         case 'r':
 975                                 v = '\r'; break;
 976                         case '\\':
 977                                 v = '\\'; break;
 978                         case 'f':
 979                                 v = '\f'; break;
 980                         case '0':
 981                                 v = 0; break;
 982                         case '"':
 983                                 v = '"'; break;
 984                         case '\'':
 985                                 v = '\''; break;
 986                         case 'x':
 987                                 v = getHex (-1, out error);
 988                                 if (error)
 989                                         goto default;
 990                                 return v;
 991                         case 'u':
 992                                 v = getHex (4, out error);
 993                                 if (error)
 994                                         goto default;
 995                                 return v;
 996                         case 'U':
 997                                 v = getHex (8, out error);
 998                                 if (error)
 999                                         goto default;
1000                                 return v;
1001                         default:
1002                                 Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);
1003                                 return d;
1004                         }
1005                         getChar ();
1006                         return v;
1007                 }
1008
1009                 int getChar ()
1010                 {
1011                         if (putback_char != -1){
1012                                 int x = putback_char;
1013                                 putback_char = -1;
1014
1015                                 return x;
1016                         }
1017                         return reader.Read ();
1018                 }
1019
1020                 int peekChar ()
1021                 {
1022                         if (putback_char != -1)
1023                                 return putback_char;
1024                         putback_char = reader.Read ();
1025                         return putback_char;
1026                 }
1027
1028                 int peekChar2 ()
1029                 {
1030                         if (putback_char != -1)
1031                                 return putback_char;
1032                         return reader.Peek ();
1033                 }
1034
1035                 void putback (int c)
1036                 {
1037                         if (putback_char != -1){
1038                                 Console.WriteLine ("Col: " + col);
1039                                 Console.WriteLine ("Row: " + line);
1040                                 Console.WriteLine ("Name: " + ref_name.Name);
1041                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1042                                 throw new Exception ("This should not happen putback on putback");
1043                         }
1044                         putback_char = c;
1045                 }
1046
1047                 public bool advance ()
1048                 {
1049                         return peekChar () != -1;
1050                 }
1051
1052                 public Object Value {
1053                         get {
1054                                 return val;
1055                         }
1056                 }
1057
1058                 public Object value ()
1059                 {
1060                         return val;
1061                 }
1062
1063                 bool IsCastToken (int token)
1064                 {
1065                         switch (token) {
1066                         case Token.BANG:
1067                         case Token.TILDE:
1068                         case Token.IDENTIFIER:
1069                         case Token.LITERAL_INTEGER:
1070                         case Token.LITERAL_FLOAT:
1071                         case Token.LITERAL_DOUBLE:
1072                         case Token.LITERAL_DECIMAL:
1073                         case Token.LITERAL_CHARACTER:
1074                         case Token.LITERAL_STRING:
1075                         case Token.BASE:
1076                         case Token.CHECKED:
1077                         case Token.FALSE:
1078                         case Token.FIXED:
1079                         case Token.NEW:
1080                         case Token.NULL:
1081                         case Token.SIZEOF:
1082                         case Token.THIS:
1083                         case Token.THROW:
1084                         case Token.TRUE:
1085                         case Token.TYPEOF:
1086                         case Token.UNCHECKED:
1087                         case Token.UNSAFE:
1088
1089                                 //
1090                                 // These can be part of a member access
1091                                 //
1092                         case Token.INT:
1093                         case Token.UINT:
1094                         case Token.SHORT:
1095                         case Token.USHORT:
1096                         case Token.LONG:
1097                         case Token.ULONG:
1098                         case Token.DOUBLE:
1099                         case Token.FLOAT:
1100                         case Token.CHAR:
1101                                 return true;
1102
1103                         default:
1104                                 return false;
1105                         }
1106                 }
1107
1108                 public int token ()
1109                 {
1110                         current_token = xtoken ();
1111                         return current_token;
1112                 }
1113
1114                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1115
1116                 void get_cmd_arg (out string cmd, out string arg)
1117                 {
1118                         int c;
1119
1120                         tokens_seen = false;
1121                         arg = "";
1122                         static_cmd_arg.Length = 0;
1123
1124                         // skip over white space
1125                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1126                                 ;
1127
1128
1129                         while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
1130                                 if (is_identifier_part_character ((char) c)){
1131                                         static_cmd_arg.Append ((char) c);
1132                                         c = getChar ();
1133                                 } else {
1134                                         putback (c);
1135                                         break;
1136                                 }
1137                         }
1138
1139                         cmd = static_cmd_arg.ToString ();
1140
1141                         if (c == '\n'){
1142                                 line++;
1143                                 ref_line++;
1144                                 return;
1145                         } else if (c == '\r')
1146                                 col = 0;
1147
1148                         // skip over white space
1149                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1150                                 ;
1151
1152                         if (c == '\n'){
1153                                 line++;
1154                                 ref_line++;
1155                                 return;
1156                         } else if (c == '\r'){
1157                                 col = 0;
1158                                 return;
1159                         }
1160
1161                         static_cmd_arg.Length = 0;
1162                         static_cmd_arg.Append ((char) c);
1163
1164                         while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
1165                                 static_cmd_arg.Append ((char) c);
1166                         }
1167
1168                         if (c == '\n'){
1169                                 line++;
1170                                 ref_line++;
1171                         } else if (c == '\r')
1172                                 col = 0;
1173                         arg = static_cmd_arg.ToString ().Trim ();
1174                 }
1175
1176                 //
1177                 // Handles the #line directive
1178                 //
1179                 bool PreProcessLine (string arg)
1180                 {
1181                         if (arg == "")
1182                                 return false;
1183
1184                         if (arg == "default"){
1185                                 ref_line = line;
1186                                 ref_name = file_name;
1187                                 Location.Push (ref_name);
1188                                 return true;
1189                         } else if (arg == "hidden"){
1190                                 //
1191                                 // We ignore #line hidden
1192                                 //
1193                                 return true;
1194                         }
1195
1196                         try {
1197                                 int pos;
1198
1199                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1200                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1201                                         pos++;
1202
1203                                         char [] quotes = { '\"' };
1204
1205                                         string name = arg.Substring (pos). Trim (quotes);
1206                                         ref_name = Location.LookupFile (name);
1207                                         file_name.HasLineDirective = true;
1208                                         ref_name.HasLineDirective = true;
1209                                         Location.Push (ref_name);
1210                                 } else {
1211                                         ref_line = System.Int32.Parse (arg);
1212                                 }
1213                         } catch {
1214                                 return false;
1215                         }
1216
1217                         return true;
1218                 }
1219
1220                 //
1221                 // Handles #define and #undef
1222                 //
1223                 void PreProcessDefinition (bool is_define, string arg)
1224                 {
1225                         if (arg == "" || arg == "true" || arg == "false"){
1226                                 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
1227                                 return;
1228                         }
1229
1230                         char[] whitespace = { ' ', '\t' };
1231                         if (arg.IndexOfAny (whitespace) != -1){
1232                                 Report.Error (1025, Location, "Single-line comment or end-of-line expected");
1233                                 return;
1234                         }
1235
1236                         if (!is_identifier_start_character (arg [0]))
1237                                 Report.Error (1001, Location, "Identifier expected: " + arg);
1238
1239                         foreach (char c in arg.Substring (1)){
1240                                 if (!is_identifier_part_character (c)){
1241                                         Report.Error (1001, Location, "Identifier expected: " + arg);
1242                                         return;
1243                                 }
1244                         }
1245
1246                         if (is_define){
1247                                 if (defines == null)
1248                                         defines = new Hashtable ();
1249                                 define (arg);
1250                         } else {
1251                                 if (defines == null)
1252                                         return;
1253                                 if (defines.Contains (arg))
1254                                         defines.Remove (arg);
1255                         }
1256                 }
1257
1258                 /// <summary>
1259                 /// Handles #pragma directive
1260                 /// </summary>
1261                 void PreProcessPragma (string arg)
1262                 {
1263                         const string disable = "warning disable";
1264                         const string restore = "warning restore";
1265
1266                         if (arg == disable) {
1267                                 Report.RegisterWarningRegion (Location).WarningDisable (line);
1268                                 return;
1269                         }
1270
1271                         if (arg == restore) {
1272                                 Report.RegisterWarningRegion (Location).WarningEnable (line);
1273                                 return;
1274                         }
1275
1276                         if (arg.StartsWith (disable)) {
1277                                 int[] codes = ParseNumbers (arg.Substring (disable.Length));
1278                                 foreach (int code in codes) {
1279                                         if (code != 0)
1280                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1281                                 }
1282                                 return;
1283                         }
1284
1285                         if (arg.StartsWith (restore)) {
1286                                 int[] codes = ParseNumbers (arg.Substring (restore.Length));
1287                                 foreach (int code in codes) {
1288                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1289                                 }
1290                                 return;
1291                         }
1292
1293                         return;
1294                 }
1295
1296                 int[] ParseNumbers (string text)
1297                 {
1298                         string[] string_array = text.Split (',');
1299                         int[] values = new int [string_array.Length];
1300                         int index = 0;
1301                         foreach (string string_code in string_array) {
1302                                 try {
1303                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1304                                 }
1305                                 catch (FormatException) {
1306                                         Report.Warning (1692, Location, "Invalid number");
1307                                 }
1308                         }
1309                         return values;
1310                 }
1311
1312                 bool eval_val (string s)
1313                 {
1314                         if (s == "true")
1315                                 return true;
1316                         if (s == "false")
1317                                 return false;
1318
1319                         if (defines == null)
1320                                 return false;
1321                         if (defines.Contains (s))
1322                                 return true;
1323
1324                         return false;
1325                 }
1326
1327                 bool pp_primary (ref string s)
1328                 {
1329                         s = s.Trim ();
1330                         int len = s.Length;
1331
1332                         if (len > 0){
1333                                 char c = s [0];
1334
1335                                 if (c == '('){
1336                                         s = s.Substring (1);
1337                                         bool val = pp_expr (ref s);
1338                                         if (s.Length > 0 && s [0] == ')'){
1339                                                 s = s.Substring (1);
1340                                                 return val;
1341                                         }
1342                                         Error_InvalidDirective ();
1343                                         return false;
1344                                 }
1345
1346                                 if (is_identifier_start_character (c)){
1347                                         int j = 1;
1348
1349                                         while (j < len){
1350                                                 c = s [j];
1351
1352                                                 if (is_identifier_part_character (c)){
1353                                                         j++;
1354                                                         continue;
1355                                                 }
1356                                                 bool v = eval_val (s.Substring (0, j));
1357                                                 s = s.Substring (j);
1358                                                 return v;
1359                                         }
1360                                         bool vv = eval_val (s);
1361                                         s = "";
1362                                         return vv;
1363                                 }
1364                         }
1365                         Error_InvalidDirective ();
1366                         return false;
1367                 }
1368
1369                 bool pp_unary (ref string s)
1370                 {
1371                         s = s.Trim ();
1372                         int len = s.Length;
1373
1374                         if (len > 0){
1375                                 if (s [0] == '!'){
1376                                         if (len > 1 && s [1] == '='){
1377                                                 Error_InvalidDirective ();
1378                                                 return false;
1379                                         }
1380                                         s = s.Substring (1);
1381                                         return ! pp_primary (ref s);
1382                                 } else
1383                                         return pp_primary (ref s);
1384                         } else {
1385                                 Error_InvalidDirective ();
1386                                 return false;
1387                         }
1388                 }
1389
1390                 bool pp_eq (ref string s)
1391                 {
1392                         bool va = pp_unary (ref s);
1393
1394                         s = s.Trim ();
1395                         int len = s.Length;
1396                         if (len > 0){
1397                                 if (s [0] == '='){
1398                                         if (len > 2 && s [1] == '='){
1399                                                 s = s.Substring (2);
1400                                                 return va == pp_unary (ref s);
1401                                         } else {
1402                                                 Error_InvalidDirective ();
1403                                                 return false;
1404                                         }
1405                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
1406                                         s = s.Substring (2);
1407
1408                                         return va != pp_unary (ref s);
1409
1410                                 }
1411                         }
1412
1413                         return va;
1414
1415                 }
1416
1417                 bool pp_and (ref string s)
1418                 {
1419                         bool va = pp_eq (ref s);
1420
1421                         s = s.Trim ();
1422                         int len = s.Length;
1423                         if (len > 0){
1424                                 if (s [0] == '&'){
1425                                         if (len > 2 && s [1] == '&'){
1426                                                 s = s.Substring (2);
1427                                                 return (va & pp_eq (ref s));
1428                                         } else {
1429                                                 Error_InvalidDirective ();
1430                                                 return false;
1431                                         }
1432                                 }
1433                         }
1434                         return va;
1435                 }
1436
1437                 //
1438                 // Evaluates an expression for `#if' or `#elif'
1439                 //
1440                 bool pp_expr (ref string s)
1441                 {
1442                         bool va = pp_and (ref s);
1443                         s = s.Trim ();
1444                         int len = s.Length;
1445                         if (len > 0){
1446                                 char c = s [0];
1447
1448                                 if (c == '|'){
1449                                         if (len > 2 && s [1] == '|'){
1450                                                 s = s.Substring (2);
1451                                                 return va | pp_expr (ref s);
1452                                         } else {
1453                                                 Error_InvalidDirective ();
1454                                                 return false;
1455                                         }
1456                                 }
1457                         }
1458
1459                         return va;
1460                 }
1461
1462                 bool eval (string s)
1463                 {
1464                         bool v = pp_expr (ref s);
1465                         s = s.Trim ();
1466                         if (s.Length != 0){
1467                                 Error_InvalidDirective ();
1468                                 return false;
1469                         }
1470
1471                         return v;
1472                 }
1473
1474                 void Error_InvalidDirective ()
1475                 {
1476                         Report.Error (1517, Location, "Invalid pre-processor directive");
1477                 }
1478
1479                 void Error_UnexpectedDirective (string extra)
1480                 {
1481                         Report.Error (
1482                                 1028, Location,
1483                                 "Unexpected processor directive (" + extra + ")");
1484                 }
1485
1486                 void Error_TokensSeen ()
1487                 {
1488                         Report.Error (
1489                                 1032, Location,
1490                                 "Cannot define or undefine pre-processor symbols after a token in the file");
1491                 }
1492
1493                 //
1494                 // if true, then the code continues processing the code
1495                 // if false, the code stays in a loop until another directive is
1496                 // reached.
1497                 //
1498                 bool handle_preprocessing_directive (bool caller_is_taking)
1499                 {
1500                         string cmd, arg;
1501                         bool region_directive = false;
1502
1503                         get_cmd_arg (out cmd, out arg);
1504
1505                         // Eat any trailing whitespaces and single-line comments
1506                         if (arg.IndexOf ("//") != -1)
1507                                 arg = arg.Substring (0, arg.IndexOf ("//"));
1508                         arg = arg.TrimEnd (' ', '\t');
1509
1510                         //
1511                         // The first group of pre-processing instructions is always processed
1512                         //
1513                         switch (cmd){
1514                         case "pragma":
1515                                 if (RootContext.Version == LanguageVersion.ISO_1) {
1516                                         Report.FeatureIsNotStandardized (Location, "#pragma");
1517                                         return caller_is_taking;
1518                                 }
1519
1520                                 PreProcessPragma (arg);
1521                                 return caller_is_taking;
1522
1523                         case "line":
1524                                 if (!PreProcessLine (arg))
1525                                         Report.Error (
1526                                                 1576, Location,
1527                                                 "Argument to #line directive is missing or invalid");
1528                                 return caller_is_taking;
1529
1530                         case "region":
1531                                 region_directive = true;
1532                                 arg = "true";
1533                                 goto case "if";
1534
1535                         case "endregion":
1536                                 region_directive = true;
1537                                 goto case "endif";
1538
1539                         case "if":
1540                                 if (arg == ""){
1541                                         Error_InvalidDirective ();
1542                                         return true;
1543                                 }
1544                                 bool taking = false;
1545                                 if (ifstack == null)
1546                                         ifstack = new Stack ();
1547
1548                                 if (ifstack.Count == 0){
1549                                         taking = true;
1550                                 } else {
1551                                         int state = (int) ifstack.Peek ();
1552                                         if ((state & TAKING) != 0)
1553                                                 taking = true;
1554                                 }
1555
1556                                 if (eval (arg) && taking){
1557                                         int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;
1558                                         if (region_directive)
1559                                                 push |= REGION;
1560                                         ifstack.Push (push);
1561                                         return true;
1562                                 } else {
1563                                         int push = (taking ? PARENT_TAKING : 0);
1564                                         if (region_directive)
1565                                                 push |= REGION;
1566                                         ifstack.Push (push);
1567                                         return false;
1568                                 }
1569
1570                         case "endif":
1571                                 if (ifstack == null || ifstack.Count == 0){
1572                                         Error_UnexpectedDirective ("no #if for this #endif");
1573                                         return true;
1574                                 } else {
1575                                         int pop = (int) ifstack.Pop ();
1576
1577                                         if (region_directive && ((pop & REGION) == 0))
1578                                                 Report.Error (1027, Location, "#endif directive expected");
1579                                         else if (!region_directive && ((pop & REGION) != 0))
1580                                                 Report.Error (1038, Location, "#endregion directive expected");
1581
1582                                         if (ifstack.Count == 0)
1583                                                 return true;
1584                                         else {
1585                                                 int state = (int) ifstack.Peek ();
1586
1587                                                 if ((state & TAKING) != 0)
1588                                                         return true;
1589                                                 else
1590                                                         return false;
1591                                         }
1592                                 }
1593
1594                         case "elif":
1595                                 if (ifstack == null || ifstack.Count == 0){
1596                                         Error_UnexpectedDirective ("no #if for this #elif");
1597                                         return true;
1598                                 } else {
1599                                         int state = (int) ifstack.Peek ();
1600
1601                                         if ((state & REGION) != 0) {
1602                                                 Report.Error (1038, Location, "#endregion directive expected");
1603                                                 return true;
1604                                         }
1605
1606                                         if ((state & ELSE_SEEN) != 0){
1607                                                 Error_UnexpectedDirective ("#elif not valid after #else");
1608                                                 return true;
1609                                         }
1610
1611                                         if ((state & (TAKEN_BEFORE | TAKING)) != 0)
1612                                                 return false;
1613
1614                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
1615                                                 state = (int) ifstack.Pop ();
1616                                                 ifstack.Push (state | TAKING | TAKEN_BEFORE);
1617                                                 return true;
1618                                         } else
1619                                                 return false;
1620                                 }
1621
1622                         case "else":
1623                                 if (ifstack == null || ifstack.Count == 0){
1624                                         Report.Error (
1625                                                 1028, Location,
1626                                                 "Unexpected processor directive (no #if for this #else)");
1627                                         return true;
1628                                 } else {
1629                                         int state = (int) ifstack.Peek ();
1630
1631                                         if ((state & REGION) != 0) {
1632                                                 Report.Error (1038, Location, "#endregion directive expected");
1633                                                 return true;
1634                                         }
1635
1636                                         if ((state & ELSE_SEEN) != 0){
1637                                                 Error_UnexpectedDirective ("#else within #else");
1638                                                 return true;
1639                                         }
1640
1641                                         ifstack.Pop ();
1642
1643                                         bool ret;
1644                                         if ((state & TAKEN_BEFORE) == 0){
1645                                                 ret = ((state & PARENT_TAKING) != 0);
1646                                         } else
1647                                                 ret = false;
1648
1649                                         if (ret)
1650                                                 state |= TAKING;
1651                                         else
1652                                                 state &= ~TAKING;
1653
1654                                         ifstack.Push (state | ELSE_SEEN);
1655
1656                                         return ret;
1657                                 }
1658                         }
1659
1660                         //
1661                         // These are only processed if we are in a `taking' block
1662                         //
1663                         if (!caller_is_taking)
1664                                 return false;
1665
1666                         switch (cmd){
1667                         case "define":
1668                                 if (any_token_seen){
1669                                         Error_TokensSeen ();
1670                                         return true;
1671                                 }
1672                                 PreProcessDefinition (true, arg);
1673                                 return true;
1674
1675                         case "undef":
1676                                 if (any_token_seen){
1677                                         Error_TokensSeen ();
1678                                         return true;
1679                                 }
1680                                 PreProcessDefinition (false, arg);
1681                                 return true;
1682
1683                         case "error":
1684                                 Report.Error (1029, Location, "#error: '" + arg + "'");
1685                                 return true;
1686
1687                         case "warning":
1688                                 Report.Warning (1030, Location, "#warning: '{0}'", arg);
1689                                 return true;
1690                         }
1691
1692                         Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");
1693                         return true;
1694
1695                 }
1696
1697                 private int consume_string (bool quoted)
1698                 {
1699                         int c;
1700                         string_builder.Length = 0;
1701
1702                         while ((c = getChar ()) != -1){
1703                                 if (c == '"'){
1704                                         if (quoted && peekChar () == '"'){
1705                                                 string_builder.Append ((char) c);
1706                                                 getChar ();
1707                                                 continue;
1708                                         } else {
1709                                                 val = string_builder.ToString ();
1710                                                 return Token.LITERAL_STRING;
1711                                         }
1712                                 }
1713
1714                                 if (c == '\n'){
1715                                         if (!quoted)
1716                                                 Report.Error (1010, Location, "Newline in constant");
1717                                         line++;
1718                                         ref_line++;
1719                                         col = 0;
1720                                 } else
1721                                         col++;
1722
1723                                 if (!quoted){
1724                                         c = escape (c);
1725                                         if (c == -1)
1726                                                 return Token.ERROR;
1727                                 }
1728                                 string_builder.Append ((char) c);
1729                         }
1730
1731                         Report.Error (1039, Location, "Unterminated string literal");
1732                         return Token.EOF;
1733                 }
1734
1735                 private int consume_identifier (int s)
1736                 {
1737                         int res = consume_identifier (s, false);
1738
1739                         if (res == Token.PARTIAL) {
1740                                 // Save current position and parse next token.
1741                                 int old = reader.Position;
1742                                 int old_putback = putback_char;
1743
1744                                 putback_char = -1;
1745
1746                                 int next_token = token ();
1747                                 bool ok = (next_token == Token.CLASS) ||
1748                                         (next_token == Token.STRUCT) ||
1749                                         (next_token == Token.INTERFACE);
1750
1751                                 reader.Position = old;
1752                                 putback_char = old_putback;
1753
1754                                 if (ok)
1755                                         return res;
1756                                 else {
1757                                         val = "partial";
1758                                         return Token.IDENTIFIER;
1759                                 }
1760                         }
1761
1762                         return res;
1763                 }
1764
1765                 private int consume_identifier (int s, bool quoted)
1766                 {
1767                         int pos = 1;
1768                         int c;
1769
1770                         id_builder [0] = (char) s;
1771
1772                         while ((c = reader.Read ()) != -1) {
1773                                 if (is_identifier_part_character ((char) c)){
1774                                         if (pos == max_id_size){
1775                                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
1776                                                 return Token.ERROR;
1777                                         }
1778
1779                                         id_builder [pos++] = (char) c;
1780                                         putback_char = -1;
1781                                         col++;
1782                                 } else {
1783                                         putback_char = c;
1784                                         break;
1785                                 }
1786                         }
1787
1788                         //
1789                         // Optimization: avoids doing the keyword lookup
1790                         // on uppercase letters and _
1791                         //
1792                         if (!quoted && (s >= 'a' || s == '_')){
1793                                 int keyword = GetKeyword (id_builder, pos);
1794                                 if (keyword != -1)
1795                                         return keyword;
1796                         }
1797
1798                         //
1799                         // Keep identifiers in an array of hashtables to avoid needless
1800                         // allocations
1801                         //
1802
1803                         if (identifiers [pos] != null) {
1804                                 val = identifiers [pos][id_builder];
1805                                 if (val != null) {
1806                                         return Token.IDENTIFIER;
1807                                 }
1808                         }
1809                         else
1810                                 identifiers [pos] = new CharArrayHashtable (pos);
1811
1812                         val = new String (id_builder, 0, pos);
1813
1814                         char [] chars = new char [pos];
1815                         Array.Copy (id_builder, chars, pos);
1816
1817                         identifiers [pos] [chars] = val;
1818
1819                         return Token.IDENTIFIER;
1820                 }
1821
1822                 public int xtoken ()
1823                 {
1824                         int t;
1825                         bool doread = false;
1826                         int c;
1827
1828                         val = null;
1829                         // optimization: eliminate col and implement #directive semantic correctly.
1830                         for (;(c = getChar ()) != -1; col++) {
1831                                 if (c == ' ')
1832                                         continue;
1833
1834                                 if (c == '\t') {
1835                                         col = (((col + 8) / 8) * 8) - 1;
1836                                         continue;
1837                                 }
1838
1839                                 if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0)
1840                                         continue;
1841
1842                                 if (c == '\r') {
1843                                         if (peekChar () == '\n')
1844                                                 getChar ();
1845
1846                                         line++;
1847                                         ref_line++;
1848                                         col = 0;
1849                                         any_token_seen |= tokens_seen;
1850                                         tokens_seen = false;
1851                                         continue;
1852                                 }
1853
1854                                 // Handle double-slash comments.
1855                                 if (c == '/'){
1856                                         int d = peekChar ();
1857
1858                                         if (d == '/'){
1859                                                 getChar ();
1860                                                 while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
1861                                                         col++;
1862                                                 if (d == '\n'){
1863                                                         line++;
1864                                                         ref_line++;
1865                                                         col = 0;
1866                                                 }
1867                                                 any_token_seen |= tokens_seen;
1868                                                 tokens_seen = false;
1869                                                 continue;
1870                                         } else if (d == '*'){
1871                                                 getChar ();
1872
1873                                                 while ((d = getChar ()) != -1){
1874                                                         if (d == '*' && peekChar () == '/'){
1875                                                                 getChar ();
1876                                                                 col++;
1877                                                                 break;
1878                                                         }
1879                                                         if (d == '\n'){
1880                                                                 line++;
1881                                                                 ref_line++;
1882                                                                 col = 0;
1883                                                                 any_token_seen |= tokens_seen;
1884                                                                 tokens_seen = false;
1885                                                         }
1886                                                 }
1887                                                 continue;
1888                                         }
1889                                         goto is_punct_label;
1890                                 }
1891
1892
1893                                 if (is_identifier_start_character ((char)c)){
1894                                         tokens_seen = true;
1895                                         return consume_identifier (c);
1896                                 }
1897
1898                         is_punct_label:
1899                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
1900                                         tokens_seen = true;
1901                                         if (doread){
1902                                                 getChar ();
1903                                                 col++;
1904                                         }
1905                                         return t;
1906                                 }
1907
1908                                 // white space
1909                                 if (c == '\n'){
1910                                         line++;
1911                                         ref_line++;
1912                                         col = 0;
1913                                         any_token_seen |= tokens_seen;
1914                                         tokens_seen = false;
1915                                         continue;
1916                                 }
1917
1918                                 if (c >= '0' && c <= '9'){
1919                                         tokens_seen = true;
1920                                         return is_number (c);
1921                                 }
1922
1923                                 if (c == '.'){
1924                                         tokens_seen = true;
1925                                         int peek = peekChar ();
1926                                         if (peek >= '0' && peek <= '9')
1927                                                 return is_number (c);
1928                                         return Token.DOT;
1929                                 }
1930
1931                                 /* For now, ignore pre-processor commands */
1932                                 // FIXME: In C# the '#' is not limited to appear
1933                                 // on the first column.
1934                                 if (c == '#' && !tokens_seen){
1935                                         bool cont = true;
1936
1937                                 start_again:
1938
1939                                         cont = handle_preprocessing_directive (cont);
1940
1941                                         if (cont){
1942                                                 col = 0;
1943                                                 continue;
1944                                         }
1945                                         col = 1;
1946
1947                                         bool skipping = false;
1948                                         for (;(c = getChar ()) != -1; col++){
1949                                                 if (c == '\n'){
1950                                                         col = 0;
1951                                                         line++;
1952                                                         ref_line++;
1953                                                         skipping = false;
1954                                                 } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
1955                                                         continue;
1956                                                 else if (c != '#')
1957                                                         skipping = true;
1958                                                 if (c == '#' && !skipping)
1959                                                         goto start_again;
1960                                         }
1961                                         any_token_seen |= tokens_seen;
1962                                         tokens_seen = false;
1963                                         if (c == -1)
1964                                                 Report.Error (1027, Location, "#endif/#endregion expected");
1965                                         continue;
1966                                 }
1967
1968                                 if (c == '"')
1969                                         return consume_string (false);
1970
1971                                 if (c == '\''){
1972                                         c = getChar ();
1973                                         tokens_seen = true;
1974                                         if (c == '\''){
1975                                                 error_details = "Empty character literal";
1976                                                 Report.Error (1011, Location, error_details);
1977                                                 return Token.ERROR;
1978                                         }
1979                                         c = escape (c);
1980                                         if (c == -1)
1981                                                 return Token.ERROR;
1982                                         val = new System.Char ();
1983                                         val = (char) c;
1984                                         c = getChar ();
1985
1986                                         if (c != '\''){
1987                                                 error_details = "Too many characters in character literal";
1988                                                 Report.Error (1012, Location, error_details);
1989
1990                                                 // Try to recover, read until newline or next "'"
1991                                                 while ((c = getChar ()) != -1){
1992                                                         if (c == '\n' || c == '\''){
1993                                                                 line++;
1994                                                                 ref_line++;
1995                                                                 col = 0;
1996                                                                 break;
1997                                                         } else
1998                                                                 col++;
1999
2000                                                 }
2001                                                 return Token.ERROR;
2002                                         }
2003                                         return Token.LITERAL_CHARACTER;
2004                                 }
2005
2006                                 if (c == '@') {
2007                                         c = getChar ();
2008                                         if (c == '"') {
2009                                                 tokens_seen = true;
2010                                                 return consume_string (true);
2011                                         } else if (is_identifier_start_character ((char) c)){
2012                                                 return consume_identifier (c, true);
2013                                         } else {
2014                                                 Report.Error (1033, Location, "'@' must be followed by string constant or identifier");
2015                                         }
2016                                 }
2017
2018                                 if (c == '#') {
2019                                         error_details = "Preprocessor directives must appear as the first non-whitespace " +
2020                                                 "character on a line.";
2021
2022                                         Report.Error (1040, Location, error_details);
2023
2024                                         return Token.ERROR;
2025                                 }
2026
2027                                 error_details = ((char)c).ToString ();
2028
2029                                 return Token.ERROR;
2030                         }
2031
2032                         return Token.EOF;
2033                 }
2034
2035                 public void cleanup ()
2036                 {
2037                         if (ifstack != null && ifstack.Count >= 1) {
2038                                 int state = (int) ifstack.Pop ();
2039                                 if ((state & REGION) != 0)
2040                                         Report.Error (1038, "#endregion directive expected");
2041                                 else
2042                                         Report.Error (1027, "#endif directive expected");
2043                         }
2044
2045                 }
2046         }
2047 }