mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //
   7 // Licensed under the terms of the GNU GPL
   8 //
   9 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
  10 // (C) 2004 Novell, Inc
  11 //
  12
  13 /*
  14  * TODO:
  15  *   Make sure we accept the proper Unicode ranges, per the spec.
  16  *   Report error 1032
  17 */
  18
  19 using System;
  20 using System.Text;
  21 using System.Collections;
  22 using System.IO;
  23 using System.Globalization;
  24 using System.Reflection;
  25
  26 namespace Mono.CSharp
  27 {
  28         /// <summary>
  29         ///    Tokenizer for C# source code.
  30         /// </summary>
  31
  32         public class Tokenizer : yyParser.yyInput
  33         {
  34                 SeekableStreamReader reader;
  35                 public SourceFile ref_name;
  36                 public SourceFile file_name;
  37                 public int ref_line = 1;
  38                 public int line = 1;
  39                 public int col = 1;
  40                 public int current_token;
  41                 bool handle_get_set = false;
  42                 bool handle_remove_add = false;
  43                 bool handle_assembly = false;
  44
  45                 //
  46                 // XML documentation buffer. The save point is used to divide
  47                 // comments on types and comments on members.
  48                 //
  49                 StringBuilder xml_comment_buffer;
  50
  51                 //
  52                 // See comment on XmlCommentState enumeration.
  53                 //
  54                 XmlCommentState xmlDocState = XmlCommentState.Allowed;
  55
  56                 //
  57                 // Whether tokens have been seen on this line
  58                 //
  59                 bool tokens_seen = false;
  60
  61                 //
  62                 // Whether a token has been seen on the file
  63                 // This is needed because `define' is not allowed to be used
  64                 // after a token has been seen.
  65                 //
  66                 bool any_token_seen = false;
  67
  68                 static Hashtable tokenValues;
  69
  70                 private static Hashtable TokenValueName
  71                 {
  72                         get {
  73                                 if (tokenValues == null)
  74                                         tokenValues = GetTokenValueNameHash ();
  75
  76                                 return tokenValues;
  77                         }
  78                 }
  79
  80                 private static Hashtable GetTokenValueNameHash ()
  81                 {
  82                         Type t = typeof (Token);
  83                         FieldInfo [] fields = t.GetFields ();
  84                         Hashtable hash = new Hashtable ();
  85                         foreach (FieldInfo field in fields) {
  86                                 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))
  87                                         hash.Add (field.GetValue (null), field.Name);
  88                         }
  89                         return hash;
  90                 }
  91
  92                 //
  93                 // Returns a verbose representation of the current location
  94                 //
  95                 public string location {
  96                         get {
  97                                 string det;
  98
  99                                 if (current_token == Token.ERROR)
 100                                         det = "detail: " + error_details;
 101                                 else
 102                                         det = "";
 103
 104                                 // return "Line:     "+line+" Col: "+col + "\n" +
 105                                 //       "VirtLine: "+ref_line +
 106                                 //       " Token: "+current_token + " " + det;
 107                                 string current_token_name = TokenValueName [current_token] as string;
 108                                 if (current_token_name == null)
 109                                         current_token_name = current_token.ToString ();
 110
 111                                 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,
 112                                                                                        ref_line,
 113                                                                                        col,
 114                                                                                        current_token_name,
 115                                                                                        det);
 116                         }
 117                 }
 118
 119                 public bool PropertyParsing {
 120                         get {
 121                                 return handle_get_set;
 122                         }
 123
 124                         set {
 125                                 handle_get_set = value;
 126                         }
 127                 }
 128
 129                 public bool AssemblyTargetParsing {
 130                         get {
 131                                 return handle_assembly;
 132                         }
 133
 134                         set {
 135                                 handle_assembly = value;
 136                         }
 137                 }
 138
 139                 public bool EventParsing {
 140                         get {
 141                                 return handle_remove_add;
 142                         }
 143
 144                         set {
 145                                 handle_remove_add = value;
 146                         }
 147                 }
 148
 149                 public XmlCommentState doc_state {
 150                         get { return xmlDocState; }
 151                         set {
 152                                 if (value == XmlCommentState.Allowed) {
 153                                         check_incorrect_doc_comment ();
 154                                         consume_doc_comment ();
 155                                 }
 156                                 xmlDocState = value;
 157                         }
 158                 }
 159
 160
 161                 //
 162                 // Class variables
 163                 //
 164                 static CharArrayHashtable[] keywords;
 165                 static Hashtable keywordStrings = new Hashtable ();
 166                 static NumberStyles styles;
 167                 static NumberFormatInfo csharp_format_info;
 168
 169                 //
 170                 // Values for the associated token returned
 171                 //
 172                 int putback_char;
 173                 Object val;
 174
 175                 //
 176                 // Pre-processor
 177                 //
 178                 Hashtable defines;
 179
 180                 const int TAKING        = 1;
 181                 const int TAKEN_BEFORE  = 2;
 182                 const int ELSE_SEEN     = 4;
 183                 const int PARENT_TAKING = 8;
 184                 const int REGION        = 16;
 185
 186                 //
 187                 // pre-processor if stack state:
 188                 //
 189                 Stack ifstack;
 190
 191                 static System.Text.StringBuilder string_builder;
 192
 193                 const int max_id_size = 512;
 194                 static char [] id_builder = new char [max_id_size];
 195
 196                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 197
 198                 const int max_number_size = 512;
 199                 static char [] number_builder = new char [max_number_size];
 200                 static int number_pos;
 201
 202                 //
 203                 // Details about the error encoutered by the tokenizer
 204                 //
 205                 string error_details;
 206
 207                 public string error {
 208                         get {
 209                                 return error_details;
 210                         }
 211                 }
 212
 213                 public int Line {
 214                         get {
 215                                 return ref_line;
 216                         }
 217                 }
 218
 219                 public int Col {
 220                         get {
 221                                 return col;
 222                         }
 223                 }
 224
 225                 static void AddKeyword (string kw, int token) {
 226                         keywordStrings.Add (kw, kw);
 227                         if (keywords [kw.Length] == null) {
 228                                 keywords [kw.Length] = new CharArrayHashtable (kw.Length);
 229                         }
 230                         keywords [kw.Length] [kw.ToCharArray ()] = token;
 231                 }
 232
 233                 static void InitTokens ()
 234                 {
 235                         keywords = new CharArrayHashtable [64];
 236
 237                         AddKeyword ("__arglist", Token.ARGLIST);
 238                         AddKeyword ("abstract", Token.ABSTRACT);
 239                         AddKeyword ("as", Token.AS);
 240                         AddKeyword ("add", Token.ADD);
 241                         AddKeyword ("assembly", Token.ASSEMBLY);
 242                         AddKeyword ("base", Token.BASE);
 243                         AddKeyword ("bool", Token.BOOL);
 244                         AddKeyword ("break", Token.BREAK);
 245                         AddKeyword ("byte", Token.BYTE);
 246                         AddKeyword ("case", Token.CASE);
 247                         AddKeyword ("catch", Token.CATCH);
 248                         AddKeyword ("char", Token.CHAR);
 249                         AddKeyword ("checked", Token.CHECKED);
 250                         AddKeyword ("class", Token.CLASS);
 251                         AddKeyword ("const", Token.CONST);
 252                         AddKeyword ("continue", Token.CONTINUE);
 253                         AddKeyword ("decimal", Token.DECIMAL);
 254                         AddKeyword ("default", Token.DEFAULT);
 255                         AddKeyword ("delegate", Token.DELEGATE);
 256                         AddKeyword ("do", Token.DO);
 257                         AddKeyword ("double", Token.DOUBLE);
 258                         AddKeyword ("else", Token.ELSE);
 259                         AddKeyword ("enum", Token.ENUM);
 260                         AddKeyword ("event", Token.EVENT);
 261                         AddKeyword ("explicit", Token.EXPLICIT);
 262                         AddKeyword ("extern", Token.EXTERN);
 263                         AddKeyword ("false", Token.FALSE);
 264                         AddKeyword ("finally", Token.FINALLY);
 265                         AddKeyword ("fixed", Token.FIXED);
 266                         AddKeyword ("float", Token.FLOAT);
 267                         AddKeyword ("for", Token.FOR);
 268                         AddKeyword ("foreach", Token.FOREACH);
 269                         AddKeyword ("goto", Token.GOTO);
 270                         AddKeyword ("get", Token.GET);
 271                         AddKeyword ("if", Token.IF);
 272                         AddKeyword ("implicit", Token.IMPLICIT);
 273                         AddKeyword ("in", Token.IN);
 274                         AddKeyword ("int", Token.INT);
 275                         AddKeyword ("interface", Token.INTERFACE);
 276                         AddKeyword ("internal", Token.INTERNAL);
 277                         AddKeyword ("is", Token.IS);
 278                         AddKeyword ("lock", Token.LOCK);
 279                         AddKeyword ("long", Token.LONG);
 280                         AddKeyword ("namespace", Token.NAMESPACE);
 281                         AddKeyword ("new", Token.NEW);
 282                         AddKeyword ("null", Token.NULL);
 283                         AddKeyword ("object", Token.OBJECT);
 284                         AddKeyword ("operator", Token.OPERATOR);
 285                         AddKeyword ("out", Token.OUT);
 286                         AddKeyword ("override", Token.OVERRIDE);
 287                         AddKeyword ("params", Token.PARAMS);
 288                         AddKeyword ("private", Token.PRIVATE);
 289                         AddKeyword ("protected", Token.PROTECTED);
 290                         AddKeyword ("public", Token.PUBLIC);
 291                         AddKeyword ("readonly", Token.READONLY);
 292                         AddKeyword ("ref", Token.REF);
 293                         AddKeyword ("remove", Token.REMOVE);
 294                         AddKeyword ("return", Token.RETURN);
 295                         AddKeyword ("sbyte", Token.SBYTE);
 296                         AddKeyword ("sealed", Token.SEALED);
 297                         AddKeyword ("set", Token.SET);
 298                         AddKeyword ("short", Token.SHORT);
 299                         AddKeyword ("sizeof", Token.SIZEOF);
 300                         AddKeyword ("stackalloc", Token.STACKALLOC);
 301                         AddKeyword ("static", Token.STATIC);
 302                         AddKeyword ("string", Token.STRING);
 303                         AddKeyword ("struct", Token.STRUCT);
 304                         AddKeyword ("switch", Token.SWITCH);
 305                         AddKeyword ("this", Token.THIS);
 306                         AddKeyword ("throw", Token.THROW);
 307                         AddKeyword ("true", Token.TRUE);
 308                         AddKeyword ("try", Token.TRY);
 309                         AddKeyword ("typeof", Token.TYPEOF);
 310                         AddKeyword ("uint", Token.UINT);
 311                         AddKeyword ("ulong", Token.ULONG);
 312                         AddKeyword ("unchecked", Token.UNCHECKED);
 313                         AddKeyword ("unsafe", Token.UNSAFE);
 314                         AddKeyword ("ushort", Token.USHORT);
 315                         AddKeyword ("using", Token.USING);
 316                         AddKeyword ("virtual", Token.VIRTUAL);
 317                         AddKeyword ("void", Token.VOID);
 318                         AddKeyword ("volatile", Token.VOLATILE);
 319                         AddKeyword ("while", Token.WHILE);
 320                         AddKeyword ("partial", Token.PARTIAL);
 321                 }
 322
 323                 //
 324                 // Class initializer
 325                 //
 326                 static Tokenizer ()
 327                 {
 328                         InitTokens ();
 329                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 330                         styles = NumberStyles.Float;
 331
 332                         string_builder = new System.Text.StringBuilder ();
 333                 }
 334
 335                 int GetKeyword (char[] id, int id_len)
 336                 {
 337                         /*
 338                          * Keywords are stored in an array of hashtables grouped by their
 339                          * length.
 340                          */
 341
 342                         if ((id_len >= keywords.Length) || (keywords [id_len] == null))
 343                                 return -1;
 344                         object o = keywords [id_len] [id];
 345
 346                         if (o == null)
 347                                 return -1;
 348
 349                         int res = (int) o;
 350
 351                         if (handle_get_set == false && (res == Token.GET || res == Token.SET))
 352                                 return -1;
 353                         if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
 354                                 return -1;
 355                         if (handle_assembly == false && res == Token.ASSEMBLY)
 356                                 return -1;
 357
 358                         return res;
 359
 360                 }
 361
 362                 public Location Location {
 363                         get {
 364                                 return new Location (ref_line);
 365                         }
 366                 }
 367
 368                 void define (string def)
 369                 {
 370                         if (!RootContext.AllDefines.Contains (def)){
 371                                 RootContext.AllDefines [def] = true;
 372                         }
 373                         if (defines.Contains (def))
 374                                 return;
 375                         defines [def] = true;
 376                 }
 377
 378                 public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)
 379                 {
 380                         this.ref_name = file;
 381                         this.file_name = file;
 382                         reader = input;
 383
 384                         putback_char = -1;
 385
 386                         if (defs != null){
 387                                 defines = new Hashtable ();
 388                                 foreach (string def in defs)
 389                                         define (def);
 390                         }
 391
 392                         xml_comment_buffer = new StringBuilder ();
 393
 394                         //
 395                         // FIXME: This could be `Location.Push' but we have to
 396                         // find out why the MS compiler allows this
 397                         //
 398                         Mono.CSharp.Location.Push (file);
 399                 }
 400
 401                 static bool is_identifier_start_character (char c)
 402                 {
 403                         return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c);
 404                 }
 405
 406                 static bool is_identifier_part_character (char c)
 407                 {
 408                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
 409                 }
 410
 411                 public static bool IsKeyword (string s)
 412                 {
 413                         return keywordStrings [s] != null;
 414                 }
 415
 416                 public static bool IsValidIdentifier (string s)
 417                 {
 418                         if (s == null || s.Length == 0)
 419                                 return false;
 420
 421                         if (!is_identifier_start_character (s [0]))
 422                                 return false;
 423
 424                         for (int i = 1; i < s.Length; i ++)
 425                                 if (! is_identifier_part_character (s [i]))
 426                                         return false;
 427
 428                         return true;
 429                 }
 430
 431                 int is_punct (char c, ref bool doread)
 432                 {
 433                         int d;
 434                         int t;
 435
 436                         doread = false;
 437
 438                         switch (c){
 439                         case '{':
 440                                 return Token.OPEN_BRACE;
 441                         case '}':
 442                                 return Token.CLOSE_BRACE;
 443                         case '[':
 444                                 // To block doccomment inside attribute declaration.
 445                                 if (doc_state == XmlCommentState.Allowed)
 446                                         doc_state = XmlCommentState.NotAllowed;
 447                                 return Token.OPEN_BRACKET;
 448                         case ']':
 449                                 return Token.CLOSE_BRACKET;
 450                         case '(':
 451                                 return Token.OPEN_PARENS;
 452                         case ')': {
 453                                 if (deambiguate_close_parens == 0)
 454                                         return Token.CLOSE_PARENS;
 455
 456                                 --deambiguate_close_parens;
 457
 458                                 // Save current position and parse next token.
 459                                 int old = reader.Position;
 460                                 int new_token = token ();
 461                                 reader.Position = old;
 462                                 putback_char = -1;
 463
 464                                 if (new_token == Token.OPEN_PARENS)
 465                                         return Token.CLOSE_PARENS_OPEN_PARENS;
 466                                 else if (new_token == Token.MINUS)
 467                                         return Token.CLOSE_PARENS_MINUS;
 468                                 else if (IsCastToken (new_token))
 469                                         return Token.CLOSE_PARENS_CAST;
 470                                 else
 471                                         return Token.CLOSE_PARENS_NO_CAST;
 472                         }
 473
 474                         case ',':
 475                                 return Token.COMMA;
 476                         case ':':
 477                                 return Token.COLON;
 478                         case ';':
 479                                 return Token.SEMICOLON;
 480                         case '~':
 481                                 return Token.TILDE;
 482                         case '?':
 483                                 return Token.INTERR;
 484                         }
 485
 486                         d = peekChar ();
 487                         if (c == '+'){
 488
 489                                 if (d == '+')
 490                                         t = Token.OP_INC;
 491                                 else if (d == '=')
 492                                         t = Token.OP_ADD_ASSIGN;
 493                                 else
 494                                         return Token.PLUS;
 495                                 doread = true;
 496                                 return t;
 497                         }
 498                         if (c == '-'){
 499                                 if (d == '-')
 500                                         t = Token.OP_DEC;
 501                                 else if (d == '=')
 502                                         t = Token.OP_SUB_ASSIGN;
 503                                 else if (d == '>')
 504                                         t = Token.OP_PTR;
 505                                 else
 506                                         return Token.MINUS;
 507                                 doread = true;
 508                                 return t;
 509                         }
 510
 511                         if (c == '!'){
 512                                 if (d == '='){
 513                                         doread = true;
 514                                         return Token.OP_NE;
 515                                 }
 516                                 return Token.BANG;
 517                         }
 518
 519                         if (c == '='){
 520                                 if (d == '='){
 521                                         doread = true;
 522                                         return Token.OP_EQ;
 523                                 }
 524                                 return Token.ASSIGN;
 525                         }
 526
 527                         if (c == '&'){
 528                                 if (d == '&'){
 529                                         doread = true;
 530                                         return Token.OP_AND;
 531                                 } else if (d == '='){
 532                                         doread = true;
 533                                         return Token.OP_AND_ASSIGN;
 534                                 }
 535                                 return Token.BITWISE_AND;
 536                         }
 537
 538                         if (c == '|'){
 539                                 if (d == '|'){
 540                                         doread = true;
 541                                         return Token.OP_OR;
 542                                 } else if (d == '='){
 543                                         doread = true;
 544                                         return Token.OP_OR_ASSIGN;
 545                                 }
 546                                 return Token.BITWISE_OR;
 547                         }
 548
 549                         if (c == '*'){
 550                                 if (d == '='){
 551                                         doread = true;
 552                                         return Token.OP_MULT_ASSIGN;
 553                                 }
 554                                 return Token.STAR;
 555                         }
 556
 557                         if (c == '/'){
 558                                 if (d == '='){
 559                                         doread = true;
 560                                         return Token.OP_DIV_ASSIGN;
 561                                 }
 562                                 return Token.DIV;
 563                         }
 564
 565                         if (c == '%'){
 566                                 if (d == '='){
 567                                         doread = true;
 568                                         return Token.OP_MOD_ASSIGN;
 569                                 }
 570                                 return Token.PERCENT;
 571                         }
 572
 573                         if (c == '^'){
 574                                 if (d == '='){
 575                                         doread = true;
 576                                         return Token.OP_XOR_ASSIGN;
 577                                 }
 578                                 return Token.CARRET;
 579                         }
 580
 581                         if (c == '<'){
 582                                 if (d == '<'){
 583                                         getChar ();
 584                                         d = peekChar ();
 585
 586                                         if (d == '='){
 587                                                 doread = true;
 588                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 589                                         }
 590                                         return Token.OP_SHIFT_LEFT;
 591                                 } else if (d == '='){
 592                                         doread = true;
 593                                         return Token.OP_LE;
 594                                 }
 595                                 return Token.OP_LT;
 596                         }
 597
 598                         if (c == '>'){
 599                                 if (d == '>'){
 600                                         getChar ();
 601                                         d = peekChar ();
 602
 603                                         if (d == '='){
 604                                                 doread = true;
 605                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 606                                         }
 607                                         return Token.OP_SHIFT_RIGHT;
 608                                 } else if (d == '='){
 609                                         doread = true;
 610                                         return Token.OP_GE;
 611                                 }
 612                                 return Token.OP_GT;
 613                         }
 614                         return Token.ERROR;
 615                 }
 616
 617                 int deambiguate_close_parens = 0;
 618
 619                 public void Deambiguate_CloseParens ()
 620                 {
 621                         putback (')');
 622                         deambiguate_close_parens++;
 623                 }
 624
 625                 void Error_NumericConstantTooLong ()
 626                 {
 627                         Report.Error (1021, Location, "Numeric constant too long");
 628                 }
 629
 630                 bool decimal_digits (int c)
 631                 {
 632                         int d;
 633                         bool seen_digits = false;
 634
 635                         if (c != -1){
 636                                 if (number_pos == max_number_size)
 637                                         Error_NumericConstantTooLong ();
 638                                 number_builder [number_pos++] = (char) c;
 639                         }
 640
 641                         //
 642                         // We use peekChar2, because decimal_digits needs to do a
 643                         // 2-character look-ahead (5.ToString for example).
 644                         //
 645                         while ((d = peekChar2 ()) != -1){
 646                                 if (d >= '0' && d <= '9'){
 647                                         if (number_pos == max_number_size)
 648                                                 Error_NumericConstantTooLong ();
 649                                         number_builder [number_pos++] = (char) d;
 650                                         getChar ();
 651                                         seen_digits = true;
 652                                 } else
 653                                         break;
 654                         }
 655
 656                         return seen_digits;
 657                 }
 658
 659                 bool is_hex (int e)
 660                 {
 661                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
 662                 }
 663
 664                 void hex_digits (int c)
 665                 {
 666                         if (c != -1)
 667                                 number_builder [number_pos++] = (char) c;
 668
 669                 }
 670
 671                 int real_type_suffix (int c)
 672                 {
 673                         int t;
 674
 675                         switch (c){
 676                         case 'F': case 'f':
 677                                 t =  Token.LITERAL_FLOAT;
 678                                 break;
 679                         case 'D': case 'd':
 680                                 t = Token.LITERAL_DOUBLE;
 681                                 break;
 682                         case 'M': case 'm':
 683                                  t= Token.LITERAL_DECIMAL;
 684                                 break;
 685                         default:
 686                                 return Token.NONE;
 687                         }
 688                         return t;
 689                 }
 690
 691                 int integer_type_suffix (ulong ul, int c)
 692                 {
 693                         bool is_unsigned = false;
 694                         bool is_long = false;
 695
 696                         if (c != -1){
 697                                 bool scanning = true;
 698                                 do {
 699                                         switch (c){
 700                                         case 'U': case 'u':
 701                                                 if (is_unsigned)
 702                                                         scanning = false;
 703                                                 is_unsigned = true;
 704                                                 getChar ();
 705                                                 break;
 706
 707                                         case 'l':
 708                                                 if (!is_unsigned && (RootContext.WarningLevel >= 4)){
 709                                                         //
 710                                                         // if we have not seen anything in between
 711                                                         // report this error
 712                                                         //
 713                                                         Report.Warning (78, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
 714                                                 }
 715                                                 //
 716                                                 // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
 717                                                 // work around that.
 718                                                 //
 719                                                 //goto case 'L';
 720                                                 if (is_long)
 721                                                         scanning = false;
 722                                                 is_long = true;
 723                                                 getChar ();
 724                                                 break;
 725
 726                                         case 'L':
 727                                                 if (is_long)
 728                                                         scanning = false;
 729                                                 is_long = true;
 730                                                 getChar ();
 731                                                 break;
 732
 733                                         default:
 734                                                 scanning = false;
 735                                                 break;
 736                                         }
 737                                         c = peekChar ();
 738                                 } while (scanning);
 739                         }
 740
 741                         if (is_long && is_unsigned){
 742                                 val = ul;
 743                                 return Token.LITERAL_INTEGER;
 744                         } else if (is_unsigned){
 745                                 // uint if possible, or ulong else.
 746
 747                                 if ((ul & 0xffffffff00000000) == 0)
 748                                         val = (uint) ul;
 749                                 else
 750                                         val = ul;
 751                         } else if (is_long){
 752                                 // long if possible, ulong otherwise
 753                                 if ((ul & 0x8000000000000000) != 0)
 754                                         val = ul;
 755                                 else
 756                                         val = (long) ul;
 757                         } else {
 758                                 // int, uint, long or ulong in that order
 759                                 if ((ul & 0xffffffff00000000) == 0){
 760                                         uint ui = (uint) ul;
 761
 762                                         if ((ui & 0x80000000) != 0)
 763                                                 val = ui;
 764                                         else
 765                                                 val = (int) ui;
 766                                 } else {
 767                                         if ((ul & 0x8000000000000000) != 0)
 768                                                 val = ul;
 769                                         else
 770                                                 val = (long) ul;
 771                                 }
 772                         }
 773                         return Token.LITERAL_INTEGER;
 774                 }
 775
 776                 //
 777                 // given `c' as the next char in the input decide whether
 778                 // we need to convert to a special type, and then choose
 779                 // the best representation for the integer
 780                 //
 781                 int adjust_int (int c)
 782                 {
 783                         try {
 784                                 if (number_pos > 9){
 785                                         ulong ul = (uint) (number_builder [0] - '0');
 786
 787                                         for (int i = 1; i < number_pos; i++){
 788                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
 789                                         }
 790                                         return integer_type_suffix (ul, c);
 791                                 } else {
 792                                         uint ui = (uint) (number_builder [0] - '0');
 793
 794                                         for (int i = 1; i < number_pos; i++){
 795                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
 796                                         }
 797                                         return integer_type_suffix (ui, c);
 798                                 }
 799                         } catch (OverflowException) {
 800                                 error_details = "Integral constant is too large";
 801                                 Report.Error (1021, Location, error_details);
 802                                 val = 0ul;
 803                                 return Token.LITERAL_INTEGER;
 804                         }
 805                 }
 806
 807                 int adjust_real (int t)
 808                 {
 809                         string s = new String (number_builder, 0, number_pos);
 810
 811                         switch (t){
 812                         case Token.LITERAL_DECIMAL:
 813                                 try {
 814                                         val = System.Decimal.Parse (s, styles, csharp_format_info);
 815                                 } catch (OverflowException) {
 816                                         val = 0m;
 817                                         error_details = "Floating-point constant is outside the range of the type 'decimal'";
 818                                         Report.Error (594, Location, error_details);
 819                                 }
 820                                 break;
 821                         case Token.LITERAL_FLOAT:
 822                                 try {
 823                                         val = (float) System.Double.Parse (s, styles, csharp_format_info);
 824                                 } catch (OverflowException) {
 825                                         val = 0.0f;
 826                                         error_details = "Floating-point constant is outside the range of the type 'float'";
 827                                         Report.Error (594, Location, error_details);
 828                                 }
 829                                 break;
 830
 831                         case Token.LITERAL_DOUBLE:
 832                         case Token.NONE:
 833                                 t = Token.LITERAL_DOUBLE;
 834                                 try {
 835                                         val = System.Double.Parse (s, styles, csharp_format_info);
 836                                 } catch (OverflowException) {
 837                                         val = 0.0;
 838                                         error_details = "Floating-point constant is outside the range of the type 'double'";
 839                                         Report.Error (594, Location, error_details);
 840                                 }
 841                                 break;
 842                         }
 843                         return t;
 844                 }
 845
 846                 int handle_hex ()
 847                 {
 848                         int d;
 849                         ulong ul;
 850
 851                         getChar ();
 852                         while ((d = peekChar ()) != -1){
 853                                 if (is_hex (d)){
 854                                         number_builder [number_pos++] = (char) d;
 855                                         getChar ();
 856                                 } else
 857                                         break;
 858                         }
 859
 860                         string s = new String (number_builder, 0, number_pos);
 861                         try {
 862                                 if (number_pos <= 8)
 863                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
 864                                 else
 865                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
 866                         } catch (OverflowException){
 867                                 error_details = "Integral constant is too large";
 868                                 Report.Error (1021, Location, error_details);
 869                                 val = 0ul;
 870                                 return Token.LITERAL_INTEGER;
 871                         }
 872                         catch (FormatException) {
 873                                 Report.Error (1013, Location, "Invalid number");
 874                                 val = 0ul;
 875                                 return Token.LITERAL_INTEGER;
 876                         }
 877
 878                         return integer_type_suffix (ul, peekChar ());
 879                 }
 880
 881                 //
 882                 // Invoked if we know we have .digits or digits
 883                 //
 884                 int is_number (int c)
 885                 {
 886                         bool is_real = false;
 887                         int type;
 888
 889                         number_pos = 0;
 890
 891                         if (c >= '0' && c <= '9'){
 892                                 if (c == '0'){
 893                                         int peek = peekChar ();
 894
 895                                         if (peek == 'x' || peek == 'X')
 896                                                 return handle_hex ();
 897                                 }
 898                                 decimal_digits (c);
 899                                 c = getChar ();
 900                         }
 901
 902                         //
 903                         // We need to handle the case of
 904                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
 905                         //
 906                         if (c == '.'){
 907                                 if (decimal_digits ('.')){
 908                                         is_real = true;
 909                                         c = getChar ();
 910                                 } else {
 911                                         putback ('.');
 912                                         number_pos--;
 913                                         return adjust_int (-1);
 914                                 }
 915                         }
 916
 917                         if (c == 'e' || c == 'E'){
 918                                 is_real = true;
 919                                 if (number_pos == max_number_size)
 920                                         Error_NumericConstantTooLong ();
 921                                 number_builder [number_pos++] = 'e';
 922                                 c = getChar ();
 923
 924                                 if (c == '+'){
 925                                         if (number_pos == max_number_size)
 926                                                 Error_NumericConstantTooLong ();
 927                                         number_builder [number_pos++] = '+';
 928                                         c = -1;
 929                                 } else if (c == '-') {
 930                                         if (number_pos == max_number_size)
 931                                                 Error_NumericConstantTooLong ();
 932                                         number_builder [number_pos++] = '-';
 933                                         c = -1;
 934                                 } else {
 935                                         if (number_pos == max_number_size)
 936                                                 Error_NumericConstantTooLong ();
 937                                         number_builder [number_pos++] = '+';
 938                                 }
 939
 940                                 decimal_digits (c);
 941                                 c = getChar ();
 942                         }
 943
 944                         type = real_type_suffix (c);
 945                         if (type == Token.NONE && !is_real){
 946                                 putback (c);
 947                                 return adjust_int (c);
 948                         } else
 949                                 is_real = true;
 950
 951                         if (type == Token.NONE){
 952                                 putback (c);
 953                         }
 954
 955                         if (is_real)
 956                                 return adjust_real (type);
 957
 958                         Console.WriteLine ("This should not be reached");
 959                         throw new Exception ("Is Number should never reach this point");
 960                 }
 961
 962                 //
 963                 // Accepts exactly count (4 or 8) hex, no more no less
 964                 //
 965                 int getHex (int count, out bool error)
 966                 {
 967                         int i;
 968                         int total = 0;
 969                         int c;
 970                         int top = count != -1 ? count : 4;
 971
 972                         getChar ();
 973                         error = false;
 974                         for (i = 0; i < top; i++){
 975                                 c = getChar ();
 976
 977                                 if (c >= '0' && c <= '9')
 978                                         c = (int) c - (int) '0';
 979                                 else if (c >= 'A' && c <= 'F')
 980                                         c = (int) c - (int) 'A' + 10;
 981                                 else if (c >= 'a' && c <= 'f')
 982                                         c = (int) c - (int) 'a' + 10;
 983                                 else {
 984                                         error = true;
 985                                         return 0;
 986                                 }
 987
 988                                 total = (total * 16) + c;
 989                                 if (count == -1){
 990                                         int p = peekChar ();
 991                                         if (p == -1)
 992                                                 break;
 993                                         if (!is_hex ((char)p))
 994                                                 break;
 995                                 }
 996                         }
 997                         return total;
 998                 }
 999
1000                 int escape (int c)
1001                 {
1002                         bool error;
1003                         int d;
1004                         int v;
1005
1006                         d = peekChar ();
1007                         if (c != '\\')
1008                                 return c;
1009
1010                         switch (d){
1011                         case 'a':
1012                                 v = '\a'; break;
1013                         case 'b':
1014                                 v = '\b'; break;
1015                         case 'n':
1016                                 v = '\n'; break;
1017                         case 't':
1018                                 v = '\t'; break;
1019                         case 'v':
1020                                 v = '\v'; break;
1021                         case 'r':
1022                                 v = '\r'; break;
1023                         case '\\':
1024                                 v = '\\'; break;
1025                         case 'f':
1026                                 v = '\f'; break;
1027                         case '0':
1028                                 v = 0; break;
1029                         case '"':
1030                                 v = '"'; break;
1031                         case '\'':
1032                                 v = '\''; break;
1033                         case 'x':
1034                                 v = getHex (-1, out error);
1035                                 if (error)
1036                                         goto default;
1037                                 return v;
1038                         case 'u':
1039                                 v = getHex (4, out error);
1040                                 if (error)
1041                                         goto default;
1042                                 return v;
1043                         case 'U':
1044                                 v = getHex (8, out error);
1045                                 if (error)
1046                                         goto default;
1047                                 return v;
1048                         default:
1049                                 Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);
1050                                 return d;
1051                         }
1052                         getChar ();
1053                         return v;
1054                 }
1055
1056                 int getChar ()
1057                 {
1058                         if (putback_char != -1){
1059                                 int x = putback_char;
1060                                 putback_char = -1;
1061
1062                                 return x;
1063                         }
1064                         return reader.Read ();
1065                 }
1066
1067                 int peekChar ()
1068                 {
1069                         if (putback_char != -1)
1070                                 return putback_char;
1071                         putback_char = reader.Read ();
1072                         return putback_char;
1073                 }
1074
1075                 int peekChar2 ()
1076                 {
1077                         if (putback_char != -1)
1078                                 return putback_char;
1079                         return reader.Peek ();
1080                 }
1081
1082                 void putback (int c)
1083                 {
1084                         if (putback_char != -1){
1085                                 Console.WriteLine ("Col: " + col);
1086                                 Console.WriteLine ("Row: " + line);
1087                                 Console.WriteLine ("Name: " + ref_name.Name);
1088                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1089                                 throw new Exception ("This should not happen putback on putback");
1090                         }
1091                         putback_char = c;
1092                 }
1093
1094                 public bool advance ()
1095                 {
1096                         return peekChar () != -1;
1097                 }
1098
1099                 public Object Value {
1100                         get {
1101                                 return val;
1102                         }
1103                 }
1104
1105                 public Object value ()
1106                 {
1107                         return val;
1108                 }
1109
1110                 bool IsCastToken (int token)
1111                 {
1112                         switch (token) {
1113                         case Token.BANG:
1114                         case Token.TILDE:
1115                         case Token.IDENTIFIER:
1116                         case Token.LITERAL_INTEGER:
1117                         case Token.LITERAL_FLOAT:
1118                         case Token.LITERAL_DOUBLE:
1119                         case Token.LITERAL_DECIMAL:
1120                         case Token.LITERAL_CHARACTER:
1121                         case Token.LITERAL_STRING:
1122                         case Token.BASE:
1123                         case Token.CHECKED:
1124                         case Token.FALSE:
1125                         case Token.FIXED:
1126                         case Token.NEW:
1127                         case Token.NULL:
1128                         case Token.SIZEOF:
1129                         case Token.THIS:
1130                         case Token.THROW:
1131                         case Token.TRUE:
1132                         case Token.TYPEOF:
1133                         case Token.UNCHECKED:
1134                         case Token.UNSAFE:
1135
1136                                 //
1137                                 // These can be part of a member access
1138                                 //
1139                         case Token.INT:
1140                         case Token.UINT:
1141                         case Token.SHORT:
1142                         case Token.USHORT:
1143                         case Token.LONG:
1144                         case Token.ULONG:
1145                         case Token.DOUBLE:
1146                         case Token.FLOAT:
1147                         case Token.CHAR:
1148                                 return true;
1149
1150                         default:
1151                                 return false;
1152                         }
1153                 }
1154
1155                 public int token ()
1156                 {
1157                         current_token = xtoken ();
1158                         return current_token;
1159                 }
1160
1161                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1162
1163                 void get_cmd_arg (out string cmd, out string arg)
1164                 {
1165                         int c;
1166
1167                         tokens_seen = false;
1168                         arg = "";
1169                         static_cmd_arg.Length = 0;
1170
1171                         // skip over white space
1172                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1173                                 ;
1174
1175
1176                         while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
1177                                 if (is_identifier_part_character ((char) c)){
1178                                         static_cmd_arg.Append ((char) c);
1179                                         c = getChar ();
1180                                 } else {
1181                                         putback (c);
1182                                         break;
1183                                 }
1184                         }
1185
1186                         cmd = static_cmd_arg.ToString ();
1187
1188                         if (c == '\n'){
1189                                 line++;
1190                                 ref_line++;
1191                                 return;
1192                         } else if (c == '\r')
1193                                 col = 0;
1194
1195                         // skip over white space
1196                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1197                                 ;
1198
1199                         if (c == '\n'){
1200                                 line++;
1201                                 ref_line++;
1202                                 return;
1203                         } else if (c == '\r'){
1204                                 col = 0;
1205                                 return;
1206                         }
1207
1208                         static_cmd_arg.Length = 0;
1209                         static_cmd_arg.Append ((char) c);
1210
1211                         while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
1212                                 static_cmd_arg.Append ((char) c);
1213                         }
1214
1215                         if (c == '\n'){
1216                                 line++;
1217                                 ref_line++;
1218                         } else if (c == '\r')
1219                                 col = 0;
1220                         arg = static_cmd_arg.ToString ().Trim ();
1221                 }
1222
1223                 //
1224                 // Handles the #line directive
1225                 //
1226                 bool PreProcessLine (string arg)
1227                 {
1228                         if (arg == "")
1229                                 return false;
1230
1231                         if (arg == "default"){
1232                                 ref_line = line;
1233                                 ref_name = file_name;
1234                                 Location.Push (ref_name);
1235                                 return true;
1236                         } else if (arg == "hidden"){
1237                                 //
1238                                 // We ignore #line hidden
1239                                 //
1240                                 return true;
1241                         }
1242
1243                         try {
1244                                 int pos;
1245
1246                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1247                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1248                                         pos++;
1249
1250                                         char [] quotes = { '\"' };
1251
1252                                         string name = arg.Substring (pos). Trim (quotes);
1253                                         ref_name = Location.LookupFile (name);
1254                                         file_name.HasLineDirective = true;
1255                                         ref_name.HasLineDirective = true;
1256                                         Location.Push (ref_name);
1257                                 } else {
1258                                         ref_line = System.Int32.Parse (arg);
1259                                 }
1260                         } catch {
1261                                 return false;
1262                         }
1263
1264                         return true;
1265                 }
1266
1267                 //
1268                 // Handles #define and #undef
1269                 //
1270                 void PreProcessDefinition (bool is_define, string arg)
1271                 {
1272                         if (arg == "" || arg == "true" || arg == "false"){
1273                                 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
1274                                 return;
1275                         }
1276
1277                         char[] whitespace = { ' ', '\t' };
1278                         if (arg.IndexOfAny (whitespace) != -1){
1279                                 Report.Error (1025, Location, "Single-line comment or end-of-line expected");
1280                                 return;
1281                         }
1282
1283                         if (!is_identifier_start_character (arg [0]))
1284                                 Report.Error (1001, Location, "Identifier expected: " + arg);
1285
1286                         foreach (char c in arg.Substring (1)){
1287                                 if (!is_identifier_part_character (c)){
1288                                         Report.Error (1001, Location, "Identifier expected: " + arg);
1289                                         return;
1290                                 }
1291                         }
1292
1293                         if (is_define){
1294                                 if (defines == null)
1295                                         defines = new Hashtable ();
1296                                 define (arg);
1297                         } else {
1298                                 if (defines == null)
1299                                         return;
1300                                 if (defines.Contains (arg))
1301                                         defines.Remove (arg);
1302                         }
1303                 }
1304
1305                 /// <summary>
1306                 /// Handles #pragma directive
1307                 /// </summary>
1308                 void PreProcessPragma (string arg)
1309                 {
1310                         const string warning = "warning";
1311                         const string w_disable = "warning disable";
1312                         const string w_restore = "warning restore";
1313
1314                         if (arg == w_disable) {
1315                                 Report.RegisterWarningRegion (Location).WarningDisable (line);
1316                                 return;
1317                         }
1318
1319                         if (arg == w_restore) {
1320                                 Report.RegisterWarningRegion (Location).WarningEnable (line);
1321                                 return;
1322                         }
1323
1324                         if (arg.StartsWith (w_disable)) {
1325                                 int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
1326                                 foreach (int code in codes) {
1327                                         if (code != 0)
1328                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1329                                 }
1330                                 return;
1331                         }
1332
1333                         if (arg.StartsWith (w_restore)) {
1334                                 int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
1335                                 Hashtable w_table = Report.warning_ignore_table;
1336                                 foreach (int code in codes) {
1337                                         if (w_table != null && w_table.Contains (code))
1338                                                 Report.Warning (1635, 1, Location, "Cannot restore warning 'CS{0:0000}' because it was disabled globally", code);
1339                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1340                                 }
1341                                 return;
1342                         }
1343
1344                         if (arg.StartsWith (warning)) {
1345                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
1346                                 return;
1347                         }
1348
1349                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
1350                 }
1351
1352                 int[] ParseNumbers (string text)
1353                 {
1354                         string[] string_array = text.Split (',');
1355                         int[] values = new int [string_array.Length];
1356                         int index = 0;
1357                         foreach (string string_code in string_array) {
1358                                 try {
1359                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1360                                 }
1361                                 catch (FormatException) {
1362                                         Report.Warning (1692, Location, "Invalid number");
1363                                 }
1364                         }
1365                         return values;
1366                 }
1367
1368                 bool eval_val (string s)
1369                 {
1370                         if (s == "true")
1371                                 return true;
1372                         if (s == "false")
1373                                 return false;
1374
1375                         if (defines == null)
1376                                 return false;
1377                         if (defines.Contains (s))
1378                                 return true;
1379
1380                         return false;
1381                 }
1382
1383                 bool pp_primary (ref string s)
1384                 {
1385                         s = s.Trim ();
1386                         int len = s.Length;
1387
1388                         if (len > 0){
1389                                 char c = s [0];
1390
1391                                 if (c == '('){
1392                                         s = s.Substring (1);
1393                                         bool val = pp_expr (ref s);
1394                                         if (s.Length > 0 && s [0] == ')'){
1395                                                 s = s.Substring (1);
1396                                                 return val;
1397                                         }
1398                                         Error_InvalidDirective ();
1399                                         return false;
1400                                 }
1401
1402                                 if (is_identifier_start_character (c)){
1403                                         int j = 1;
1404
1405                                         while (j < len){
1406                                                 c = s [j];
1407
1408                                                 if (is_identifier_part_character (c)){
1409                                                         j++;
1410                                                         continue;
1411                                                 }
1412                                                 bool v = eval_val (s.Substring (0, j));
1413                                                 s = s.Substring (j);
1414                                                 return v;
1415                                         }
1416                                         bool vv = eval_val (s);
1417                                         s = "";
1418                                         return vv;
1419                                 }
1420                         }
1421                         Error_InvalidDirective ();
1422                         return false;
1423                 }
1424
1425                 bool pp_unary (ref string s)
1426                 {
1427                         s = s.Trim ();
1428                         int len = s.Length;
1429
1430                         if (len > 0){
1431                                 if (s [0] == '!'){
1432                                         if (len > 1 && s [1] == '='){
1433                                                 Error_InvalidDirective ();
1434                                                 return false;
1435                                         }
1436                                         s = s.Substring (1);
1437                                         return ! pp_primary (ref s);
1438                                 } else
1439                                         return pp_primary (ref s);
1440                         } else {
1441                                 Error_InvalidDirective ();
1442                                 return false;
1443                         }
1444                 }
1445
1446                 bool pp_eq (ref string s)
1447                 {
1448                         bool va = pp_unary (ref s);
1449
1450                         s = s.Trim ();
1451                         int len = s.Length;
1452                         if (len > 0){
1453                                 if (s [0] == '='){
1454                                         if (len > 2 && s [1] == '='){
1455                                                 s = s.Substring (2);
1456                                                 return va == pp_unary (ref s);
1457                                         } else {
1458                                                 Error_InvalidDirective ();
1459                                                 return false;
1460                                         }
1461                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
1462                                         s = s.Substring (2);
1463
1464                                         return va != pp_unary (ref s);
1465
1466                                 }
1467                         }
1468
1469                         return va;
1470
1471                 }
1472
1473                 bool pp_and (ref string s)
1474                 {
1475                         bool va = pp_eq (ref s);
1476
1477                         s = s.Trim ();
1478                         int len = s.Length;
1479                         if (len > 0){
1480                                 if (s [0] == '&'){
1481                                         if (len > 2 && s [1] == '&'){
1482                                                 s = s.Substring (2);
1483                                                 return (va & pp_and (ref s));
1484                                         } else {
1485                                                 Error_InvalidDirective ();
1486                                                 return false;
1487                                         }
1488                                 }
1489                         }
1490                         return va;
1491                 }
1492
1493                 //
1494                 // Evaluates an expression for `#if' or `#elif'
1495                 //
1496                 bool pp_expr (ref string s)
1497                 {
1498                         bool va = pp_and (ref s);
1499                         s = s.Trim ();
1500                         int len = s.Length;
1501                         if (len > 0){
1502                                 char c = s [0];
1503
1504                                 if (c == '|'){
1505                                         if (len > 2 && s [1] == '|'){
1506                                                 s = s.Substring (2);
1507                                                 return va | pp_expr (ref s);
1508                                         } else {
1509                                                 Error_InvalidDirective ();
1510                                                 return false;
1511                                         }
1512                                 }
1513                         }
1514
1515                         return va;
1516                 }
1517
1518                 bool eval (string s)
1519                 {
1520                         bool v = pp_expr (ref s);
1521                         s = s.Trim ();
1522                         if (s.Length != 0){
1523                                 Error_InvalidDirective ();
1524                                 return false;
1525                         }
1526
1527                         return v;
1528                 }
1529
1530                 void Error_InvalidDirective ()
1531                 {
1532                         Report.Error (1517, Location, "Invalid pre-processor directive");
1533                 }
1534
1535                 void Error_UnexpectedDirective (string extra)
1536                 {
1537                         Report.Error (
1538                                 1028, Location,
1539                                 "Unexpected processor directive (" + extra + ")");
1540                 }
1541
1542                 void Error_TokensSeen ()
1543                 {
1544                         Report.Error (
1545                                 1032, Location,
1546                                 "Cannot define or undefine pre-processor symbols after a token in the file");
1547                 }
1548
1549                 //
1550                 // if true, then the code continues processing the code
1551                 // if false, the code stays in a loop until another directive is
1552                 // reached.
1553                 //
1554                 bool handle_preprocessing_directive (bool caller_is_taking)
1555                 {
1556                         string cmd, arg;
1557                         bool region_directive = false;
1558
1559                         get_cmd_arg (out cmd, out arg);
1560
1561                         // Eat any trailing whitespaces and single-line comments
1562                         if (arg.IndexOf ("//") != -1)
1563                                 arg = arg.Substring (0, arg.IndexOf ("//"));
1564                         arg = arg.TrimEnd (' ', '\t');
1565
1566                         //
1567                         // The first group of pre-processing instructions is always processed
1568                         //
1569                         switch (cmd){
1570                         case "pragma":
1571                                 if (RootContext.Version == LanguageVersion.ISO_1) {
1572                                         Report.FeatureIsNotStandardized (Location, "#pragma");
1573                                         return caller_is_taking;
1574                                 }
1575
1576                                 PreProcessPragma (arg);
1577                                 return caller_is_taking;
1578
1579                         case "line":
1580                                 if (!PreProcessLine (arg))
1581                                         Report.Error (
1582                                                 1576, Location,
1583                                                 "Argument to #line directive is missing or invalid");
1584                                 return caller_is_taking;
1585
1586                         case "region":
1587                                 region_directive = true;
1588                                 arg = "true";
1589                                 goto case "if";
1590
1591                         case "endregion":
1592                                 region_directive = true;
1593                                 goto case "endif";
1594
1595                         case "if":
1596                                 if (arg == ""){
1597                                         Error_InvalidDirective ();
1598                                         return true;
1599                                 }
1600                                 bool taking = false;
1601                                 if (ifstack == null)
1602                                         ifstack = new Stack ();
1603
1604                                 if (ifstack.Count == 0){
1605                                         taking = true;
1606                                 } else {
1607                                         int state = (int) ifstack.Peek ();
1608                                         if ((state & TAKING) != 0)
1609                                                 taking = true;
1610                                 }
1611
1612                                 if (eval (arg) && taking){
1613                                         int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;
1614                                         if (region_directive)
1615                                                 push |= REGION;
1616                                         ifstack.Push (push);
1617                                         return true;
1618                                 } else {
1619                                         int push = (taking ? PARENT_TAKING : 0);
1620                                         if (region_directive)
1621                                                 push |= REGION;
1622                                         ifstack.Push (push);
1623                                         return false;
1624                                 }
1625
1626                         case "endif":
1627                                 if (ifstack == null || ifstack.Count == 0){
1628                                         Error_UnexpectedDirective ("no #if for this #endif");
1629                                         return true;
1630                                 } else {
1631                                         int pop = (int) ifstack.Pop ();
1632
1633                                         if (region_directive && ((pop & REGION) == 0))
1634                                                 Report.Error (1027, Location, "#endif directive expected");
1635                                         else if (!region_directive && ((pop & REGION) != 0))
1636                                                 Report.Error (1038, Location, "#endregion directive expected");
1637
1638                                         if (ifstack.Count == 0)
1639                                                 return true;
1640                                         else {
1641                                                 int state = (int) ifstack.Peek ();
1642
1643                                                 if ((state & TAKING) != 0)
1644                                                         return true;
1645                                                 else
1646                                                         return false;
1647                                         }
1648                                 }
1649
1650                         case "elif":
1651                                 if (ifstack == null || ifstack.Count == 0){
1652                                         Error_UnexpectedDirective ("no #if for this #elif");
1653                                         return true;
1654                                 } else {
1655                                         int state = (int) ifstack.Peek ();
1656
1657                                         if ((state & REGION) != 0) {
1658                                                 Report.Error (1038, Location, "#endregion directive expected");
1659                                                 return true;
1660                                         }
1661
1662                                         if ((state & ELSE_SEEN) != 0){
1663                                                 Error_UnexpectedDirective ("#elif not valid after #else");
1664                                                 return true;
1665                                         }
1666
1667                                         if ((state & (TAKEN_BEFORE | TAKING)) != 0)
1668                                                 return false;
1669
1670                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
1671                                                 state = (int) ifstack.Pop ();
1672                                                 ifstack.Push (state | TAKING | TAKEN_BEFORE);
1673                                                 return true;
1674                                         } else
1675                                                 return false;
1676                                 }
1677
1678                         case "else":
1679                                 if (ifstack == null || ifstack.Count == 0){
1680                                         Report.Error (
1681                                                 1028, Location,
1682                                                 "Unexpected processor directive (no #if for this #else)");
1683                                         return true;
1684                                 } else {
1685                                         int state = (int) ifstack.Peek ();
1686
1687                                         if ((state & REGION) != 0) {
1688                                                 Report.Error (1038, Location, "#endregion directive expected");
1689                                                 return true;
1690                                         }
1691
1692                                         if ((state & ELSE_SEEN) != 0){
1693                                                 Error_UnexpectedDirective ("#else within #else");
1694                                                 return true;
1695                                         }
1696
1697                                         ifstack.Pop ();
1698
1699                                         bool ret;
1700                                         if ((state & TAKEN_BEFORE) == 0){
1701                                                 ret = ((state & PARENT_TAKING) != 0);
1702                                         } else
1703                                                 ret = false;
1704
1705                                         if (ret)
1706                                                 state |= TAKING;
1707                                         else
1708                                                 state &= ~TAKING;
1709
1710                                         ifstack.Push (state | ELSE_SEEN);
1711
1712                                         return ret;
1713                                 }
1714                         }
1715
1716                         //
1717                         // These are only processed if we are in a `taking' block
1718                         //
1719                         if (!caller_is_taking)
1720                                 return false;
1721
1722                         switch (cmd){
1723                         case "define":
1724                                 if (any_token_seen){
1725                                         Error_TokensSeen ();
1726                                         return true;
1727                                 }
1728                                 PreProcessDefinition (true, arg);
1729                                 return true;
1730
1731                         case "undef":
1732                                 if (any_token_seen){
1733                                         Error_TokensSeen ();
1734                                         return true;
1735                                 }
1736                                 PreProcessDefinition (false, arg);
1737                                 return true;
1738
1739                         case "error":
1740                                 Report.Error (1029, Location, "#error: '" + arg + "'");
1741                                 return true;
1742
1743                         case "warning":
1744                                 Report.Warning (1030, Location, "#warning: '{0}'", arg);
1745                                 return true;
1746                         }
1747
1748                         Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");
1749                         return true;
1750
1751                 }
1752
1753                 private int consume_string (bool quoted)
1754                 {
1755                         int c;
1756                         string_builder.Length = 0;
1757
1758                         while ((c = getChar ()) != -1){
1759                                 if (c == '"'){
1760                                         if (quoted && peekChar () == '"'){
1761                                                 string_builder.Append ((char) c);
1762                                                 getChar ();
1763                                                 continue;
1764                                         } else {
1765                                                 val = string_builder.ToString ();
1766                                                 return Token.LITERAL_STRING;
1767                                         }
1768                                 }
1769
1770                                 if (c == '\n'){
1771                                         if (!quoted)
1772                                                 Report.Error (1010, Location, "Newline in constant");
1773                                         line++;
1774                                         ref_line++;
1775                                         col = 0;
1776                                 } else
1777                                         col++;
1778
1779                                 if (!quoted){
1780                                         c = escape (c);
1781                                         if (c == -1)
1782                                                 return Token.ERROR;
1783                                 }
1784                                 string_builder.Append ((char) c);
1785                         }
1786
1787                         Report.Error (1039, Location, "Unterminated string literal");
1788                         return Token.EOF;
1789                 }
1790
1791                 private int consume_identifier (int s)
1792                 {
1793                         int res = consume_identifier (s, false);
1794
1795                         if (doc_state == XmlCommentState.Allowed)
1796                                 doc_state = XmlCommentState.NotAllowed;
1797                         switch (res) {
1798                         case Token.USING:
1799                         case Token.NAMESPACE:
1800                                 check_incorrect_doc_comment ();
1801                                 break;
1802                         }
1803
1804                         if (res == Token.PARTIAL) {
1805                                 // Save current position and parse next token.
1806                                 int old = reader.Position;
1807                                 int old_putback = putback_char;
1808
1809                                 putback_char = -1;
1810
1811                                 int next_token = token ();
1812                                 bool ok = (next_token == Token.CLASS) ||
1813                                         (next_token == Token.STRUCT) ||
1814                                         (next_token == Token.INTERFACE);
1815
1816                                 reader.Position = old;
1817                                 putback_char = old_putback;
1818
1819                                 if (ok)
1820                                         return res;
1821                                 else {
1822                                         val = "partial";
1823                                         return Token.IDENTIFIER;
1824                                 }
1825                         }
1826
1827                         return res;
1828                 }
1829
1830                 private int consume_identifier (int s, bool quoted)
1831                 {
1832                         int pos = 1;
1833                         int c;
1834
1835                         id_builder [0] = (char) s;
1836
1837                         while ((c = reader.Read ()) != -1) {
1838                                 if (is_identifier_part_character ((char) c)){
1839                                         if (pos == max_id_size){
1840                                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
1841                                                 return Token.ERROR;
1842                                         }
1843
1844                                         id_builder [pos++] = (char) c;
1845                                         putback_char = -1;
1846                                         col++;
1847                                 } else {
1848                                         putback_char = c;
1849                                         break;
1850                                 }
1851                         }
1852
1853                         //
1854                         // Optimization: avoids doing the keyword lookup
1855                         // on uppercase letters and _
1856                         //
1857                         if (!quoted && (s >= 'a' || s == '_')){
1858                                 int keyword = GetKeyword (id_builder, pos);
1859                                 if (keyword != -1)
1860                                         return keyword;
1861                         }
1862
1863                         //
1864                         // Keep identifiers in an array of hashtables to avoid needless
1865                         // allocations
1866                         //
1867
1868                         if (identifiers [pos] != null) {
1869                                 val = identifiers [pos][id_builder];
1870                                 if (val != null) {
1871                                         return Token.IDENTIFIER;
1872                                 }
1873                         }
1874                         else
1875                                 identifiers [pos] = new CharArrayHashtable (pos);
1876
1877                         val = new String (id_builder, 0, pos);
1878
1879                         char [] chars = new char [pos];
1880                         Array.Copy (id_builder, chars, pos);
1881
1882                         identifiers [pos] [chars] = val;
1883
1884                         return Token.IDENTIFIER;
1885                 }
1886
1887                 public int xtoken ()
1888                 {
1889                         int t;
1890                         bool doread = false;
1891                         int c;
1892
1893                         // Whether we have seen comments on the current line
1894                         bool comments_seen = false;
1895
1896                         val = null;
1897                         // optimization: eliminate col and implement #directive semantic correctly.
1898                         for (;(c = getChar ()) != -1; col++) {
1899                                 if (c == ' ')
1900                                         continue;
1901
1902                                 if (c == '\t') {
1903                                         col = (((col + 8) / 8) * 8) - 1;
1904                                         continue;
1905                                 }
1906
1907                                 if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0)
1908                                         continue;
1909
1910                                 if (c == '\r') {
1911                                         if (peekChar () == '\n')
1912                                                 getChar ();
1913
1914                                         line++;
1915                                         ref_line++;
1916                                         col = 0;
1917                                         any_token_seen |= tokens_seen;
1918                                         tokens_seen = false;
1919                                         comments_seen = false;
1920                                         continue;
1921                                 }
1922
1923                                 // Handle double-slash comments.
1924                                 if (c == '/'){
1925                                         int d = peekChar ();
1926
1927                                         if (d == '/'){
1928                                                 getChar ();
1929                                                 if (RootContext.Documentation != null && peekChar () == '/') {
1930                                                         getChar ();
1931                                                         // Don't allow ////.
1932                                                         if ((d = peekChar ()) != '/') {
1933                                                                 if (doc_state == XmlCommentState.Allowed)
1934                                                                         handle_one_line_xml_comment ();
1935                                                                 else if (doc_state == XmlCommentState.NotAllowed)
1936                                                                         warn_incorrect_doc_comment ();
1937                                                         }
1938                                                 }
1939                                                 while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
1940                                                         col++;
1941                                                 if (d == '\n'){
1942                                                         line++;
1943                                                         ref_line++;
1944                                                         col = 0;
1945                                                 }
1946                                                 any_token_seen |= tokens_seen;
1947                                                 tokens_seen = false;
1948                                                 comments_seen = false;
1949                                                 continue;
1950                                         } else if (d == '*'){
1951                                                 getChar ();
1952                                                 bool docAppend = false;
1953                                                 if (RootContext.Documentation != null && peekChar () == '*') {
1954                                                         getChar ();
1955                                                         // But when it is /**/, just do nothing.
1956                                                         if (peekChar () == '/') {
1957                                                                 getChar ();
1958                                                                 continue;
1959                                                         }
1960                                                         if (doc_state == XmlCommentState.Allowed)
1961                                                                 docAppend = true;
1962                                                         else if (doc_state == XmlCommentState.NotAllowed)
1963                                                                 warn_incorrect_doc_comment ();
1964                                                 }
1965
1966                                                 int current_comment_start = 0;
1967                                                 if (docAppend) {
1968                                                         current_comment_start = xml_comment_buffer.Length;
1969                                                         xml_comment_buffer.Append (Environment.NewLine);
1970                                                 }
1971
1972                                                 Location start_location = Location;
1973
1974                                                 while ((d = getChar ()) != -1){
1975                                                         if (d == '*' && peekChar () == '/'){
1976                                                                 getChar ();
1977                                                                 col++;
1978                                                                 comments_seen = true;
1979                                                                 break;
1980                                                         }
1981                                                         if (docAppend)
1982                                                                 xml_comment_buffer.Append ((char) d);
1983
1984                                                         if (d == '\n'){
1985                                                                 line++;
1986                                                                 ref_line++;
1987                                                                 col = 0;
1988                                                                 any_token_seen |= tokens_seen;
1989                                                                 tokens_seen = false;
1990                                                                 //
1991                                                                 // Reset 'comments_seen' just to be consistent.
1992                                                                 // It doesn't matter either way, here.
1993                                                                 //
1994                                                                 comments_seen = false;
1995                                                         }
1996                                                 }
1997                                                 if (!comments_seen)
1998                                                         Report.Error (1035, start_location, "End-of-file found, '*/' expected");
1999
2000                                                 if (docAppend)
2001                                                         update_formatted_doc_comment (current_comment_start);
2002                                                 continue;
2003                                         }
2004                                         goto is_punct_label;
2005                                 }
2006
2007
2008                                 if (is_identifier_start_character ((char)c)){
2009                                         tokens_seen = true;
2010                                         return consume_identifier (c);
2011                                 }
2012
2013                         is_punct_label:
2014                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
2015                                         tokens_seen = true;
2016                                         if (doread){
2017                                                 getChar ();
2018                                                 col++;
2019                                         }
2020                                         return t;
2021                                 }
2022
2023                                 // white space
2024                                 if (c == '\n'){
2025                                         line++;
2026                                         ref_line++;
2027                                         col = 0;
2028                                         any_token_seen |= tokens_seen;
2029                                         tokens_seen = false;
2030                                         comments_seen = false;
2031                                         continue;
2032                                 }
2033
2034                                 if (c >= '0' && c <= '9'){
2035                                         tokens_seen = true;
2036                                         return is_number (c);
2037                                 }
2038
2039                                 if (c == '.'){
2040                                         tokens_seen = true;
2041                                         int peek = peekChar ();
2042                                         if (peek >= '0' && peek <= '9')
2043                                                 return is_number (c);
2044                                         return Token.DOT;
2045                                 }
2046
2047                                 /* For now, ignore pre-processor commands */
2048                                 // FIXME: In C# the '#' is not limited to appear
2049                                 // on the first column.
2050                                 if (c == '#') {
2051                                         bool cont = true;
2052
2053                                         if (tokens_seen || comments_seen) {
2054                                                error_details = "Preprocessor directives must appear as the first non-whitespace " +
2055                                                        "character on a line.";
2056
2057                                                Report.Error (1040, Location, error_details);
2058
2059                                                return Token.ERROR;
2060                                        }
2061
2062                                 start_again:
2063
2064                                         cont = handle_preprocessing_directive (cont);
2065
2066                                         if (cont){
2067                                                 col = 0;
2068                                                 continue;
2069                                         }
2070                                         col = 1;
2071
2072                                         bool skipping = false;
2073                                         for (;(c = getChar ()) != -1; col++){
2074                                                 if (c == '\n'){
2075                                                         col = 0;
2076                                                         line++;
2077                                                         ref_line++;
2078                                                         skipping = false;
2079                                                 } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
2080                                                         continue;
2081                                                 else if (c != '#')
2082                                                         skipping = true;
2083                                                 if (c == '#' && !skipping)
2084                                                         goto start_again;
2085                                         }
2086                                         any_token_seen |= tokens_seen;
2087                                         tokens_seen = false;
2088                                         if (c == -1)
2089                                                 Report.Error (1027, Location, "#endif/#endregion expected");
2090                                         continue;
2091                                 }
2092
2093                                 if (c == '"')
2094                                         return consume_string (false);
2095
2096                                 if (c == '\''){
2097                                         c = getChar ();
2098                                         tokens_seen = true;
2099                                         if (c == '\''){
2100                                                 error_details = "Empty character literal";
2101                                                 Report.Error (1011, Location, error_details);
2102                                                 return Token.ERROR;
2103                                         }
2104                                         c = escape (c);
2105                                         if (c == -1)
2106                                                 return Token.ERROR;
2107                                         val = new System.Char ();
2108                                         val = (char) c;
2109                                         c = getChar ();
2110
2111                                         if (c != '\''){
2112                                                 error_details = "Too many characters in character literal";
2113                                                 Report.Error (1012, Location, error_details);
2114
2115                                                 // Try to recover, read until newline or next "'"
2116                                                 while ((c = getChar ()) != -1){
2117                                                         if (c == '\n' || c == '\''){
2118                                                                 line++;
2119                                                                 ref_line++;
2120                                                                 col = 0;
2121                                                                 break;
2122                                                         } else
2123                                                                 col++;
2124
2125                                                 }
2126                                                 return Token.ERROR;
2127                                         }
2128                                         return Token.LITERAL_CHARACTER;
2129                                 }
2130
2131                                 if (c == '@') {
2132                                         c = getChar ();
2133                                         if (c == '"') {
2134                                                 tokens_seen = true;
2135                                                 return consume_string (true);
2136                                         } else if (is_identifier_start_character ((char) c)){
2137                                                 return consume_identifier (c, true);
2138                                         } else {
2139                                                 Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
2140                                         }
2141                                 }
2142
2143                                 error_details = ((char)c).ToString ();
2144
2145                                 return Token.ERROR;
2146                         }
2147
2148                         return Token.EOF;
2149                 }
2150
2151                 //
2152                 // Handles one line xml comment
2153                 //
2154                 private void handle_one_line_xml_comment ()
2155                 {
2156                         int c;
2157                         while ((c = peekChar ()) == ' ')
2158                                 getChar (); // skip heading whitespaces.
2159                         while ((c = peekChar ()) != -1 && c != '\n' && c != '\r') {
2160                                 col++;
2161                                 xml_comment_buffer.Append ((char) getChar ());
2162                         }
2163                         if (c == '\r' || c == '\n')
2164                                 xml_comment_buffer.Append (Environment.NewLine);
2165                 }
2166
2167                 //
2168                 // Remove heading "*" in Javadoc-like xml documentation.
2169                 //
2170                 private void update_formatted_doc_comment (int current_comment_start)
2171                 {
2172                         int length = xml_comment_buffer.Length - current_comment_start;
2173                         string [] lines = xml_comment_buffer.ToString (
2174                                 current_comment_start,
2175                                 length).Replace ("\r", "").Split ('\n');
2176
2177                         // The first line starts with /**, thus it is not target
2178                         // for the format check.
2179                         for (int i = 1; i < lines.Length; i++) {
2180                                 string s = lines [i];
2181                                 int idx = s.IndexOf ('*');
2182                                 string head = null;
2183                                 if (idx < 0) {
2184                                         if (i < lines.Length - 1)
2185                                                 return;
2186                                         head = s;
2187                                 } else
2188                                         head = s.Substring (0, idx);
2189                                 foreach (char c in head)
2190                                         if (c != ' ')
2191                                                 return;
2192                                 lines [i] = s.Substring (idx + 1);
2193                         }
2194                         xml_comment_buffer.Remove (current_comment_start, length);
2195                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
2196                 }
2197
2198                 //
2199                 // Checks if there was incorrect doc comments and raise
2200                 // warnings.
2201                 //
2202                 public void check_incorrect_doc_comment ()
2203                 {
2204                         if (xml_comment_buffer.Length > 0)
2205                                 warn_incorrect_doc_comment ();
2206                 }
2207
2208                 //
2209                 // Raises a warning when tokenizer found incorrect doccomment
2210                 // markup.
2211                 //
2212                 private void warn_incorrect_doc_comment ()
2213                 {
2214                         doc_state = XmlCommentState.Error;
2215                         // in csc, it is 'XML comment is not placed on a valid
2216                         // language element'. But that does not make sense.
2217                         Report.Warning (1587, 2, Location, "XML comment is placed on an invalid language element which can not accept it.");
2218                 }
2219
2220                 //
2221                 // Consumes the saved xml comment lines (if any)
2222                 // as for current target member or type.
2223                 //
2224                 public string consume_doc_comment ()
2225                 {
2226                         if (xml_comment_buffer.Length > 0) {
2227                                 string ret = xml_comment_buffer.ToString ();
2228                                 xml_comment_buffer.Length = 0;
2229                                 return ret;
2230                         }
2231                         return null;
2232                 }
2233
2234                 public void cleanup ()
2235                 {
2236                         if (ifstack != null && ifstack.Count >= 1) {
2237                                 int state = (int) ifstack.Pop ();
2238                                 if ((state & REGION) != 0)
2239                                         Report.Error (1038, Location, "#endregion directive expected");
2240                                 else
2241                                         Report.Error (1027, "#endif directive expected");
2242                         }
2243
2244                 }
2245         }
2246
2247         //
2248         // Indicates whether it accepts XML documentation or not.
2249         //
2250         public enum XmlCommentState {
2251                 // comment is allowed in this state.
2252                 Allowed,
2253                 // comment is not allowed in this state.
2254                 NotAllowed,
2255                 // once comments appeared when it is NotAllowed, then the
2256                 // state is changed to it, until the state is changed to
2257                 // .Allowed.
2258                 Error
2259         }
2260 }