mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //
   7 // Licensed under the terms of the GNU GPL
   8 //
   9 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
  10 // (C) 2004 Novell, Inc
  11 //
  12
  13 /*
  14  * TODO:
  15  *   Make sure we accept the proper Unicode ranges, per the spec.
  16  *   Report error 1032
  17 */
  18
  19 using System;
  20 using System.Text;
  21 using System.Collections;
  22 using System.IO;
  23 using System.Globalization;
  24 using System.Reflection;
  25
  26 namespace Mono.CSharp
  27 {
  28         /// <summary>
  29         ///    Tokenizer for C# source code.
  30         /// </summary>
  31
  32         public class Tokenizer : yyParser.yyInput
  33         {
  34                 SeekableStreamReader reader;
  35                 public SourceFile ref_name;
  36                 public SourceFile file_name;
  37                 public int ref_line = 1;
  38                 public int line = 1;
  39                 public int col = 1;
  40                 public int current_token;
  41                 bool handle_get_set = false;
  42                 bool handle_remove_add = false;
  43                 bool handle_assembly = false;
  44
  45                 //
  46                 // XML documentation buffer. The save point is used to divide
  47                 // comments on types and comments on members.
  48                 //
  49                 StringBuilder xml_comment_buffer;
  50
  51                 //
  52                 // See comment on XmlCommentState enumeration.
  53                 //
  54                 XmlCommentState xmlDocState = XmlCommentState.Allowed;
  55
  56                 //
  57                 // Whether tokens have been seen on this line
  58                 //
  59                 bool tokens_seen = false;
  60
  61                 //
  62                 // Whether a token has been seen on the file
  63                 // This is needed because `define' is not allowed to be used
  64                 // after a token has been seen.
  65                 //
  66                 bool any_token_seen = false;
  67
  68                 static Hashtable tokenValues;
  69
  70                 private static Hashtable TokenValueName
  71                 {
  72                         get {
  73                                 if (tokenValues == null)
  74                                         tokenValues = GetTokenValueNameHash ();
  75
  76                                 return tokenValues;
  77                         }
  78                 }
  79
  80                 private static Hashtable GetTokenValueNameHash ()
  81                 {
  82                         Type t = typeof (Token);
  83                         FieldInfo [] fields = t.GetFields ();
  84                         Hashtable hash = new Hashtable ();
  85                         foreach (FieldInfo field in fields) {
  86                                 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))
  87                                         hash.Add (field.GetValue (null), field.Name);
  88                         }
  89                         return hash;
  90                 }
  91
  92                 //
  93                 // Returns a verbose representation of the current location
  94                 //
  95                 public string location {
  96                         get {
  97                                 string det;
  98
  99                                 if (current_token == Token.ERROR)
 100                                         det = "detail: " + error_details;
 101                                 else
 102                                         det = "";
 103
 104                                 // return "Line:     "+line+" Col: "+col + "\n" +
 105                                 //       "VirtLine: "+ref_line +
 106                                 //       " Token: "+current_token + " " + det;
 107                                 string current_token_name = TokenValueName [current_token] as string;
 108                                 if (current_token_name == null)
 109                                         current_token_name = current_token.ToString ();
 110
 111                                 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,
 112                                                                                        ref_line,
 113                                                                                        col,
 114                                                                                        current_token_name,
 115                                                                                        det);
 116                         }
 117                 }
 118
 119                 public bool PropertyParsing {
 120                         get {
 121                                 return handle_get_set;
 122                         }
 123
 124                         set {
 125                                 handle_get_set = value;
 126                         }
 127                 }
 128
 129                 public bool AssemblyTargetParsing {
 130                         get {
 131                                 return handle_assembly;
 132                         }
 133
 134                         set {
 135                                 handle_assembly = value;
 136                         }
 137                 }
 138
 139                 public bool EventParsing {
 140                         get {
 141                                 return handle_remove_add;
 142                         }
 143
 144                         set {
 145                                 handle_remove_add = value;
 146                         }
 147                 }
 148
 149                 public XmlCommentState doc_state {
 150                         get { return xmlDocState; }
 151                         set {
 152                                 if (value == XmlCommentState.Allowed) {
 153                                         check_incorrect_doc_comment ();
 154                                         consume_doc_comment ();
 155                                 }
 156                                 xmlDocState = value;
 157                         }
 158                 }
 159
 160
 161                 //
 162                 // Class variables
 163                 //
 164                 static CharArrayHashtable[] keywords;
 165                 static Hashtable keywordStrings = new Hashtable ();
 166                 static NumberStyles styles;
 167                 static NumberFormatInfo csharp_format_info;
 168
 169                 //
 170                 // Values for the associated token returned
 171                 //
 172                 int putback_char;
 173                 Object val;
 174
 175                 //
 176                 // Pre-processor
 177                 //
 178                 Hashtable defines;
 179
 180                 const int TAKING        = 1;
 181                 const int TAKEN_BEFORE  = 2;
 182                 const int ELSE_SEEN     = 4;
 183                 const int PARENT_TAKING = 8;
 184                 const int REGION        = 16;
 185
 186                 //
 187                 // pre-processor if stack state:
 188                 //
 189                 Stack ifstack;
 190
 191                 static System.Text.StringBuilder string_builder;
 192
 193                 const int max_id_size = 512;
 194                 static char [] id_builder = new char [max_id_size];
 195
 196                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 197
 198                 const int max_number_size = 512;
 199                 static char [] number_builder = new char [max_number_size];
 200                 static int number_pos;
 201
 202                 //
 203                 // Details about the error encoutered by the tokenizer
 204                 //
 205                 string error_details;
 206
 207                 public string error {
 208                         get {
 209                                 return error_details;
 210                         }
 211                 }
 212
 213                 public int Line {
 214                         get {
 215                                 return ref_line;
 216                         }
 217                 }
 218
 219                 public int Col {
 220                         get {
 221                                 return col;
 222                         }
 223                 }
 224
 225                 static void AddKeyword (string kw, int token) {
 226                         keywordStrings.Add (kw, kw);
 227                         if (keywords [kw.Length] == null) {
 228                                 keywords [kw.Length] = new CharArrayHashtable (kw.Length);
 229                         }
 230                         keywords [kw.Length] [kw.ToCharArray ()] = token;
 231                 }
 232
 233                 static void InitTokens ()
 234                 {
 235                         keywords = new CharArrayHashtable [64];
 236
 237                         AddKeyword ("__arglist", Token.ARGLIST);
 238                         AddKeyword ("abstract", Token.ABSTRACT);
 239                         AddKeyword ("as", Token.AS);
 240                         AddKeyword ("add", Token.ADD);
 241                         AddKeyword ("assembly", Token.ASSEMBLY);
 242                         AddKeyword ("base", Token.BASE);
 243                         AddKeyword ("bool", Token.BOOL);
 244                         AddKeyword ("break", Token.BREAK);
 245                         AddKeyword ("byte", Token.BYTE);
 246                         AddKeyword ("case", Token.CASE);
 247                         AddKeyword ("catch", Token.CATCH);
 248                         AddKeyword ("char", Token.CHAR);
 249                         AddKeyword ("checked", Token.CHECKED);
 250                         AddKeyword ("class", Token.CLASS);
 251                         AddKeyword ("const", Token.CONST);
 252                         AddKeyword ("continue", Token.CONTINUE);
 253                         AddKeyword ("decimal", Token.DECIMAL);
 254                         AddKeyword ("default", Token.DEFAULT);
 255                         AddKeyword ("delegate", Token.DELEGATE);
 256                         AddKeyword ("do", Token.DO);
 257                         AddKeyword ("double", Token.DOUBLE);
 258                         AddKeyword ("else", Token.ELSE);
 259                         AddKeyword ("enum", Token.ENUM);
 260                         AddKeyword ("event", Token.EVENT);
 261                         AddKeyword ("explicit", Token.EXPLICIT);
 262                         AddKeyword ("extern", Token.EXTERN);
 263                         AddKeyword ("false", Token.FALSE);
 264                         AddKeyword ("finally", Token.FINALLY);
 265                         AddKeyword ("fixed", Token.FIXED);
 266                         AddKeyword ("float", Token.FLOAT);
 267                         AddKeyword ("for", Token.FOR);
 268                         AddKeyword ("foreach", Token.FOREACH);
 269                         AddKeyword ("goto", Token.GOTO);
 270                         AddKeyword ("get", Token.GET);
 271                         AddKeyword ("if", Token.IF);
 272                         AddKeyword ("implicit", Token.IMPLICIT);
 273                         AddKeyword ("in", Token.IN);
 274                         AddKeyword ("int", Token.INT);
 275                         AddKeyword ("interface", Token.INTERFACE);
 276                         AddKeyword ("internal", Token.INTERNAL);
 277                         AddKeyword ("is", Token.IS);
 278                         AddKeyword ("lock", Token.LOCK);
 279                         AddKeyword ("long", Token.LONG);
 280                         AddKeyword ("namespace", Token.NAMESPACE);
 281                         AddKeyword ("new", Token.NEW);
 282                         AddKeyword ("null", Token.NULL);
 283                         AddKeyword ("object", Token.OBJECT);
 284                         AddKeyword ("operator", Token.OPERATOR);
 285                         AddKeyword ("out", Token.OUT);
 286                         AddKeyword ("override", Token.OVERRIDE);
 287                         AddKeyword ("params", Token.PARAMS);
 288                         AddKeyword ("private", Token.PRIVATE);
 289                         AddKeyword ("protected", Token.PROTECTED);
 290                         AddKeyword ("public", Token.PUBLIC);
 291                         AddKeyword ("readonly", Token.READONLY);
 292                         AddKeyword ("ref", Token.REF);
 293                         AddKeyword ("remove", Token.REMOVE);
 294                         AddKeyword ("return", Token.RETURN);
 295                         AddKeyword ("sbyte", Token.SBYTE);
 296                         AddKeyword ("sealed", Token.SEALED);
 297                         AddKeyword ("set", Token.SET);
 298                         AddKeyword ("short", Token.SHORT);
 299                         AddKeyword ("sizeof", Token.SIZEOF);
 300                         AddKeyword ("stackalloc", Token.STACKALLOC);
 301                         AddKeyword ("static", Token.STATIC);
 302                         AddKeyword ("string", Token.STRING);
 303                         AddKeyword ("struct", Token.STRUCT);
 304                         AddKeyword ("switch", Token.SWITCH);
 305                         AddKeyword ("this", Token.THIS);
 306                         AddKeyword ("throw", Token.THROW);
 307                         AddKeyword ("true", Token.TRUE);
 308                         AddKeyword ("try", Token.TRY);
 309                         AddKeyword ("typeof", Token.TYPEOF);
 310                         AddKeyword ("uint", Token.UINT);
 311                         AddKeyword ("ulong", Token.ULONG);
 312                         AddKeyword ("unchecked", Token.UNCHECKED);
 313                         AddKeyword ("unsafe", Token.UNSAFE);
 314                         AddKeyword ("ushort", Token.USHORT);
 315                         AddKeyword ("using", Token.USING);
 316                         AddKeyword ("virtual", Token.VIRTUAL);
 317                         AddKeyword ("void", Token.VOID);
 318                         AddKeyword ("volatile", Token.VOLATILE);
 319                         AddKeyword ("while", Token.WHILE);
 320                         AddKeyword ("partial", Token.PARTIAL);
 321                 }
 322
 323                 //
 324                 // Class initializer
 325                 //
 326                 static Tokenizer ()
 327                 {
 328                         InitTokens ();
 329                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 330                         styles = NumberStyles.Float;
 331
 332                         string_builder = new System.Text.StringBuilder ();
 333                 }
 334
 335                 int GetKeyword (char[] id, int id_len)
 336                 {
 337                         /*
 338                          * Keywords are stored in an array of hashtables grouped by their
 339                          * length.
 340                          */
 341
 342                         if ((id_len >= keywords.Length) || (keywords [id_len] == null))
 343                                 return -1;
 344                         object o = keywords [id_len] [id];
 345
 346                         if (o == null)
 347                                 return -1;
 348
 349                         int res = (int) o;
 350
 351                         if (handle_get_set == false && (res == Token.GET || res == Token.SET))
 352                                 return -1;
 353                         if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
 354                                 return -1;
 355                         if (handle_assembly == false && res == Token.ASSEMBLY)
 356                                 return -1;
 357
 358                         return res;
 359
 360                 }
 361
 362                 public Location Location {
 363                         get {
 364                                 return new Location (ref_line);
 365                         }
 366                 }
 367
 368                 void define (string def)
 369                 {
 370                         if (!RootContext.AllDefines.Contains (def)){
 371                                 RootContext.AllDefines [def] = true;
 372                         }
 373                         if (defines.Contains (def))
 374                                 return;
 375                         defines [def] = true;
 376                 }
 377
 378                 public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)
 379                 {
 380                         this.ref_name = file;
 381                         this.file_name = file;
 382                         reader = input;
 383
 384                         putback_char = -1;
 385
 386                         if (defs != null){
 387                                 defines = new Hashtable ();
 388                                 foreach (string def in defs)
 389                                         define (def);
 390                         }
 391
 392                         xml_comment_buffer = new StringBuilder ();
 393
 394                         //
 395                         // FIXME: This could be `Location.Push' but we have to
 396                         // find out why the MS compiler allows this
 397                         //
 398                         Mono.CSharp.Location.Push (file);
 399                 }
 400
 401                 static bool is_identifier_start_character (char c)
 402                 {
 403                         return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c);
 404                 }
 405
 406                 static bool is_identifier_part_character (char c)
 407                 {
 408                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
 409                 }
 410
 411                 public static bool IsKeyword (string s)
 412                 {
 413                         return keywordStrings [s] != null;
 414                 }
 415
 416                 public static bool IsValidIdentifier (string s)
 417                 {
 418                         if (s == null || s.Length == 0)
 419                                 return false;
 420
 421                         if (!is_identifier_start_character (s [0]))
 422                                 return false;
 423
 424                         for (int i = 1; i < s.Length; i ++)
 425                                 if (! is_identifier_part_character (s [i]))
 426                                         return false;
 427
 428                         return true;
 429                 }
 430
 431                 int is_punct (char c, ref bool doread)
 432                 {
 433                         int d;
 434                         int t;
 435
 436                         doread = false;
 437
 438                         switch (c){
 439                         case '{':
 440                                 return Token.OPEN_BRACE;
 441                         case '}':
 442                                 return Token.CLOSE_BRACE;
 443                         case '[':
 444                                 // To block doccomment inside attribute declaration.
 445                                 if (doc_state == XmlCommentState.Allowed)
 446                                         doc_state = XmlCommentState.NotAllowed;
 447                                 return Token.OPEN_BRACKET;
 448                         case ']':
 449                                 return Token.CLOSE_BRACKET;
 450                         case '(':
 451                                 return Token.OPEN_PARENS;
 452                         case ')': {
 453                                 if (deambiguate_close_parens == 0)
 454                                         return Token.CLOSE_PARENS;
 455
 456                                 --deambiguate_close_parens;
 457
 458                                 // Save current position and parse next token.
 459                                 int old = reader.Position;
 460                                 int new_token = token ();
 461                                 reader.Position = old;
 462                                 putback_char = -1;
 463
 464                                 if (new_token == Token.OPEN_PARENS)
 465                                         return Token.CLOSE_PARENS_OPEN_PARENS;
 466                                 else if (new_token == Token.MINUS)
 467                                         return Token.CLOSE_PARENS_MINUS;
 468                                 else if (IsCastToken (new_token))
 469                                         return Token.CLOSE_PARENS_CAST;
 470                                 else
 471                                         return Token.CLOSE_PARENS_NO_CAST;
 472                         }
 473
 474                         case ',':
 475                                 return Token.COMMA;
 476                         case ':':
 477                                 return Token.COLON;
 478                         case ';':
 479                                 return Token.SEMICOLON;
 480                         case '~':
 481                                 return Token.TILDE;
 482                         case '?':
 483                                 return Token.INTERR;
 484                         }
 485
 486                         d = peekChar ();
 487                         if (c == '+'){
 488
 489                                 if (d == '+')
 490                                         t = Token.OP_INC;
 491                                 else if (d == '=')
 492                                         t = Token.OP_ADD_ASSIGN;
 493                                 else
 494                                         return Token.PLUS;
 495                                 doread = true;
 496                                 return t;
 497                         }
 498                         if (c == '-'){
 499                                 if (d == '-')
 500                                         t = Token.OP_DEC;
 501                                 else if (d == '=')
 502                                         t = Token.OP_SUB_ASSIGN;
 503                                 else if (d == '>')
 504                                         t = Token.OP_PTR;
 505                                 else
 506                                         return Token.MINUS;
 507                                 doread = true;
 508                                 return t;
 509                         }
 510
 511                         if (c == '!'){
 512                                 if (d == '='){
 513                                         doread = true;
 514                                         return Token.OP_NE;
 515                                 }
 516                                 return Token.BANG;
 517                         }
 518
 519                         if (c == '='){
 520                                 if (d == '='){
 521                                         doread = true;
 522                                         return Token.OP_EQ;
 523                                 }
 524                                 return Token.ASSIGN;
 525                         }
 526
 527                         if (c == '&'){
 528                                 if (d == '&'){
 529                                         doread = true;
 530                                         return Token.OP_AND;
 531                                 } else if (d == '='){
 532                                         doread = true;
 533                                         return Token.OP_AND_ASSIGN;
 534                                 }
 535                                 return Token.BITWISE_AND;
 536                         }
 537
 538                         if (c == '|'){
 539                                 if (d == '|'){
 540                                         doread = true;
 541                                         return Token.OP_OR;
 542                                 } else if (d == '='){
 543                                         doread = true;
 544                                         return Token.OP_OR_ASSIGN;
 545                                 }
 546                                 return Token.BITWISE_OR;
 547                         }
 548
 549                         if (c == '*'){
 550                                 if (d == '='){
 551                                         doread = true;
 552                                         return Token.OP_MULT_ASSIGN;
 553                                 }
 554                                 return Token.STAR;
 555                         }
 556
 557                         if (c == '/'){
 558                                 if (d == '='){
 559                                         doread = true;
 560                                         return Token.OP_DIV_ASSIGN;
 561                                 }
 562                                 return Token.DIV;
 563                         }
 564
 565                         if (c == '%'){
 566                                 if (d == '='){
 567                                         doread = true;
 568                                         return Token.OP_MOD_ASSIGN;
 569                                 }
 570                                 return Token.PERCENT;
 571                         }
 572
 573                         if (c == '^'){
 574                                 if (d == '='){
 575                                         doread = true;
 576                                         return Token.OP_XOR_ASSIGN;
 577                                 }
 578                                 return Token.CARRET;
 579                         }
 580
 581                         if (c == '<'){
 582                                 if (d == '<'){
 583                                         getChar ();
 584                                         d = peekChar ();
 585
 586                                         if (d == '='){
 587                                                 doread = true;
 588                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 589                                         }
 590                                         return Token.OP_SHIFT_LEFT;
 591                                 } else if (d == '='){
 592                                         doread = true;
 593                                         return Token.OP_LE;
 594                                 }
 595                                 return Token.OP_LT;
 596                         }
 597
 598                         if (c == '>'){
 599                                 if (d == '>'){
 600                                         getChar ();
 601                                         d = peekChar ();
 602
 603                                         if (d == '='){
 604                                                 doread = true;
 605                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 606                                         }
 607                                         return Token.OP_SHIFT_RIGHT;
 608                                 } else if (d == '='){
 609                                         doread = true;
 610                                         return Token.OP_GE;
 611                                 }
 612                                 return Token.OP_GT;
 613                         }
 614                         return Token.ERROR;
 615                 }
 616
 617                 int deambiguate_close_parens = 0;
 618
 619                 public void Deambiguate_CloseParens ()
 620                 {
 621                         putback (')');
 622                         deambiguate_close_parens++;
 623                 }
 624
 625                 void Error_NumericConstantTooLong ()
 626                 {
 627                         Report.Error (1021, Location, "Numeric constant too long");
 628                 }
 629
 630                 bool decimal_digits (int c)
 631                 {
 632                         int d;
 633                         bool seen_digits = false;
 634
 635                         if (c != -1){
 636                                 if (number_pos == max_number_size)
 637                                         Error_NumericConstantTooLong ();
 638                                 number_builder [number_pos++] = (char) c;
 639                         }
 640
 641                         //
 642                         // We use peekChar2, because decimal_digits needs to do a
 643                         // 2-character look-ahead (5.ToString for example).
 644                         //
 645                         while ((d = peekChar2 ()) != -1){
 646                                 if (d >= '0' && d <= '9'){
 647                                         if (number_pos == max_number_size)
 648                                                 Error_NumericConstantTooLong ();
 649                                         number_builder [number_pos++] = (char) d;
 650                                         getChar ();
 651                                         seen_digits = true;
 652                                 } else
 653                                         break;
 654                         }
 655
 656                         return seen_digits;
 657                 }
 658
 659                 bool is_hex (int e)
 660                 {
 661                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
 662                 }
 663
 664                 int real_type_suffix (int c)
 665                 {
 666                         int t;
 667
 668                         switch (c){
 669                         case 'F': case 'f':
 670                                 t =  Token.LITERAL_FLOAT;
 671                                 break;
 672                         case 'D': case 'd':
 673                                 t = Token.LITERAL_DOUBLE;
 674                                 break;
 675                         case 'M': case 'm':
 676                                  t= Token.LITERAL_DECIMAL;
 677                                 break;
 678                         default:
 679                                 return Token.NONE;
 680                         }
 681                         return t;
 682                 }
 683
 684                 int integer_type_suffix (ulong ul, int c)
 685                 {
 686                         bool is_unsigned = false;
 687                         bool is_long = false;
 688
 689                         if (c != -1){
 690                                 bool scanning = true;
 691                                 do {
 692                                         switch (c){
 693                                         case 'U': case 'u':
 694                                                 if (is_unsigned)
 695                                                         scanning = false;
 696                                                 is_unsigned = true;
 697                                                 getChar ();
 698                                                 break;
 699
 700                                         case 'l':
 701                                                 if (!is_unsigned && (RootContext.WarningLevel >= 4)){
 702                                                         //
 703                                                         // if we have not seen anything in between
 704                                                         // report this error
 705                                                         //
 706                                                         Report.Warning (78, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
 707                                                 }
 708                                                 //
 709                                                 // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
 710                                                 // work around that.
 711                                                 //
 712                                                 //goto case 'L';
 713                                                 if (is_long)
 714                                                         scanning = false;
 715                                                 is_long = true;
 716                                                 getChar ();
 717                                                 break;
 718
 719                                         case 'L':
 720                                                 if (is_long)
 721                                                         scanning = false;
 722                                                 is_long = true;
 723                                                 getChar ();
 724                                                 break;
 725
 726                                         default:
 727                                                 scanning = false;
 728                                                 break;
 729                                         }
 730                                         c = peekChar ();
 731                                 } while (scanning);
 732                         }
 733
 734                         if (is_long && is_unsigned){
 735                                 val = ul;
 736                                 return Token.LITERAL_INTEGER;
 737                         } else if (is_unsigned){
 738                                 // uint if possible, or ulong else.
 739
 740                                 if ((ul & 0xffffffff00000000) == 0)
 741                                         val = (uint) ul;
 742                                 else
 743                                         val = ul;
 744                         } else if (is_long){
 745                                 // long if possible, ulong otherwise
 746                                 if ((ul & 0x8000000000000000) != 0)
 747                                         val = ul;
 748                                 else
 749                                         val = (long) ul;
 750                         } else {
 751                                 // int, uint, long or ulong in that order
 752                                 if ((ul & 0xffffffff00000000) == 0){
 753                                         uint ui = (uint) ul;
 754
 755                                         if ((ui & 0x80000000) != 0)
 756                                                 val = ui;
 757                                         else
 758                                                 val = (int) ui;
 759                                 } else {
 760                                         if ((ul & 0x8000000000000000) != 0)
 761                                                 val = ul;
 762                                         else
 763                                                 val = (long) ul;
 764                                 }
 765                         }
 766                         return Token.LITERAL_INTEGER;
 767                 }
 768
 769                 //
 770                 // given `c' as the next char in the input decide whether
 771                 // we need to convert to a special type, and then choose
 772                 // the best representation for the integer
 773                 //
 774                 int adjust_int (int c)
 775                 {
 776                         try {
 777                                 if (number_pos > 9){
 778                                         ulong ul = (uint) (number_builder [0] - '0');
 779
 780                                         for (int i = 1; i < number_pos; i++){
 781                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
 782                                         }
 783                                         return integer_type_suffix (ul, c);
 784                                 } else {
 785                                         uint ui = (uint) (number_builder [0] - '0');
 786
 787                                         for (int i = 1; i < number_pos; i++){
 788                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
 789                                         }
 790                                         return integer_type_suffix (ui, c);
 791                                 }
 792                         } catch (OverflowException) {
 793                                 error_details = "Integral constant is too large";
 794                                 Report.Error (1021, Location, error_details);
 795                                 val = 0ul;
 796                                 return Token.LITERAL_INTEGER;
 797                         }
 798                 }
 799
 800                 int adjust_real (int t)
 801                 {
 802                         string s = new String (number_builder, 0, number_pos);
 803
 804                         switch (t){
 805                         case Token.LITERAL_DECIMAL:
 806                                 try {
 807                                         val = System.Decimal.Parse (s, styles, csharp_format_info);
 808                                 } catch (OverflowException) {
 809                                         val = 0m;
 810                                         error_details = "Floating-point constant is outside the range of the type 'decimal'";
 811                                         Report.Error (594, Location, error_details);
 812                                 }
 813                                 break;
 814                         case Token.LITERAL_FLOAT:
 815                                 try {
 816                                         val = (float) System.Double.Parse (s, styles, csharp_format_info);
 817                                 } catch (OverflowException) {
 818                                         val = 0.0f;
 819                                         error_details = "Floating-point constant is outside the range of the type 'float'";
 820                                         Report.Error (594, Location, error_details);
 821                                 }
 822                                 break;
 823
 824                         case Token.LITERAL_DOUBLE:
 825                         case Token.NONE:
 826                                 t = Token.LITERAL_DOUBLE;
 827                                 try {
 828                                         val = System.Double.Parse (s, styles, csharp_format_info);
 829                                 } catch (OverflowException) {
 830                                         val = 0.0;
 831                                         error_details = "Floating-point constant is outside the range of the type 'double'";
 832                                         Report.Error (594, Location, error_details);
 833                                 }
 834                                 break;
 835                         }
 836                         return t;
 837                 }
 838
 839                 int handle_hex ()
 840                 {
 841                         int d;
 842                         ulong ul;
 843
 844                         getChar ();
 845                         while ((d = peekChar ()) != -1){
 846                                 if (is_hex (d)){
 847                                         number_builder [number_pos++] = (char) d;
 848                                         getChar ();
 849                                 } else
 850                                         break;
 851                         }
 852
 853                         string s = new String (number_builder, 0, number_pos);
 854                         try {
 855                                 if (number_pos <= 8)
 856                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
 857                                 else
 858                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
 859                         } catch (OverflowException){
 860                                 error_details = "Integral constant is too large";
 861                                 Report.Error (1021, Location, error_details);
 862                                 val = 0ul;
 863                                 return Token.LITERAL_INTEGER;
 864                         }
 865                         catch (FormatException) {
 866                                 Report.Error (1013, Location, "Invalid number");
 867                                 val = 0ul;
 868                                 return Token.LITERAL_INTEGER;
 869                         }
 870
 871                         return integer_type_suffix (ul, peekChar ());
 872                 }
 873
 874                 //
 875                 // Invoked if we know we have .digits or digits
 876                 //
 877                 int is_number (int c)
 878                 {
 879                         bool is_real = false;
 880                         int type;
 881
 882                         number_pos = 0;
 883
 884                         if (c >= '0' && c <= '9'){
 885                                 if (c == '0'){
 886                                         int peek = peekChar ();
 887
 888                                         if (peek == 'x' || peek == 'X')
 889                                                 return handle_hex ();
 890                                 }
 891                                 decimal_digits (c);
 892                                 c = getChar ();
 893                         }
 894
 895                         //
 896                         // We need to handle the case of
 897                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
 898                         //
 899                         if (c == '.'){
 900                                 if (decimal_digits ('.')){
 901                                         is_real = true;
 902                                         c = getChar ();
 903                                 } else {
 904                                         putback ('.');
 905                                         number_pos--;
 906                                         return adjust_int (-1);
 907                                 }
 908                         }
 909
 910                         if (c == 'e' || c == 'E'){
 911                                 is_real = true;
 912                                 if (number_pos == max_number_size)
 913                                         Error_NumericConstantTooLong ();
 914                                 number_builder [number_pos++] = 'e';
 915                                 c = getChar ();
 916
 917                                 if (c == '+'){
 918                                         if (number_pos == max_number_size)
 919                                                 Error_NumericConstantTooLong ();
 920                                         number_builder [number_pos++] = '+';
 921                                         c = -1;
 922                                 } else if (c == '-') {
 923                                         if (number_pos == max_number_size)
 924                                                 Error_NumericConstantTooLong ();
 925                                         number_builder [number_pos++] = '-';
 926                                         c = -1;
 927                                 } else {
 928                                         if (number_pos == max_number_size)
 929                                                 Error_NumericConstantTooLong ();
 930                                         number_builder [number_pos++] = '+';
 931                                 }
 932
 933                                 decimal_digits (c);
 934                                 c = getChar ();
 935                         }
 936
 937                         type = real_type_suffix (c);
 938                         if (type == Token.NONE && !is_real){
 939                                 putback (c);
 940                                 return adjust_int (c);
 941                         } else
 942                                 is_real = true;
 943
 944                         if (type == Token.NONE){
 945                                 putback (c);
 946                         }
 947
 948                         if (is_real)
 949                                 return adjust_real (type);
 950
 951                         Console.WriteLine ("This should not be reached");
 952                         throw new Exception ("Is Number should never reach this point");
 953                 }
 954
 955                 //
 956                 // Accepts exactly count (4 or 8) hex, no more no less
 957                 //
 958                 int getHex (int count, out bool error)
 959                 {
 960                         int i;
 961                         int total = 0;
 962                         int c;
 963                         int top = count != -1 ? count : 4;
 964
 965                         getChar ();
 966                         error = false;
 967                         for (i = 0; i < top; i++){
 968                                 c = getChar ();
 969
 970                                 if (c >= '0' && c <= '9')
 971                                         c = (int) c - (int) '0';
 972                                 else if (c >= 'A' && c <= 'F')
 973                                         c = (int) c - (int) 'A' + 10;
 974                                 else if (c >= 'a' && c <= 'f')
 975                                         c = (int) c - (int) 'a' + 10;
 976                                 else {
 977                                         error = true;
 978                                         return 0;
 979                                 }
 980
 981                                 total = (total * 16) + c;
 982                                 if (count == -1){
 983                                         int p = peekChar ();
 984                                         if (p == -1)
 985                                                 break;
 986                                         if (!is_hex ((char)p))
 987                                                 break;
 988                                 }
 989                         }
 990                         return total;
 991                 }
 992
 993                 int escape (int c)
 994                 {
 995                         bool error;
 996                         int d;
 997                         int v;
 998
 999                         d = peekChar ();
1000                         if (c != '\\')
1001                                 return c;
1002
1003                         switch (d){
1004                         case 'a':
1005                                 v = '\a'; break;
1006                         case 'b':
1007                                 v = '\b'; break;
1008                         case 'n':
1009                                 v = '\n'; break;
1010                         case 't':
1011                                 v = '\t'; break;
1012                         case 'v':
1013                                 v = '\v'; break;
1014                         case 'r':
1015                                 v = '\r'; break;
1016                         case '\\':
1017                                 v = '\\'; break;
1018                         case 'f':
1019                                 v = '\f'; break;
1020                         case '0':
1021                                 v = 0; break;
1022                         case '"':
1023                                 v = '"'; break;
1024                         case '\'':
1025                                 v = '\''; break;
1026                         case 'x':
1027                                 v = getHex (-1, out error);
1028                                 if (error)
1029                                         goto default;
1030                                 return v;
1031                         case 'u':
1032                                 v = getHex (4, out error);
1033                                 if (error)
1034                                         goto default;
1035                                 return v;
1036                         case 'U':
1037                                 v = getHex (8, out error);
1038                                 if (error)
1039                                         goto default;
1040                                 return v;
1041                         default:
1042                                 Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);
1043                                 return d;
1044                         }
1045                         getChar ();
1046                         return v;
1047                 }
1048
1049                 int getChar ()
1050                 {
1051                         if (putback_char != -1){
1052                                 int x = putback_char;
1053                                 putback_char = -1;
1054
1055                                 return x;
1056                         }
1057                         return reader.Read ();
1058                 }
1059
1060                 int peekChar ()
1061                 {
1062                         if (putback_char != -1)
1063                                 return putback_char;
1064                         putback_char = reader.Read ();
1065                         return putback_char;
1066                 }
1067
1068                 int peekChar2 ()
1069                 {
1070                         if (putback_char != -1)
1071                                 return putback_char;
1072                         return reader.Peek ();
1073                 }
1074
1075                 void putback (int c)
1076                 {
1077                         if (putback_char != -1){
1078                                 Console.WriteLine ("Col: " + col);
1079                                 Console.WriteLine ("Row: " + line);
1080                                 Console.WriteLine ("Name: " + ref_name.Name);
1081                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1082                                 throw new Exception ("This should not happen putback on putback");
1083                         }
1084                         putback_char = c;
1085                 }
1086
1087                 public bool advance ()
1088                 {
1089                         return peekChar () != -1;
1090                 }
1091
1092                 public Object Value {
1093                         get {
1094                                 return val;
1095                         }
1096                 }
1097
1098                 public Object value ()
1099                 {
1100                         return val;
1101                 }
1102
1103                 bool IsCastToken (int token)
1104                 {
1105                         switch (token) {
1106                         case Token.BANG:
1107                         case Token.TILDE:
1108                         case Token.IDENTIFIER:
1109                         case Token.LITERAL_INTEGER:
1110                         case Token.LITERAL_FLOAT:
1111                         case Token.LITERAL_DOUBLE:
1112                         case Token.LITERAL_DECIMAL:
1113                         case Token.LITERAL_CHARACTER:
1114                         case Token.LITERAL_STRING:
1115                         case Token.BASE:
1116                         case Token.CHECKED:
1117                         case Token.FALSE:
1118                         case Token.FIXED:
1119                         case Token.NEW:
1120                         case Token.NULL:
1121                         case Token.SIZEOF:
1122                         case Token.THIS:
1123                         case Token.THROW:
1124                         case Token.TRUE:
1125                         case Token.TYPEOF:
1126                         case Token.UNCHECKED:
1127                         case Token.UNSAFE:
1128
1129                                 //
1130                                 // These can be part of a member access
1131                                 //
1132                         case Token.INT:
1133                         case Token.UINT:
1134                         case Token.SHORT:
1135                         case Token.USHORT:
1136                         case Token.LONG:
1137                         case Token.ULONG:
1138                         case Token.DOUBLE:
1139                         case Token.FLOAT:
1140                         case Token.CHAR:
1141                                 return true;
1142
1143                         default:
1144                                 return false;
1145                         }
1146                 }
1147
1148                 public int token ()
1149                 {
1150                         current_token = xtoken ();
1151                         return current_token;
1152                 }
1153
1154                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1155
1156                 void get_cmd_arg (out string cmd, out string arg)
1157                 {
1158                         int c;
1159
1160                         tokens_seen = false;
1161                         arg = "";
1162                         static_cmd_arg.Length = 0;
1163
1164                         // skip over white space
1165                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1166                                 ;
1167
1168
1169                         while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
1170                                 if (is_identifier_part_character ((char) c)){
1171                                         static_cmd_arg.Append ((char) c);
1172                                         c = getChar ();
1173                                 } else {
1174                                         putback (c);
1175                                         break;
1176                                 }
1177                         }
1178
1179                         cmd = static_cmd_arg.ToString ();
1180
1181                         if (c == '\n'){
1182                                 line++;
1183                                 ref_line++;
1184                                 return;
1185                         } else if (c == '\r')
1186                                 col = 0;
1187
1188                         // skip over white space
1189                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1190                                 ;
1191
1192                         if (c == '\n'){
1193                                 line++;
1194                                 ref_line++;
1195                                 return;
1196                         } else if (c == '\r'){
1197                                 col = 0;
1198                                 return;
1199                         } else if (c == -1){
1200                                 arg = "";
1201                                 return;
1202                         }
1203
1204                         static_cmd_arg.Length = 0;
1205                         static_cmd_arg.Append ((char) c);
1206
1207                         while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
1208                                 static_cmd_arg.Append ((char) c);
1209                         }
1210
1211                         if (c == '\n'){
1212                                 line++;
1213                                 ref_line++;
1214                         } else if (c == '\r')
1215                                 col = 0;
1216                         arg = static_cmd_arg.ToString ().Trim ();
1217                 }
1218
1219                 //
1220                 // Handles the #line directive
1221                 //
1222                 bool PreProcessLine (string arg)
1223                 {
1224                         if (arg == "")
1225                                 return false;
1226
1227                         if (arg == "default"){
1228                                 ref_line = line;
1229                                 ref_name = file_name;
1230                                 Location.Push (ref_name);
1231                                 return true;
1232                         } else if (arg == "hidden"){
1233                                 //
1234                                 // We ignore #line hidden
1235                                 //
1236                                 return true;
1237                         }
1238
1239                         try {
1240                                 int pos;
1241
1242                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1243                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1244                                         pos++;
1245
1246                                         char [] quotes = { '\"' };
1247
1248                                         string name = arg.Substring (pos). Trim (quotes);
1249                                         ref_name = Location.LookupFile (name);
1250                                         file_name.HasLineDirective = true;
1251                                         ref_name.HasLineDirective = true;
1252                                         Location.Push (ref_name);
1253                                 } else {
1254                                         ref_line = System.Int32.Parse (arg);
1255                                 }
1256                         } catch {
1257                                 return false;
1258                         }
1259
1260                         return true;
1261                 }
1262
1263                 //
1264                 // Handles #define and #undef
1265                 //
1266                 void PreProcessDefinition (bool is_define, string arg)
1267                 {
1268                         if (arg == "" || arg == "true" || arg == "false"){
1269                                 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
1270                                 return;
1271                         }
1272
1273                         char[] whitespace = { ' ', '\t' };
1274                         if (arg.IndexOfAny (whitespace) != -1){
1275                                 Report.Error (1025, Location, "Single-line comment or end-of-line expected");
1276                                 return;
1277                         }
1278
1279                         if (!is_identifier_start_character (arg [0]))
1280                                 Report.Error (1001, Location, "Identifier expected: " + arg);
1281
1282                         foreach (char c in arg.Substring (1)){
1283                                 if (!is_identifier_part_character (c)){
1284                                         Report.Error (1001, Location, "Identifier expected: " + arg);
1285                                         return;
1286                                 }
1287                         }
1288
1289                         if (is_define){
1290                                 if (defines == null)
1291                                         defines = new Hashtable ();
1292                                 define (arg);
1293                         } else {
1294                                 if (defines == null)
1295                                         return;
1296                                 if (defines.Contains (arg))
1297                                         defines.Remove (arg);
1298                         }
1299                 }
1300
1301                 /// <summary>
1302                 /// Handles #pragma directive
1303                 /// </summary>
1304                 void PreProcessPragma (string arg)
1305                 {
1306                         const string warning = "warning";
1307                         const string w_disable = "warning disable";
1308                         const string w_restore = "warning restore";
1309
1310                         if (arg == w_disable) {
1311                                 Report.RegisterWarningRegion (Location).WarningDisable (line);
1312                                 return;
1313                         }
1314
1315                         if (arg == w_restore) {
1316                                 Report.RegisterWarningRegion (Location).WarningEnable (line);
1317                                 return;
1318                         }
1319
1320                         if (arg.StartsWith (w_disable)) {
1321                                 int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
1322                                 foreach (int code in codes) {
1323                                         if (code != 0)
1324                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1325                                 }
1326                                 return;
1327                         }
1328
1329                         if (arg.StartsWith (w_restore)) {
1330                                 int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
1331                                 Hashtable w_table = Report.warning_ignore_table;
1332                                 foreach (int code in codes) {
1333                                         if (w_table != null && w_table.Contains (code))
1334                                                 Report.Warning (1635, 1, Location, "Cannot restore warning 'CS{0:0000}' because it was disabled globally", code);
1335                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1336                                 }
1337                                 return;
1338                         }
1339
1340                         if (arg.StartsWith (warning)) {
1341                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
1342                                 return;
1343                         }
1344
1345                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
1346                 }
1347
1348                 int[] ParseNumbers (string text)
1349                 {
1350                         string[] string_array = text.Split (',');
1351                         int[] values = new int [string_array.Length];
1352                         int index = 0;
1353                         foreach (string string_code in string_array) {
1354                                 try {
1355                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1356                                 }
1357                                 catch (FormatException) {
1358                                         Report.Warning (1692, Location, "Invalid number");
1359                                 }
1360                         }
1361                         return values;
1362                 }
1363
1364                 bool eval_val (string s)
1365                 {
1366                         if (s == "true")
1367                                 return true;
1368                         if (s == "false")
1369                                 return false;
1370
1371                         if (defines == null)
1372                                 return false;
1373                         if (defines.Contains (s))
1374                                 return true;
1375
1376                         return false;
1377                 }
1378
1379                 bool pp_primary (ref string s)
1380                 {
1381                         s = s.Trim ();
1382                         int len = s.Length;
1383
1384                         if (len > 0){
1385                                 char c = s [0];
1386
1387                                 if (c == '('){
1388                                         s = s.Substring (1);
1389                                         bool val = pp_expr (ref s);
1390                                         if (s.Length > 0 && s [0] == ')'){
1391                                                 s = s.Substring (1);
1392                                                 return val;
1393                                         }
1394                                         Error_InvalidDirective ();
1395                                         return false;
1396                                 }
1397
1398                                 if (is_identifier_start_character (c)){
1399                                         int j = 1;
1400
1401                                         while (j < len){
1402                                                 c = s [j];
1403
1404                                                 if (is_identifier_part_character (c)){
1405                                                         j++;
1406                                                         continue;
1407                                                 }
1408                                                 bool v = eval_val (s.Substring (0, j));
1409                                                 s = s.Substring (j);
1410                                                 return v;
1411                                         }
1412                                         bool vv = eval_val (s);
1413                                         s = "";
1414                                         return vv;
1415                                 }
1416                         }
1417                         Error_InvalidDirective ();
1418                         return false;
1419                 }
1420
1421                 bool pp_unary (ref string s)
1422                 {
1423                         s = s.Trim ();
1424                         int len = s.Length;
1425
1426                         if (len > 0){
1427                                 if (s [0] == '!'){
1428                                         if (len > 1 && s [1] == '='){
1429                                                 Error_InvalidDirective ();
1430                                                 return false;
1431                                         }
1432                                         s = s.Substring (1);
1433                                         return ! pp_primary (ref s);
1434                                 } else
1435                                         return pp_primary (ref s);
1436                         } else {
1437                                 Error_InvalidDirective ();
1438                                 return false;
1439                         }
1440                 }
1441
1442                 bool pp_eq (ref string s)
1443                 {
1444                         bool va = pp_unary (ref s);
1445
1446                         s = s.Trim ();
1447                         int len = s.Length;
1448                         if (len > 0){
1449                                 if (s [0] == '='){
1450                                         if (len > 2 && s [1] == '='){
1451                                                 s = s.Substring (2);
1452                                                 return va == pp_unary (ref s);
1453                                         } else {
1454                                                 Error_InvalidDirective ();
1455                                                 return false;
1456                                         }
1457                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
1458                                         s = s.Substring (2);
1459
1460                                         return va != pp_unary (ref s);
1461
1462                                 }
1463                         }
1464
1465                         return va;
1466
1467                 }
1468
1469                 bool pp_and (ref string s)
1470                 {
1471                         bool va = pp_eq (ref s);
1472
1473                         s = s.Trim ();
1474                         int len = s.Length;
1475                         if (len > 0){
1476                                 if (s [0] == '&'){
1477                                         if (len > 2 && s [1] == '&'){
1478                                                 s = s.Substring (2);
1479                                                 return (va & pp_and (ref s));
1480                                         } else {
1481                                                 Error_InvalidDirective ();
1482                                                 return false;
1483                                         }
1484                                 }
1485                         }
1486                         return va;
1487                 }
1488
1489                 //
1490                 // Evaluates an expression for `#if' or `#elif'
1491                 //
1492                 bool pp_expr (ref string s)
1493                 {
1494                         bool va = pp_and (ref s);
1495                         s = s.Trim ();
1496                         int len = s.Length;
1497                         if (len > 0){
1498                                 char c = s [0];
1499
1500                                 if (c == '|'){
1501                                         if (len > 2 && s [1] == '|'){
1502                                                 s = s.Substring (2);
1503                                                 return va | pp_expr (ref s);
1504                                         } else {
1505                                                 Error_InvalidDirective ();
1506                                                 return false;
1507                                         }
1508                                 }
1509                         }
1510
1511                         return va;
1512                 }
1513
1514                 bool eval (string s)
1515                 {
1516                         bool v = pp_expr (ref s);
1517                         s = s.Trim ();
1518                         if (s.Length != 0){
1519                                 Error_InvalidDirective ();
1520                                 return false;
1521                         }
1522
1523                         return v;
1524                 }
1525
1526                 void Error_InvalidDirective ()
1527                 {
1528                         Report.Error (1517, Location, "Invalid pre-processor directive");
1529                 }
1530
1531                 void Error_UnexpectedDirective (string extra)
1532                 {
1533                         Report.Error (
1534                                 1028, Location,
1535                                 "Unexpected processor directive (" + extra + ")");
1536                 }
1537
1538                 void Error_TokensSeen ()
1539                 {
1540                         Report.Error (
1541                                 1032, Location,
1542                                 "Cannot define or undefine pre-processor symbols after a token in the file");
1543                 }
1544
1545                 //
1546                 // if true, then the code continues processing the code
1547                 // if false, the code stays in a loop until another directive is
1548                 // reached.
1549                 //
1550                 bool handle_preprocessing_directive (bool caller_is_taking)
1551                 {
1552                         string cmd, arg;
1553                         bool region_directive = false;
1554
1555                         get_cmd_arg (out cmd, out arg);
1556
1557                         // Eat any trailing whitespaces and single-line comments
1558                         if (arg.IndexOf ("//") != -1)
1559                                 arg = arg.Substring (0, arg.IndexOf ("//"));
1560                         arg = arg.TrimEnd (' ', '\t');
1561
1562                         //
1563                         // The first group of pre-processing instructions is always processed
1564                         //
1565                         switch (cmd){
1566                         case "pragma":
1567                                 if (RootContext.Version == LanguageVersion.ISO_1) {
1568                                         Report.FeatureIsNotStandardized (Location, "#pragma");
1569                                         return caller_is_taking;
1570                                 }
1571
1572                                 PreProcessPragma (arg);
1573                                 return caller_is_taking;
1574
1575                         case "line":
1576                                 if (!PreProcessLine (arg))
1577                                         Report.Error (
1578                                                 1576, Location,
1579                                                 "Argument to #line directive is missing or invalid");
1580                                 return caller_is_taking;
1581
1582                         case "region":
1583                                 region_directive = true;
1584                                 arg = "true";
1585                                 goto case "if";
1586
1587                         case "endregion":
1588                                 region_directive = true;
1589                                 goto case "endif";
1590
1591                         case "if":
1592                                 if (arg == ""){
1593                                         Error_InvalidDirective ();
1594                                         return true;
1595                                 }
1596                                 bool taking = false;
1597                                 if (ifstack == null)
1598                                         ifstack = new Stack ();
1599
1600                                 if (ifstack.Count == 0){
1601                                         taking = true;
1602                                 } else {
1603                                         int state = (int) ifstack.Peek ();
1604                                         if ((state & TAKING) != 0)
1605                                                 taking = true;
1606                                 }
1607
1608                                 if (eval (arg) && taking){
1609                                         int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;
1610                                         if (region_directive)
1611                                                 push |= REGION;
1612                                         ifstack.Push (push);
1613                                         return true;
1614                                 } else {
1615                                         int push = (taking ? PARENT_TAKING : 0);
1616                                         if (region_directive)
1617                                                 push |= REGION;
1618                                         ifstack.Push (push);
1619                                         return false;
1620                                 }
1621
1622                         case "endif":
1623                                 if (ifstack == null || ifstack.Count == 0){
1624                                         Error_UnexpectedDirective ("no #if for this #endif");
1625                                         return true;
1626                                 } else {
1627                                         int pop = (int) ifstack.Pop ();
1628
1629                                         if (region_directive && ((pop & REGION) == 0))
1630                                                 Report.Error (1027, Location, "#endif directive expected");
1631                                         else if (!region_directive && ((pop & REGION) != 0))
1632                                                 Report.Error (1038, Location, "#endregion directive expected");
1633
1634                                         if (!region_directive && arg.Length != 0) {
1635                                                 Report.Error (1025, Location,
1636                                                         "Single line comment, or end-of-line expected");
1637                                         }
1638
1639                                         if (ifstack.Count == 0)
1640                                                 return true;
1641                                         else {
1642                                                 int state = (int) ifstack.Peek ();
1643
1644                                                 if ((state & TAKING) != 0)
1645                                                         return true;
1646                                                 else
1647                                                         return false;
1648                                         }
1649                                 }
1650
1651                         case "elif":
1652                                 if (ifstack == null || ifstack.Count == 0){
1653                                         Error_UnexpectedDirective ("no #if for this #elif");
1654                                         return true;
1655                                 } else {
1656                                         int state = (int) ifstack.Peek ();
1657
1658                                         if ((state & REGION) != 0) {
1659                                                 Report.Error (1038, Location, "#endregion directive expected");
1660                                                 return true;
1661                                         }
1662
1663                                         if ((state & ELSE_SEEN) != 0){
1664                                                 Error_UnexpectedDirective ("#elif not valid after #else");
1665                                                 return true;
1666                                         }
1667
1668                                         if ((state & (TAKEN_BEFORE | TAKING)) != 0)
1669                                                 return false;
1670
1671                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
1672                                                 state = (int) ifstack.Pop ();
1673                                                 ifstack.Push (state | TAKING | TAKEN_BEFORE);
1674                                                 return true;
1675                                         } else
1676                                                 return false;
1677                                 }
1678
1679                         case "else":
1680                                 if (ifstack == null || ifstack.Count == 0){
1681                                         Report.Error (
1682                                                 1028, Location,
1683                                                 "Unexpected processor directive (no #if for this #else)");
1684                                         return true;
1685                                 } else {
1686                                         int state = (int) ifstack.Peek ();
1687
1688                                         if ((state & REGION) != 0) {
1689                                                 Report.Error (1038, Location, "#endregion directive expected");
1690                                                 return true;
1691                                         }
1692
1693                                         if ((state & ELSE_SEEN) != 0){
1694                                                 Error_UnexpectedDirective ("#else within #else");
1695                                                 return true;
1696                                         }
1697
1698                                         ifstack.Pop ();
1699
1700                                         bool ret;
1701                                         if ((state & TAKEN_BEFORE) == 0){
1702                                                 ret = ((state & PARENT_TAKING) != 0);
1703                                         } else
1704                                                 ret = false;
1705
1706                                         if (ret)
1707                                                 state |= TAKING;
1708                                         else
1709                                                 state &= ~TAKING;
1710
1711                                         ifstack.Push (state | ELSE_SEEN);
1712
1713                                         return ret;
1714                                 }
1715                         }
1716
1717                         //
1718                         // These are only processed if we are in a `taking' block
1719                         //
1720                         if (!caller_is_taking)
1721                                 return false;
1722
1723                         switch (cmd){
1724                         case "define":
1725                                 if (any_token_seen){
1726                                         Error_TokensSeen ();
1727                                         return true;
1728                                 }
1729                                 PreProcessDefinition (true, arg);
1730                                 return true;
1731
1732                         case "undef":
1733                                 if (any_token_seen){
1734                                         Error_TokensSeen ();
1735                                         return true;
1736                                 }
1737                                 PreProcessDefinition (false, arg);
1738                                 return true;
1739
1740                         case "error":
1741                                 Report.Error (1029, Location, "#error: '" + arg + "'");
1742                                 return true;
1743
1744                         case "warning":
1745                                 Report.Warning (1030, Location, "#warning: '{0}'", arg);
1746                                 return true;
1747                         }
1748
1749                         Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");
1750                         return true;
1751
1752                 }
1753
1754                 private int consume_string (bool quoted)
1755                 {
1756                         int c;
1757                         string_builder.Length = 0;
1758
1759                         while ((c = getChar ()) != -1){
1760                                 if (c == '"'){
1761                                         if (quoted && peekChar () == '"'){
1762                                                 string_builder.Append ((char) c);
1763                                                 getChar ();
1764                                                 continue;
1765                                         } else {
1766                                                 val = string_builder.ToString ();
1767                                                 return Token.LITERAL_STRING;
1768                                         }
1769                                 }
1770
1771                                 if (c == '\n'){
1772                                         if (!quoted)
1773                                                 Report.Error (1010, Location, "Newline in constant");
1774                                         line++;
1775                                         ref_line++;
1776                                         col = 0;
1777                                 } else
1778                                         col++;
1779
1780                                 if (!quoted){
1781                                         c = escape (c);
1782                                         if (c == -1)
1783                                                 return Token.ERROR;
1784                                 }
1785                                 string_builder.Append ((char) c);
1786                         }
1787
1788                         Report.Error (1039, Location, "Unterminated string literal");
1789                         return Token.EOF;
1790                 }
1791
1792                 private int consume_identifier (int s)
1793                 {
1794                         int res = consume_identifier (s, false);
1795
1796                         if (doc_state == XmlCommentState.Allowed)
1797                                 doc_state = XmlCommentState.NotAllowed;
1798                         switch (res) {
1799                         case Token.USING:
1800                         case Token.NAMESPACE:
1801                                 check_incorrect_doc_comment ();
1802                                 break;
1803                         }
1804
1805                         if (res == Token.PARTIAL) {
1806                                 // Save current position and parse next token.
1807                                 int old = reader.Position;
1808                                 int old_putback = putback_char;
1809
1810                                 putback_char = -1;
1811
1812                                 int next_token = token ();
1813                                 bool ok = (next_token == Token.CLASS) ||
1814                                         (next_token == Token.STRUCT) ||
1815                                         (next_token == Token.INTERFACE) ||
1816                                         (next_token == Token.ENUM); // "partial" is a keyword in 'partial enum', even though it's not valid
1817
1818                                 reader.Position = old;
1819                                 putback_char = old_putback;
1820
1821                                 if (ok)
1822                                         return res;
1823                                 else {
1824                                         val = "partial";
1825                                         return Token.IDENTIFIER;
1826                                 }
1827                         }
1828
1829                         return res;
1830                 }
1831
1832                 private int consume_identifier (int s, bool quoted)
1833                 {
1834                         int pos = 1;
1835                         int c;
1836
1837                         id_builder [0] = (char) s;
1838
1839                         while ((c = reader.Read ()) != -1) {
1840                                 if (is_identifier_part_character ((char) c)){
1841                                         if (pos == max_id_size){
1842                                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
1843                                                 return Token.ERROR;
1844                                         }
1845
1846                                         id_builder [pos++] = (char) c;
1847                                         putback_char = -1;
1848                                         col++;
1849                                 } else {
1850                                         putback_char = c;
1851                                         break;
1852                                 }
1853                         }
1854
1855                         //
1856                         // Optimization: avoids doing the keyword lookup
1857                         // on uppercase letters and _
1858                         //
1859                         if (!quoted && (s >= 'a' || s == '_')){
1860                                 int keyword = GetKeyword (id_builder, pos);
1861                                 if (keyword != -1)
1862                                         return keyword;
1863                         }
1864
1865                         //
1866                         // Keep identifiers in an array of hashtables to avoid needless
1867                         // allocations
1868                         //
1869
1870                         if (identifiers [pos] != null) {
1871                                 val = identifiers [pos][id_builder];
1872                                 if (val != null) {
1873                                         return Token.IDENTIFIER;
1874                                 }
1875                         }
1876                         else
1877                                 identifiers [pos] = new CharArrayHashtable (pos);
1878
1879                         val = new String (id_builder, 0, pos);
1880
1881                         char [] chars = new char [pos];
1882                         Array.Copy (id_builder, chars, pos);
1883
1884                         identifiers [pos] [chars] = val;
1885
1886                         return Token.IDENTIFIER;
1887                 }
1888
1889                 public int xtoken ()
1890                 {
1891                         int t;
1892                         bool doread = false;
1893                         int c;
1894
1895                         // Whether we have seen comments on the current line
1896                         bool comments_seen = false;
1897
1898                         val = null;
1899                         // optimization: eliminate col and implement #directive semantic correctly.
1900                         for (;(c = getChar ()) != -1; col++) {
1901                                 if (c == ' ')
1902                                         continue;
1903
1904                                 if (c == '\t') {
1905                                         col = (((col + 8) / 8) * 8) - 1;
1906                                         continue;
1907                                 }
1908
1909                                 if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0)
1910                                         continue;
1911
1912                                 if (c == '\r') {
1913                                         if (peekChar () == '\n')
1914                                                 getChar ();
1915
1916                                         line++;
1917                                         ref_line++;
1918                                         col = 0;
1919                                         any_token_seen |= tokens_seen;
1920                                         tokens_seen = false;
1921                                         comments_seen = false;
1922                                         continue;
1923                                 }
1924
1925                                 // Handle double-slash comments.
1926                                 if (c == '/'){
1927                                         int d = peekChar ();
1928
1929                                         if (d == '/'){
1930                                                 getChar ();
1931                                                 if (RootContext.Documentation != null && peekChar () == '/') {
1932                                                         getChar ();
1933                                                         // Don't allow ////.
1934                                                         if ((d = peekChar ()) != '/') {
1935                                                                 if (doc_state == XmlCommentState.Allowed)
1936                                                                         handle_one_line_xml_comment ();
1937                                                                 else if (doc_state == XmlCommentState.NotAllowed)
1938                                                                         warn_incorrect_doc_comment ();
1939                                                         }
1940                                                 }
1941                                                 while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
1942                                                         col++;
1943                                                 if (d == '\n'){
1944                                                         line++;
1945                                                         ref_line++;
1946                                                         col = 0;
1947                                                 }
1948                                                 any_token_seen |= tokens_seen;
1949                                                 tokens_seen = false;
1950                                                 comments_seen = false;
1951                                                 continue;
1952                                         } else if (d == '*'){
1953                                                 getChar ();
1954                                                 bool docAppend = false;
1955                                                 if (RootContext.Documentation != null && peekChar () == '*') {
1956                                                         getChar ();
1957                                                         // But when it is /**/, just do nothing.
1958                                                         if (peekChar () == '/') {
1959                                                                 getChar ();
1960                                                                 continue;
1961                                                         }
1962                                                         if (doc_state == XmlCommentState.Allowed)
1963                                                                 docAppend = true;
1964                                                         else if (doc_state == XmlCommentState.NotAllowed)
1965                                                                 warn_incorrect_doc_comment ();
1966                                                 }
1967
1968                                                 int current_comment_start = 0;
1969                                                 if (docAppend) {
1970                                                         current_comment_start = xml_comment_buffer.Length;
1971                                                         xml_comment_buffer.Append (Environment.NewLine);
1972                                                 }
1973
1974                                                 Location start_location = Location;
1975
1976                                                 while ((d = getChar ()) != -1){
1977                                                         if (d == '*' && peekChar () == '/'){
1978                                                                 getChar ();
1979                                                                 col++;
1980                                                                 comments_seen = true;
1981                                                                 break;
1982                                                         }
1983                                                         if (docAppend)
1984                                                                 xml_comment_buffer.Append ((char) d);
1985
1986                                                         if (d == '\n'){
1987                                                                 line++;
1988                                                                 ref_line++;
1989                                                                 col = 0;
1990                                                                 any_token_seen |= tokens_seen;
1991                                                                 tokens_seen = false;
1992                                                                 //
1993                                                                 // Reset 'comments_seen' just to be consistent.
1994                                                                 // It doesn't matter either way, here.
1995                                                                 //
1996                                                                 comments_seen = false;
1997                                                         }
1998                                                 }
1999                                                 if (!comments_seen)
2000                                                         Report.Error (1035, start_location, "End-of-file found, '*/' expected");
2001
2002                                                 if (docAppend)
2003                                                         update_formatted_doc_comment (current_comment_start);
2004                                                 continue;
2005                                         }
2006                                         goto is_punct_label;
2007                                 }
2008
2009
2010                                 if (is_identifier_start_character ((char)c)){
2011                                         tokens_seen = true;
2012                                         return consume_identifier (c);
2013                                 }
2014
2015                         is_punct_label:
2016                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
2017                                         tokens_seen = true;
2018                                         if (doread){
2019                                                 getChar ();
2020                                                 col++;
2021                                         }
2022                                         return t;
2023                                 }
2024
2025                                 // white space
2026                                 if (c == '\n'){
2027                                         line++;
2028                                         ref_line++;
2029                                         col = 0;
2030                                         any_token_seen |= tokens_seen;
2031                                         tokens_seen = false;
2032                                         comments_seen = false;
2033                                         continue;
2034                                 }
2035
2036                                 if (c >= '0' && c <= '9'){
2037                                         tokens_seen = true;
2038                                         return is_number (c);
2039                                 }
2040
2041                                 if (c == '.'){
2042                                         tokens_seen = true;
2043                                         int peek = peekChar ();
2044                                         if (peek >= '0' && peek <= '9')
2045                                                 return is_number (c);
2046                                         return Token.DOT;
2047                                 }
2048
2049                                 /* For now, ignore pre-processor commands */
2050                                 // FIXME: In C# the '#' is not limited to appear
2051                                 // on the first column.
2052                                 if (c == '#') {
2053                                         bool cont = true;
2054
2055                                         if (tokens_seen || comments_seen) {
2056                                                error_details = "Preprocessor directives must appear as the first non-whitespace " +
2057                                                        "character on a line.";
2058
2059                                                Report.Error (1040, Location, error_details);
2060
2061                                                return Token.ERROR;
2062                                        }
2063
2064                                 start_again:
2065
2066                                         cont = handle_preprocessing_directive (cont);
2067
2068                                         if (cont){
2069                                                 col = 0;
2070                                                 continue;
2071                                         }
2072                                         col = 1;
2073
2074                                         bool skipping = false;
2075                                         for (;(c = getChar ()) != -1; col++){
2076                                                 if (c == '\n'){
2077                                                         col = 0;
2078                                                         line++;
2079                                                         ref_line++;
2080                                                         skipping = false;
2081                                                 } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
2082                                                         continue;
2083                                                 else if (c != '#')
2084                                                         skipping = true;
2085                                                 if (c == '#' && !skipping)
2086                                                         goto start_again;
2087                                         }
2088                                         any_token_seen |= tokens_seen;
2089                                         tokens_seen = false;
2090                                         if (c == -1)
2091                                                 Report.Error (1027, Location, "#endif/#endregion expected");
2092                                         continue;
2093                                 }
2094
2095                                 if (c == '"')
2096                                         return consume_string (false);
2097
2098                                 if (c == '\''){
2099                                         c = getChar ();
2100                                         tokens_seen = true;
2101                                         if (c == '\''){
2102                                                 error_details = "Empty character literal";
2103                                                 Report.Error (1011, Location, error_details);
2104                                                 return Token.ERROR;
2105                                         }
2106                                         c = escape (c);
2107                                         if (c == -1)
2108                                                 return Token.ERROR;
2109                                         val = new System.Char ();
2110                                         val = (char) c;
2111                                         c = getChar ();
2112
2113                                         if (c != '\''){
2114                                                 error_details = "Too many characters in character literal";
2115                                                 Report.Error (1012, Location, error_details);
2116
2117                                                 // Try to recover, read until newline or next "'"
2118                                                 while ((c = getChar ()) != -1){
2119                                                         if (c == '\n' || c == '\''){
2120                                                                 line++;
2121                                                                 ref_line++;
2122                                                                 col = 0;
2123                                                                 break;
2124                                                         } else
2125                                                                 col++;
2126
2127                                                 }
2128                                                 return Token.ERROR;
2129                                         }
2130                                         return Token.LITERAL_CHARACTER;
2131                                 }
2132
2133                                 if (c == '@') {
2134                                         c = getChar ();
2135                                         if (c == '"') {
2136                                                 tokens_seen = true;
2137                                                 return consume_string (true);
2138                                         } else if (is_identifier_start_character ((char) c)){
2139                                                 return consume_identifier (c, true);
2140                                         } else {
2141                                                 Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
2142                                         }
2143                                 }
2144
2145                                 error_details = ((char)c).ToString ();
2146
2147                                 return Token.ERROR;
2148                         }
2149
2150                         return Token.EOF;
2151                 }
2152
2153                 //
2154                 // Handles one line xml comment
2155                 //
2156                 private void handle_one_line_xml_comment ()
2157                 {
2158                         int c;
2159                         while ((c = peekChar ()) == ' ')
2160                                 getChar (); // skip heading whitespaces.
2161                         while ((c = peekChar ()) != -1 && c != '\n' && c != '\r') {
2162                                 col++;
2163                                 xml_comment_buffer.Append ((char) getChar ());
2164                         }
2165                         if (c == '\r' || c == '\n')
2166                                 xml_comment_buffer.Append (Environment.NewLine);
2167                 }
2168
2169                 //
2170                 // Remove heading "*" in Javadoc-like xml documentation.
2171                 //
2172                 private void update_formatted_doc_comment (int current_comment_start)
2173                 {
2174                         int length = xml_comment_buffer.Length - current_comment_start;
2175                         string [] lines = xml_comment_buffer.ToString (
2176                                 current_comment_start,
2177                                 length).Replace ("\r", "").Split ('\n');
2178
2179                         // The first line starts with /**, thus it is not target
2180                         // for the format check.
2181                         for (int i = 1; i < lines.Length; i++) {
2182                                 string s = lines [i];
2183                                 int idx = s.IndexOf ('*');
2184                                 string head = null;
2185                                 if (idx < 0) {
2186                                         if (i < lines.Length - 1)
2187                                                 return;
2188                                         head = s;
2189                                 } else
2190                                         head = s.Substring (0, idx);
2191                                 foreach (char c in head)
2192                                         if (c != ' ')
2193                                                 return;
2194                                 lines [i] = s.Substring (idx + 1);
2195                         }
2196                         xml_comment_buffer.Remove (current_comment_start, length);
2197                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
2198                 }
2199
2200                 //
2201                 // Checks if there was incorrect doc comments and raise
2202                 // warnings.
2203                 //
2204                 public void check_incorrect_doc_comment ()
2205                 {
2206                         if (xml_comment_buffer.Length > 0)
2207                                 warn_incorrect_doc_comment ();
2208                 }
2209
2210                 //
2211                 // Raises a warning when tokenizer found incorrect doccomment
2212                 // markup.
2213                 //
2214                 private void warn_incorrect_doc_comment ()
2215                 {
2216                         doc_state = XmlCommentState.Error;
2217                         // in csc, it is 'XML comment is not placed on a valid
2218                         // language element'. But that does not make sense.
2219                         Report.Warning (1587, 2, Location, "XML comment is placed on an invalid language element which can not accept it.");
2220                 }
2221
2222                 //
2223                 // Consumes the saved xml comment lines (if any)
2224                 // as for current target member or type.
2225                 //
2226                 public string consume_doc_comment ()
2227                 {
2228                         if (xml_comment_buffer.Length > 0) {
2229                                 string ret = xml_comment_buffer.ToString ();
2230                                 xml_comment_buffer.Length = 0;
2231                                 return ret;
2232                         }
2233                         return null;
2234                 }
2235
2236                 public void cleanup ()
2237                 {
2238                         if (ifstack != null && ifstack.Count >= 1) {
2239                                 int state = (int) ifstack.Pop ();
2240                                 if ((state & REGION) != 0)
2241                                         Report.Error (1038, Location, "#endregion directive expected");
2242                                 else
2243                                         Report.Error (1027, "#endif directive expected");
2244                         }
2245
2246                 }
2247         }
2248
2249         //
2250         // Indicates whether it accepts XML documentation or not.
2251         //
2252         public enum XmlCommentState {
2253                 // comment is allowed in this state.
2254                 Allowed,
2255                 // comment is not allowed in this state.
2256                 NotAllowed,
2257                 // once comments appeared when it is NotAllowed, then the
2258                 // state is changed to it, until the state is changed to
2259                 // .Allowed.
2260                 Error
2261         }
2262 }