mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //
   7 // Licensed under the terms of the GNU GPL
   8 //
   9 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
  10 // (C) 2004 Novell, Inc
  11 //
  12
  13 /*
  14  * TODO:
  15  *   Make sure we accept the proper Unicode ranges, per the spec.
  16  *   Report error 1032
  17 */
  18
  19 using System;
  20 using System.Text;
  21 using System.Collections;
  22 using System.IO;
  23 using System.Globalization;
  24 using System.Reflection;
  25
  26 namespace Mono.CSharp
  27 {
  28         /// <summary>
  29         ///    Tokenizer for C# source code.
  30         /// </summary>
  31
  32         public class Tokenizer : yyParser.yyInput
  33         {
  34                 SeekableStreamReader reader;
  35                 public SourceFile ref_name;
  36                 public SourceFile file_name;
  37                 public int ref_line = 1;
  38                 public int line = 1;
  39                 public int col = 1;
  40                 public int current_token;
  41                 bool handle_get_set = false;
  42                 bool handle_remove_add = false;
  43                 bool handle_assembly = false;
  44
  45                 //
  46                 // XML documentation buffer. The save point is used to divide
  47                 // comments on types and comments on members.
  48                 //
  49                 StringBuilder xml_comment_buffer;
  50
  51                 //
  52                 // See comment on XmlCommentState enumeration.
  53                 //
  54                 XmlCommentState xmlDocState = XmlCommentState.Allowed;
  55
  56                 //
  57                 // Whether tokens have been seen on this line
  58                 //
  59                 bool tokens_seen = false;
  60
  61                 //
  62                 // Whether a token has been seen on the file
  63                 // This is needed because `define' is not allowed to be used
  64                 // after a token has been seen.
  65                 //
  66                 bool any_token_seen = false;
  67                 static Hashtable tokenValues;
  68
  69                 private static Hashtable TokenValueName
  70                 {
  71                         get {
  72                                 if (tokenValues == null)
  73                                         tokenValues = GetTokenValueNameHash ();
  74
  75                                 return tokenValues;
  76                         }
  77                 }
  78
  79                 private static Hashtable GetTokenValueNameHash ()
  80                 {
  81                         Type t = typeof (Token);
  82                         FieldInfo [] fields = t.GetFields ();
  83                         Hashtable hash = new Hashtable ();
  84                         foreach (FieldInfo field in fields) {
  85                                 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))
  86                                         hash.Add (field.GetValue (null), field.Name);
  87                         }
  88                         return hash;
  89                 }
  90
  91                 //
  92                 // Returns a verbose representation of the current location
  93                 //
  94                 public string location {
  95                         get {
  96                                 string det;
  97
  98                                 if (current_token == Token.ERROR)
  99                                         det = "detail: " + error_details;
 100                                 else
 101                                         det = "";
 102
 103                                 // return "Line:     "+line+" Col: "+col + "\n" +
 104                                 //       "VirtLine: "+ref_line +
 105                                 //       " Token: "+current_token + " " + det;
 106                                 string current_token_name = TokenValueName [current_token] as string;
 107                                 if (current_token_name == null)
 108                                         current_token_name = current_token.ToString ();
 109
 110                                 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,
 111                                                                                        ref_line,
 112                                                                                        col,
 113                                                                                        current_token_name,
 114                                                                                        det);
 115                         }
 116                 }
 117
 118                 public bool PropertyParsing {
 119                         get {
 120                                 return handle_get_set;
 121                         }
 122
 123                         set {
 124                                 handle_get_set = value;
 125                         }
 126                 }
 127
 128                 public bool AssemblyTargetParsing {
 129                         get {
 130                                 return handle_assembly;
 131                         }
 132
 133                         set {
 134                                 handle_assembly = value;
 135                         }
 136                 }
 137
 138                 public bool EventParsing {
 139                         get {
 140                                 return handle_remove_add;
 141                         }
 142
 143                         set {
 144                                 handle_remove_add = value;
 145                         }
 146                 }
 147
 148                 public XmlCommentState doc_state {
 149                         get { return xmlDocState; }
 150                         set {
 151                                 if (value == XmlCommentState.Allowed) {
 152                                         check_incorrect_doc_comment ();
 153                                         consume_doc_comment ();
 154                                 }
 155                                 xmlDocState = value;
 156                         }
 157                 }
 158
 159
 160                 //
 161                 // Class variables
 162                 //
 163                 static CharArrayHashtable[] keywords;
 164                 static NumberStyles styles;
 165                 static NumberFormatInfo csharp_format_info;
 166
 167                 //
 168                 // Values for the associated token returned
 169                 //
 170                 int putback_char;
 171                 Object val;
 172
 173                 //
 174                 // Pre-processor
 175                 //
 176                 Hashtable defines;
 177
 178                 const int TAKING        = 1;
 179                 const int TAKEN_BEFORE  = 2;
 180                 const int ELSE_SEEN     = 4;
 181                 const int PARENT_TAKING = 8;
 182                 const int REGION        = 16;
 183
 184                 //
 185                 // pre-processor if stack state:
 186                 //
 187                 Stack ifstack;
 188
 189                 static System.Text.StringBuilder string_builder;
 190
 191                 const int max_id_size = 512;
 192                 static char [] id_builder = new char [max_id_size];
 193
 194                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 195
 196                 const int max_number_size = 128;
 197                 static char [] number_builder = new char [max_number_size];
 198                 static int number_pos;
 199
 200                 //
 201                 // Details about the error encoutered by the tokenizer
 202                 //
 203                 string error_details;
 204
 205                 public string error {
 206                         get {
 207                                 return error_details;
 208                         }
 209                 }
 210
 211                 public int Line {
 212                         get {
 213                                 return ref_line;
 214                         }
 215                 }
 216
 217                 public int Col {
 218                         get {
 219                                 return col;
 220                         }
 221                 }
 222
 223                 static void AddKeyword (string kw, int token) {
 224                         if (keywords [kw.Length] == null) {
 225                                 keywords [kw.Length] = new CharArrayHashtable (kw.Length);
 226                         }
 227                         keywords [kw.Length] [kw.ToCharArray ()] = token;
 228                 }
 229
 230                 static void InitTokens ()
 231                 {
 232                         keywords = new CharArrayHashtable [64];
 233
 234                         AddKeyword ("__arglist", Token.ARGLIST);
 235                         AddKeyword ("abstract", Token.ABSTRACT);
 236                         AddKeyword ("as", Token.AS);
 237                         AddKeyword ("add", Token.ADD);
 238                         AddKeyword ("assembly", Token.ASSEMBLY);
 239                         AddKeyword ("base", Token.BASE);
 240                         AddKeyword ("bool", Token.BOOL);
 241                         AddKeyword ("break", Token.BREAK);
 242                         AddKeyword ("byte", Token.BYTE);
 243                         AddKeyword ("case", Token.CASE);
 244                         AddKeyword ("catch", Token.CATCH);
 245                         AddKeyword ("char", Token.CHAR);
 246                         AddKeyword ("checked", Token.CHECKED);
 247                         AddKeyword ("class", Token.CLASS);
 248                         AddKeyword ("const", Token.CONST);
 249                         AddKeyword ("continue", Token.CONTINUE);
 250                         AddKeyword ("decimal", Token.DECIMAL);
 251                         AddKeyword ("default", Token.DEFAULT);
 252                         AddKeyword ("delegate", Token.DELEGATE);
 253                         AddKeyword ("do", Token.DO);
 254                         AddKeyword ("double", Token.DOUBLE);
 255                         AddKeyword ("else", Token.ELSE);
 256                         AddKeyword ("enum", Token.ENUM);
 257                         AddKeyword ("event", Token.EVENT);
 258                         AddKeyword ("explicit", Token.EXPLICIT);
 259                         AddKeyword ("extern", Token.EXTERN);
 260                         AddKeyword ("false", Token.FALSE);
 261                         AddKeyword ("finally", Token.FINALLY);
 262                         AddKeyword ("fixed", Token.FIXED);
 263                         AddKeyword ("float", Token.FLOAT);
 264                         AddKeyword ("for", Token.FOR);
 265                         AddKeyword ("foreach", Token.FOREACH);
 266                         AddKeyword ("goto", Token.GOTO);
 267                         AddKeyword ("get", Token.GET);
 268                         AddKeyword ("if", Token.IF);
 269                         AddKeyword ("implicit", Token.IMPLICIT);
 270                         AddKeyword ("in", Token.IN);
 271                         AddKeyword ("int", Token.INT);
 272                         AddKeyword ("interface", Token.INTERFACE);
 273                         AddKeyword ("internal", Token.INTERNAL);
 274                         AddKeyword ("is", Token.IS);
 275                         AddKeyword ("lock", Token.LOCK);
 276                         AddKeyword ("long", Token.LONG);
 277                         AddKeyword ("namespace", Token.NAMESPACE);
 278                         AddKeyword ("new", Token.NEW);
 279                         AddKeyword ("null", Token.NULL);
 280                         AddKeyword ("object", Token.OBJECT);
 281                         AddKeyword ("operator", Token.OPERATOR);
 282                         AddKeyword ("out", Token.OUT);
 283                         AddKeyword ("override", Token.OVERRIDE);
 284                         AddKeyword ("params", Token.PARAMS);
 285                         AddKeyword ("private", Token.PRIVATE);
 286                         AddKeyword ("protected", Token.PROTECTED);
 287                         AddKeyword ("public", Token.PUBLIC);
 288                         AddKeyword ("readonly", Token.READONLY);
 289                         AddKeyword ("ref", Token.REF);
 290                         AddKeyword ("remove", Token.REMOVE);
 291                         AddKeyword ("return", Token.RETURN);
 292                         AddKeyword ("sbyte", Token.SBYTE);
 293                         AddKeyword ("sealed", Token.SEALED);
 294                         AddKeyword ("set", Token.SET);
 295                         AddKeyword ("short", Token.SHORT);
 296                         AddKeyword ("sizeof", Token.SIZEOF);
 297                         AddKeyword ("stackalloc", Token.STACKALLOC);
 298                         AddKeyword ("static", Token.STATIC);
 299                         AddKeyword ("string", Token.STRING);
 300                         AddKeyword ("struct", Token.STRUCT);
 301                         AddKeyword ("switch", Token.SWITCH);
 302                         AddKeyword ("this", Token.THIS);
 303                         AddKeyword ("throw", Token.THROW);
 304                         AddKeyword ("true", Token.TRUE);
 305                         AddKeyword ("try", Token.TRY);
 306                         AddKeyword ("typeof", Token.TYPEOF);
 307                         AddKeyword ("uint", Token.UINT);
 308                         AddKeyword ("ulong", Token.ULONG);
 309                         AddKeyword ("unchecked", Token.UNCHECKED);
 310                         AddKeyword ("unsafe", Token.UNSAFE);
 311                         AddKeyword ("ushort", Token.USHORT);
 312                         AddKeyword ("using", Token.USING);
 313                         AddKeyword ("virtual", Token.VIRTUAL);
 314                         AddKeyword ("void", Token.VOID);
 315                         AddKeyword ("volatile", Token.VOLATILE);
 316                         AddKeyword ("while", Token.WHILE);
 317                         AddKeyword ("partial", Token.PARTIAL);
 318                 }
 319
 320                 //
 321                 // Class initializer
 322                 //
 323                 static Tokenizer ()
 324                 {
 325                         InitTokens ();
 326                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 327                         styles = NumberStyles.Float;
 328
 329                         string_builder = new System.Text.StringBuilder ();
 330                 }
 331
 332                 int GetKeyword (char[] id, int id_len)
 333                 {
 334                         /*
 335                          * Keywords are stored in an array of hashtables grouped by their
 336                          * length.
 337                          */
 338
 339                         if ((id_len >= keywords.Length) || (keywords [id_len] == null))
 340                                 return -1;
 341                         object o = keywords [id_len] [id];
 342
 343                         if (o == null)
 344                                 return -1;
 345
 346                         int res = (int) o;
 347
 348                         if (handle_get_set == false && (res == Token.GET || res == Token.SET))
 349                                 return -1;
 350                         if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
 351                                 return -1;
 352                         if (handle_assembly == false && res == Token.ASSEMBLY)
 353                                 return -1;
 354
 355                         return res;
 356
 357                 }
 358
 359                 public Location Location {
 360                         get {
 361                                 return new Location (ref_line);
 362                         }
 363                 }
 364
 365                 void define (string def)
 366                 {
 367                         if (!RootContext.AllDefines.Contains (def)){
 368                                 RootContext.AllDefines [def] = true;
 369                         }
 370                         if (defines.Contains (def))
 371                                 return;
 372                         defines [def] = true;
 373                 }
 374
 375                 public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)
 376                 {
 377                         this.ref_name = file;
 378                         this.file_name = file;
 379                         reader = input;
 380
 381                         putback_char = -1;
 382
 383                         if (defs != null){
 384                                 defines = new Hashtable ();
 385                                 foreach (string def in defs)
 386                                         define (def);
 387                         }
 388
 389                         xml_comment_buffer = new StringBuilder ();
 390
 391                         //
 392                         // FIXME: This could be `Location.Push' but we have to
 393                         // find out why the MS compiler allows this
 394                         //
 395                         Mono.CSharp.Location.Push (file);
 396                 }
 397
 398                 public static void Cleanup () {
 399                         identifiers = null;
 400                 }
 401
 402                 static bool is_identifier_start_character (char c)
 403                 {
 404                         return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c);
 405                 }
 406
 407                 static bool is_identifier_part_character (char c)
 408                 {
 409                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
 410                 }
 411
 412                 public static bool IsValidIdentifier (string s)
 413                 {
 414                         if (s == null || s.Length == 0)
 415                                 return false;
 416
 417                         if (!is_identifier_start_character (s [0]))
 418                                 return false;
 419
 420                         for (int i = 1; i < s.Length; i ++)
 421                                 if (! is_identifier_part_character (s [i]))
 422                                         return false;
 423
 424                         return true;
 425                 }
 426
 427                 int is_punct (char c, ref bool doread)
 428                 {
 429                         int d;
 430                         int t;
 431
 432                         doread = false;
 433
 434                         switch (c){
 435                         case '{':
 436                                 return Token.OPEN_BRACE;
 437                         case '}':
 438                                 return Token.CLOSE_BRACE;
 439                         case '[':
 440                                 // To block doccomment inside attribute declaration.
 441                                 if (doc_state == XmlCommentState.Allowed)
 442                                         doc_state = XmlCommentState.NotAllowed;
 443                                 return Token.OPEN_BRACKET;
 444                         case ']':
 445                                 return Token.CLOSE_BRACKET;
 446                         case '(':
 447                                 return Token.OPEN_PARENS;
 448                         case ')': {
 449                                 if (deambiguate_close_parens == 0)
 450                                         return Token.CLOSE_PARENS;
 451
 452                                 --deambiguate_close_parens;
 453
 454                                 // Save current position and parse next token.
 455                                 int old = reader.Position;
 456                                 int new_token = token ();
 457                                 reader.Position = old;
 458                                 putback_char = -1;
 459
 460                                 if (new_token == Token.OPEN_PARENS)
 461                                         return Token.CLOSE_PARENS_OPEN_PARENS;
 462                                 else if (new_token == Token.MINUS)
 463                                         return Token.CLOSE_PARENS_MINUS;
 464                                 else if (IsCastToken (new_token))
 465                                         return Token.CLOSE_PARENS_CAST;
 466                                 else
 467                                         return Token.CLOSE_PARENS_NO_CAST;
 468                         }
 469
 470                         case ',':
 471                                 return Token.COMMA;
 472                         case ':':
 473                                 return Token.COLON;
 474                         case ';':
 475                                 return Token.SEMICOLON;
 476                         case '~':
 477                                 return Token.TILDE;
 478                         case '?':
 479                                 return Token.INTERR;
 480                         }
 481
 482                         d = peekChar ();
 483                         if (c == '+'){
 484
 485                                 if (d == '+')
 486                                         t = Token.OP_INC;
 487                                 else if (d == '=')
 488                                         t = Token.OP_ADD_ASSIGN;
 489                                 else
 490                                         return Token.PLUS;
 491                                 doread = true;
 492                                 return t;
 493                         }
 494                         if (c == '-'){
 495                                 if (d == '-')
 496                                         t = Token.OP_DEC;
 497                                 else if (d == '=')
 498                                         t = Token.OP_SUB_ASSIGN;
 499                                 else if (d == '>')
 500                                         t = Token.OP_PTR;
 501                                 else
 502                                         return Token.MINUS;
 503                                 doread = true;
 504                                 return t;
 505                         }
 506
 507                         if (c == '!'){
 508                                 if (d == '='){
 509                                         doread = true;
 510                                         return Token.OP_NE;
 511                                 }
 512                                 return Token.BANG;
 513                         }
 514
 515                         if (c == '='){
 516                                 if (d == '='){
 517                                         doread = true;
 518                                         return Token.OP_EQ;
 519                                 }
 520                                 return Token.ASSIGN;
 521                         }
 522
 523                         if (c == '&'){
 524                                 if (d == '&'){
 525                                         doread = true;
 526                                         return Token.OP_AND;
 527                                 } else if (d == '='){
 528                                         doread = true;
 529                                         return Token.OP_AND_ASSIGN;
 530                                 }
 531                                 return Token.BITWISE_AND;
 532                         }
 533
 534                         if (c == '|'){
 535                                 if (d == '|'){
 536                                         doread = true;
 537                                         return Token.OP_OR;
 538                                 } else if (d == '='){
 539                                         doread = true;
 540                                         return Token.OP_OR_ASSIGN;
 541                                 }
 542                                 return Token.BITWISE_OR;
 543                         }
 544
 545                         if (c == '*'){
 546                                 if (d == '='){
 547                                         doread = true;
 548                                         return Token.OP_MULT_ASSIGN;
 549                                 }
 550                                 return Token.STAR;
 551                         }
 552
 553                         if (c == '/'){
 554                                 if (d == '='){
 555                                         doread = true;
 556                                         return Token.OP_DIV_ASSIGN;
 557                                 }
 558                                 return Token.DIV;
 559                         }
 560
 561                         if (c == '%'){
 562                                 if (d == '='){
 563                                         doread = true;
 564                                         return Token.OP_MOD_ASSIGN;
 565                                 }
 566                                 return Token.PERCENT;
 567                         }
 568
 569                         if (c == '^'){
 570                                 if (d == '='){
 571                                         doread = true;
 572                                         return Token.OP_XOR_ASSIGN;
 573                                 }
 574                                 return Token.CARRET;
 575                         }
 576
 577                         if (c == '<'){
 578                                 if (d == '<'){
 579                                         getChar ();
 580                                         d = peekChar ();
 581
 582                                         if (d == '='){
 583                                                 doread = true;
 584                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 585                                         }
 586                                         return Token.OP_SHIFT_LEFT;
 587                                 } else if (d == '='){
 588                                         doread = true;
 589                                         return Token.OP_LE;
 590                                 }
 591                                 return Token.OP_LT;
 592                         }
 593
 594                         if (c == '>'){
 595                                 if (d == '>'){
 596                                         getChar ();
 597                                         d = peekChar ();
 598
 599                                         if (d == '='){
 600                                                 doread = true;
 601                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 602                                         }
 603                                         return Token.OP_SHIFT_RIGHT;
 604                                 } else if (d == '='){
 605                                         doread = true;
 606                                         return Token.OP_GE;
 607                                 }
 608                                 return Token.OP_GT;
 609                         }
 610                         return Token.ERROR;
 611                 }
 612
 613                 int deambiguate_close_parens = 0;
 614
 615                 public void Deambiguate_CloseParens ()
 616                 {
 617                         putback (')');
 618                         deambiguate_close_parens++;
 619                 }
 620
 621                 void Error_NumericConstantTooLong ()
 622                 {
 623                         Report.Error (1021, Location, "Numeric constant too long");
 624                 }
 625
 626                 bool decimal_digits (int c)
 627                 {
 628                         int d;
 629                         bool seen_digits = false;
 630
 631                         if (c != -1){
 632                                 if (number_pos == max_number_size)
 633                                         Error_NumericConstantTooLong ();
 634                                 number_builder [number_pos++] = (char) c;
 635                         }
 636
 637                         //
 638                         // We use peekChar2, because decimal_digits needs to do a
 639                         // 2-character look-ahead (5.ToString for example).
 640                         //
 641                         while ((d = peekChar2 ()) != -1){
 642                                 if (d >= '0' && d <= '9'){
 643                                         if (number_pos == max_number_size)
 644                                                 Error_NumericConstantTooLong ();
 645                                         number_builder [number_pos++] = (char) d;
 646                                         getChar ();
 647                                         seen_digits = true;
 648                                 } else
 649                                         break;
 650                         }
 651
 652                         return seen_digits;
 653                 }
 654
 655                 bool is_hex (int e)
 656                 {
 657                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
 658                 }
 659
 660                 void hex_digits (int c)
 661                 {
 662                         if (c != -1)
 663                                 number_builder [number_pos++] = (char) c;
 664
 665                 }
 666
 667                 int real_type_suffix (int c)
 668                 {
 669                         int t;
 670
 671                         switch (c){
 672                         case 'F': case 'f':
 673                                 t =  Token.LITERAL_FLOAT;
 674                                 break;
 675                         case 'D': case 'd':
 676                                 t = Token.LITERAL_DOUBLE;
 677                                 break;
 678                         case 'M': case 'm':
 679                                  t= Token.LITERAL_DECIMAL;
 680                                 break;
 681                         default:
 682                                 return Token.NONE;
 683                         }
 684                         return t;
 685                 }
 686
 687                 int integer_type_suffix (ulong ul, int c)
 688                 {
 689                         bool is_unsigned = false;
 690                         bool is_long = false;
 691
 692                         if (c != -1){
 693                                 bool scanning = true;
 694                                 do {
 695                                         switch (c){
 696                                         case 'U': case 'u':
 697                                                 if (is_unsigned)
 698                                                         scanning = false;
 699                                                 is_unsigned = true;
 700                                                 getChar ();
 701                                                 break;
 702
 703                                         case 'l':
 704                                                 if (!is_unsigned && (RootContext.WarningLevel >= 4)){
 705                                                         //
 706                                                         // if we have not seen anything in between
 707                                                         // report this error
 708                                                         //
 709                                                         Report.Warning (78, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
 710                                                 }
 711                                                 //
 712                                                 // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
 713                                                 // work around that.
 714                                                 //
 715                                                 //goto case 'L';
 716                                                 if (is_long)
 717                                                         scanning = false;
 718                                                 is_long = true;
 719                                                 getChar ();
 720                                                 break;
 721
 722                                         case 'L':
 723                                                 if (is_long)
 724                                                         scanning = false;
 725                                                 is_long = true;
 726                                                 getChar ();
 727                                                 break;
 728
 729                                         default:
 730                                                 scanning = false;
 731                                                 break;
 732                                         }
 733                                         c = peekChar ();
 734                                 } while (scanning);
 735                         }
 736
 737                         if (is_long && is_unsigned){
 738                                 val = ul;
 739                                 return Token.LITERAL_INTEGER;
 740                         } else if (is_unsigned){
 741                                 // uint if possible, or ulong else.
 742
 743                                 if ((ul & 0xffffffff00000000) == 0)
 744                                         val = (uint) ul;
 745                                 else
 746                                         val = ul;
 747                         } else if (is_long){
 748                                 // long if possible, ulong otherwise
 749                                 if ((ul & 0x8000000000000000) != 0)
 750                                         val = ul;
 751                                 else
 752                                         val = (long) ul;
 753                         } else {
 754                                 // int, uint, long or ulong in that order
 755                                 if ((ul & 0xffffffff00000000) == 0){
 756                                         uint ui = (uint) ul;
 757
 758                                         if ((ui & 0x80000000) != 0)
 759                                                 val = ui;
 760                                         else
 761                                                 val = (int) ui;
 762                                 } else {
 763                                         if ((ul & 0x8000000000000000) != 0)
 764                                                 val = ul;
 765                                         else
 766                                                 val = (long) ul;
 767                                 }
 768                         }
 769                         return Token.LITERAL_INTEGER;
 770                 }
 771
 772                 //
 773                 // given `c' as the next char in the input decide whether
 774                 // we need to convert to a special type, and then choose
 775                 // the best representation for the integer
 776                 //
 777                 int adjust_int (int c)
 778                 {
 779                         try {
 780                                 if (number_pos > 9){
 781                                         ulong ul = (uint) (number_builder [0] - '0');
 782
 783                                         for (int i = 1; i < number_pos; i++){
 784                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
 785                                         }
 786                                         return integer_type_suffix (ul, c);
 787                                 } else {
 788                                         uint ui = (uint) (number_builder [0] - '0');
 789
 790                                         for (int i = 1; i < number_pos; i++){
 791                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
 792                                         }
 793                                         return integer_type_suffix (ui, c);
 794                                 }
 795                         } catch (OverflowException) {
 796                                 error_details = "Integral constant is too large";
 797                                 Report.Error (1021, Location, error_details);
 798                                 val = 0ul;
 799                                 return Token.LITERAL_INTEGER;
 800                         }
 801                 }
 802
 803                 int adjust_real (int t)
 804                 {
 805                         string s = new String (number_builder, 0, number_pos);
 806
 807                         switch (t){
 808                         case Token.LITERAL_DECIMAL:
 809                                 try {
 810                                         val = System.Decimal.Parse (s, styles, csharp_format_info);
 811                                 } catch (OverflowException) {
 812                                         val = 0m;
 813                                         error_details = "Floating-point constant is outside the range of the type 'decimal'";
 814                                         Report.Error (594, Location, error_details);
 815                                 }
 816                                 break;
 817                         case Token.LITERAL_FLOAT:
 818                                 try {
 819                                         val = (float) System.Double.Parse (s, styles, csharp_format_info);
 820                                 } catch (OverflowException) {
 821                                         val = 0.0f;
 822                                         error_details = "Floating-point constant is outside the range of the type 'float'";
 823                                         Report.Error (594, Location, error_details);
 824                                 }
 825                                 break;
 826
 827                         case Token.LITERAL_DOUBLE:
 828                         case Token.NONE:
 829                                 t = Token.LITERAL_DOUBLE;
 830                                 try {
 831                                         val = System.Double.Parse (s, styles, csharp_format_info);
 832                                 } catch (OverflowException) {
 833                                         val = 0.0;
 834                                         error_details = "Floating-point constant is outside the range of the type 'double'";
 835                                         Report.Error (594, Location, error_details);
 836                                 }
 837                                 break;
 838                         }
 839                         return t;
 840                 }
 841
 842                 int handle_hex ()
 843                 {
 844                         int d;
 845                         ulong ul;
 846
 847                         getChar ();
 848                         while ((d = peekChar ()) != -1){
 849                                 if (is_hex (d)){
 850                                         number_builder [number_pos++] = (char) d;
 851                                         getChar ();
 852                                 } else
 853                                         break;
 854                         }
 855
 856                         string s = new String (number_builder, 0, number_pos);
 857                         try {
 858                                 if (number_pos <= 8)
 859                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
 860                                 else
 861                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
 862                         } catch (OverflowException){
 863                                 error_details = "Integral constant is too large";
 864                                 Report.Error (1021, Location, error_details);
 865                                 val = 0ul;
 866                                 return Token.LITERAL_INTEGER;
 867                         }
 868                         catch (FormatException) {
 869                                 Report.Error (1013, Location, "Invalid number");
 870                                 val = 0ul;
 871                                 return Token.LITERAL_INTEGER;
 872                         }
 873
 874                         return integer_type_suffix (ul, peekChar ());
 875                 }
 876
 877                 //
 878                 // Invoked if we know we have .digits or digits
 879                 //
 880                 int is_number (int c)
 881                 {
 882                         bool is_real = false;
 883                         int type;
 884
 885                         number_pos = 0;
 886
 887                         if (c >= '0' && c <= '9'){
 888                                 if (c == '0'){
 889                                         int peek = peekChar ();
 890
 891                                         if (peek == 'x' || peek == 'X')
 892                                                 return handle_hex ();
 893                                 }
 894                                 decimal_digits (c);
 895                                 c = getChar ();
 896                         }
 897
 898                         //
 899                         // We need to handle the case of
 900                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
 901                         //
 902                         if (c == '.'){
 903                                 if (decimal_digits ('.')){
 904                                         is_real = true;
 905                                         c = getChar ();
 906                                 } else {
 907                                         putback ('.');
 908                                         number_pos--;
 909                                         return adjust_int (-1);
 910                                 }
 911                         }
 912
 913                         if (c == 'e' || c == 'E'){
 914                                 is_real = true;
 915                                 if (number_pos == max_number_size)
 916                                         Error_NumericConstantTooLong ();
 917                                 number_builder [number_pos++] = 'e';
 918                                 c = getChar ();
 919
 920                                 if (c == '+'){
 921                                         if (number_pos == max_number_size)
 922                                                 Error_NumericConstantTooLong ();
 923                                         number_builder [number_pos++] = '+';
 924                                         c = -1;
 925                                 } else if (c == '-') {
 926                                         if (number_pos == max_number_size)
 927                                                 Error_NumericConstantTooLong ();
 928                                         number_builder [number_pos++] = '-';
 929                                         c = -1;
 930                                 } else {
 931                                         if (number_pos == max_number_size)
 932                                                 Error_NumericConstantTooLong ();
 933                                         number_builder [number_pos++] = '+';
 934                                 }
 935
 936                                 decimal_digits (c);
 937                                 c = getChar ();
 938                         }
 939
 940                         type = real_type_suffix (c);
 941                         if (type == Token.NONE && !is_real){
 942                                 putback (c);
 943                                 return adjust_int (c);
 944                         } else
 945                                 is_real = true;
 946
 947                         if (type == Token.NONE){
 948                                 putback (c);
 949                         }
 950
 951                         if (is_real)
 952                                 return adjust_real (type);
 953
 954                         Console.WriteLine ("This should not be reached");
 955                         throw new Exception ("Is Number should never reach this point");
 956                 }
 957
 958                 //
 959                 // Accepts exactly count (4 or 8) hex, no more no less
 960                 //
 961                 int getHex (int count, out bool error)
 962                 {
 963                         int i;
 964                         int total = 0;
 965                         int c;
 966                         int top = count != -1 ? count : 4;
 967
 968                         getChar ();
 969                         error = false;
 970                         for (i = 0; i < top; i++){
 971                                 c = getChar ();
 972
 973                                 if (c >= '0' && c <= '9')
 974                                         c = (int) c - (int) '0';
 975                                 else if (c >= 'A' && c <= 'F')
 976                                         c = (int) c - (int) 'A' + 10;
 977                                 else if (c >= 'a' && c <= 'f')
 978                                         c = (int) c - (int) 'a' + 10;
 979                                 else {
 980                                         error = true;
 981                                         return 0;
 982                                 }
 983
 984                                 total = (total * 16) + c;
 985                                 if (count == -1){
 986                                         int p = peekChar ();
 987                                         if (p == -1)
 988                                                 break;
 989                                         if (!is_hex ((char)p))
 990                                                 break;
 991                                 }
 992                         }
 993                         return total;
 994                 }
 995
 996                 int escape (int c)
 997                 {
 998                         bool error;
 999                         int d;
1000                         int v;
1001
1002                         d = peekChar ();
1003                         if (c != '\\')
1004                                 return c;
1005
1006                         switch (d){
1007                         case 'a':
1008                                 v = '\a'; break;
1009                         case 'b':
1010                                 v = '\b'; break;
1011                         case 'n':
1012                                 v = '\n'; break;
1013                         case 't':
1014                                 v = '\t'; break;
1015                         case 'v':
1016                                 v = '\v'; break;
1017                         case 'r':
1018                                 v = '\r'; break;
1019                         case '\\':
1020                                 v = '\\'; break;
1021                         case 'f':
1022                                 v = '\f'; break;
1023                         case '0':
1024                                 v = 0; break;
1025                         case '"':
1026                                 v = '"'; break;
1027                         case '\'':
1028                                 v = '\''; break;
1029                         case 'x':
1030                                 v = getHex (-1, out error);
1031                                 if (error)
1032                                         goto default;
1033                                 return v;
1034                         case 'u':
1035                                 v = getHex (4, out error);
1036                                 if (error)
1037                                         goto default;
1038                                 return v;
1039                         case 'U':
1040                                 v = getHex (8, out error);
1041                                 if (error)
1042                                         goto default;
1043                                 return v;
1044                         default:
1045                                 Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);
1046                                 return d;
1047                         }
1048                         getChar ();
1049                         return v;
1050                 }
1051
1052                 int getChar ()
1053                 {
1054                         if (putback_char != -1){
1055                                 int x = putback_char;
1056                                 putback_char = -1;
1057
1058                                 return x;
1059                         }
1060                         return reader.Read ();
1061                 }
1062
1063                 int peekChar ()
1064                 {
1065                         if (putback_char != -1)
1066                                 return putback_char;
1067                         putback_char = reader.Read ();
1068                         return putback_char;
1069                 }
1070
1071                 int peekChar2 ()
1072                 {
1073                         if (putback_char != -1)
1074                                 return putback_char;
1075                         return reader.Peek ();
1076                 }
1077
1078                 void putback (int c)
1079                 {
1080                         if (putback_char != -1){
1081                                 Console.WriteLine ("Col: " + col);
1082                                 Console.WriteLine ("Row: " + line);
1083                                 Console.WriteLine ("Name: " + ref_name.Name);
1084                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1085                                 throw new Exception ("This should not happen putback on putback");
1086                         }
1087                         putback_char = c;
1088                 }
1089
1090                 public bool advance ()
1091                 {
1092                         return peekChar () != -1;
1093                 }
1094
1095                 public Object Value {
1096                         get {
1097                                 return val;
1098                         }
1099                 }
1100
1101                 public Object value ()
1102                 {
1103                         return val;
1104                 }
1105
1106                 bool IsCastToken (int token)
1107                 {
1108                         switch (token) {
1109                         case Token.BANG:
1110                         case Token.TILDE:
1111                         case Token.IDENTIFIER:
1112                         case Token.LITERAL_INTEGER:
1113                         case Token.LITERAL_FLOAT:
1114                         case Token.LITERAL_DOUBLE:
1115                         case Token.LITERAL_DECIMAL:
1116                         case Token.LITERAL_CHARACTER:
1117                         case Token.LITERAL_STRING:
1118                         case Token.BASE:
1119                         case Token.CHECKED:
1120                         case Token.FALSE:
1121                         case Token.FIXED:
1122                         case Token.NEW:
1123                         case Token.NULL:
1124                         case Token.SIZEOF:
1125                         case Token.THIS:
1126                         case Token.THROW:
1127                         case Token.TRUE:
1128                         case Token.TYPEOF:
1129                         case Token.UNCHECKED:
1130                         case Token.UNSAFE:
1131
1132                                 //
1133                                 // These can be part of a member access
1134                                 //
1135                         case Token.INT:
1136                         case Token.UINT:
1137                         case Token.SHORT:
1138                         case Token.USHORT:
1139                         case Token.LONG:
1140                         case Token.ULONG:
1141                         case Token.DOUBLE:
1142                         case Token.FLOAT:
1143                         case Token.CHAR:
1144                                 return true;
1145
1146                         default:
1147                                 return false;
1148                         }
1149                 }
1150
1151                 public int token ()
1152                 {
1153                         current_token = xtoken ();
1154                         return current_token;
1155                 }
1156
1157                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1158
1159                 void get_cmd_arg (out string cmd, out string arg)
1160                 {
1161                         int c;
1162
1163                         tokens_seen = false;
1164                         arg = "";
1165                         static_cmd_arg.Length = 0;
1166
1167                         // skip over white space
1168                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1169                                 ;
1170
1171
1172                         while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
1173                                 if (is_identifier_part_character ((char) c)){
1174                                         static_cmd_arg.Append ((char) c);
1175                                         c = getChar ();
1176                                 } else {
1177                                         putback (c);
1178                                         break;
1179                                 }
1180                         }
1181
1182                         cmd = static_cmd_arg.ToString ();
1183
1184                         if (c == '\n'){
1185                                 line++;
1186                                 ref_line++;
1187                                 return;
1188                         } else if (c == '\r')
1189                                 col = 0;
1190
1191                         // skip over white space
1192                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1193                                 ;
1194
1195                         if (c == '\n'){
1196                                 line++;
1197                                 ref_line++;
1198                                 return;
1199                         } else if (c == '\r'){
1200                                 col = 0;
1201                                 return;
1202                         }
1203
1204                         static_cmd_arg.Length = 0;
1205                         static_cmd_arg.Append ((char) c);
1206
1207                         while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
1208                                 static_cmd_arg.Append ((char) c);
1209                         }
1210
1211                         if (c == '\n'){
1212                                 line++;
1213                                 ref_line++;
1214                         } else if (c == '\r')
1215                                 col = 0;
1216                         arg = static_cmd_arg.ToString ().Trim ();
1217                 }
1218
1219                 //
1220                 // Handles the #line directive
1221                 //
1222                 bool PreProcessLine (string arg)
1223                 {
1224                         if (arg == "")
1225                                 return false;
1226
1227                         if (arg == "default"){
1228                                 ref_line = line;
1229                                 ref_name = file_name;
1230                                 Location.Push (ref_name);
1231                                 return true;
1232                         } else if (arg == "hidden"){
1233                                 //
1234                                 // We ignore #line hidden
1235                                 //
1236                                 return true;
1237                         }
1238
1239                         try {
1240                                 int pos;
1241
1242                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1243                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1244                                         pos++;
1245
1246                                         char [] quotes = { '\"' };
1247
1248                                         string name = arg.Substring (pos). Trim (quotes);
1249                                         ref_name = Location.LookupFile (name);
1250                                         file_name.HasLineDirective = true;
1251                                         ref_name.HasLineDirective = true;
1252                                         Location.Push (ref_name);
1253                                 } else {
1254                                         ref_line = System.Int32.Parse (arg);
1255                                 }
1256                         } catch {
1257                                 return false;
1258                         }
1259
1260                         return true;
1261                 }
1262
1263                 //
1264                 // Handles #define and #undef
1265                 //
1266                 void PreProcessDefinition (bool is_define, string arg)
1267                 {
1268                         if (arg == "" || arg == "true" || arg == "false"){
1269                                 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
1270                                 return;
1271                         }
1272
1273                         char[] whitespace = { ' ', '\t' };
1274                         if (arg.IndexOfAny (whitespace) != -1){
1275                                 Report.Error (1025, Location, "Single-line comment or end-of-line expected");
1276                                 return;
1277                         }
1278
1279                         if (!is_identifier_start_character (arg [0]))
1280                                 Report.Error (1001, Location, "Identifier expected: " + arg);
1281
1282                         foreach (char c in arg.Substring (1)){
1283                                 if (!is_identifier_part_character (c)){
1284                                         Report.Error (1001, Location, "Identifier expected: " + arg);
1285                                         return;
1286                                 }
1287                         }
1288
1289                         if (is_define){
1290                                 if (defines == null)
1291                                         defines = new Hashtable ();
1292                                 define (arg);
1293                         } else {
1294                                 if (defines == null)
1295                                         return;
1296                                 if (defines.Contains (arg))
1297                                         defines.Remove (arg);
1298                         }
1299                 }
1300
1301                 /// <summary>
1302                 /// Handles #pragma directive
1303                 /// </summary>
1304                 void PreProcessPragma (string arg)
1305                 {
1306                         const string disable = "warning disable";
1307                         const string restore = "warning restore";
1308
1309                         if (arg == disable) {
1310                                 Report.RegisterWarningRegion (Location).WarningDisable (line);
1311                                 return;
1312                         }
1313
1314                         if (arg == restore) {
1315                                 Report.RegisterWarningRegion (Location).WarningEnable (line);
1316                                 return;
1317                         }
1318
1319                         if (arg.StartsWith (disable)) {
1320                                 int[] codes = ParseNumbers (arg.Substring (disable.Length));
1321                                 foreach (int code in codes) {
1322                                         if (code != 0)
1323                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1324                                 }
1325                                 return;
1326                         }
1327
1328                         if (arg.StartsWith (restore)) {
1329                                 int[] codes = ParseNumbers (arg.Substring (restore.Length));
1330                                 foreach (int code in codes) {
1331                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1332                                 }
1333                                 return;
1334                         }
1335
1336                         return;
1337                 }
1338
1339                 int[] ParseNumbers (string text)
1340                 {
1341                         string[] string_array = text.Split (',');
1342                         int[] values = new int [string_array.Length];
1343                         int index = 0;
1344                         foreach (string string_code in string_array) {
1345                                 try {
1346                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1347                                 }
1348                                 catch (FormatException) {
1349                                         Report.Warning (1692, Location, "Invalid number");
1350                                 }
1351                         }
1352                         return values;
1353                 }
1354
1355                 bool eval_val (string s)
1356                 {
1357                         if (s == "true")
1358                                 return true;
1359                         if (s == "false")
1360                                 return false;
1361
1362                         if (defines == null)
1363                                 return false;
1364                         if (defines.Contains (s))
1365                                 return true;
1366
1367                         return false;
1368                 }
1369
1370                 bool pp_primary (ref string s)
1371                 {
1372                         s = s.Trim ();
1373                         int len = s.Length;
1374
1375                         if (len > 0){
1376                                 char c = s [0];
1377
1378                                 if (c == '('){
1379                                         s = s.Substring (1);
1380                                         bool val = pp_expr (ref s);
1381                                         if (s.Length > 0 && s [0] == ')'){
1382                                                 s = s.Substring (1);
1383                                                 return val;
1384                                         }
1385                                         Error_InvalidDirective ();
1386                                         return false;
1387                                 }
1388
1389                                 if (is_identifier_start_character (c)){
1390                                         int j = 1;
1391
1392                                         while (j < len){
1393                                                 c = s [j];
1394
1395                                                 if (is_identifier_part_character (c)){
1396                                                         j++;
1397                                                         continue;
1398                                                 }
1399                                                 bool v = eval_val (s.Substring (0, j));
1400                                                 s = s.Substring (j);
1401                                                 return v;
1402                                         }
1403                                         bool vv = eval_val (s);
1404                                         s = "";
1405                                         return vv;
1406                                 }
1407                         }
1408                         Error_InvalidDirective ();
1409                         return false;
1410                 }
1411
1412                 bool pp_unary (ref string s)
1413                 {
1414                         s = s.Trim ();
1415                         int len = s.Length;
1416
1417                         if (len > 0){
1418                                 if (s [0] == '!'){
1419                                         if (len > 1 && s [1] == '='){
1420                                                 Error_InvalidDirective ();
1421                                                 return false;
1422                                         }
1423                                         s = s.Substring (1);
1424                                         return ! pp_primary (ref s);
1425                                 } else
1426                                         return pp_primary (ref s);
1427                         } else {
1428                                 Error_InvalidDirective ();
1429                                 return false;
1430                         }
1431                 }
1432
1433                 bool pp_eq (ref string s)
1434                 {
1435                         bool va = pp_unary (ref s);
1436
1437                         s = s.Trim ();
1438                         int len = s.Length;
1439                         if (len > 0){
1440                                 if (s [0] == '='){
1441                                         if (len > 2 && s [1] == '='){
1442                                                 s = s.Substring (2);
1443                                                 return va == pp_unary (ref s);
1444                                         } else {
1445                                                 Error_InvalidDirective ();
1446                                                 return false;
1447                                         }
1448                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
1449                                         s = s.Substring (2);
1450
1451                                         return va != pp_unary (ref s);
1452
1453                                 }
1454                         }
1455
1456                         return va;
1457
1458                 }
1459
1460                 bool pp_and (ref string s)
1461                 {
1462                         bool va = pp_eq (ref s);
1463
1464                         s = s.Trim ();
1465                         int len = s.Length;
1466                         if (len > 0){
1467                                 if (s [0] == '&'){
1468                                         if (len > 2 && s [1] == '&'){
1469                                                 s = s.Substring (2);
1470                                                 return (va & pp_eq (ref s));
1471                                         } else {
1472                                                 Error_InvalidDirective ();
1473                                                 return false;
1474                                         }
1475                                 }
1476                         }
1477                         return va;
1478                 }
1479
1480                 //
1481                 // Evaluates an expression for `#if' or `#elif'
1482                 //
1483                 bool pp_expr (ref string s)
1484                 {
1485                         bool va = pp_and (ref s);
1486                         s = s.Trim ();
1487                         int len = s.Length;
1488                         if (len > 0){
1489                                 char c = s [0];
1490
1491                                 if (c == '|'){
1492                                         if (len > 2 && s [1] == '|'){
1493                                                 s = s.Substring (2);
1494                                                 return va | pp_expr (ref s);
1495                                         } else {
1496                                                 Error_InvalidDirective ();
1497                                                 return false;
1498                                         }
1499                                 }
1500                         }
1501
1502                         return va;
1503                 }
1504
1505                 bool eval (string s)
1506                 {
1507                         bool v = pp_expr (ref s);
1508                         s = s.Trim ();
1509                         if (s.Length != 0){
1510                                 Error_InvalidDirective ();
1511                                 return false;
1512                         }
1513
1514                         return v;
1515                 }
1516
1517                 void Error_InvalidDirective ()
1518                 {
1519                         Report.Error (1517, Location, "Invalid pre-processor directive");
1520                 }
1521
1522                 void Error_UnexpectedDirective (string extra)
1523                 {
1524                         Report.Error (
1525                                 1028, Location,
1526                                 "Unexpected processor directive (" + extra + ")");
1527                 }
1528
1529                 void Error_TokensSeen ()
1530                 {
1531                         Report.Error (
1532                                 1032, Location,
1533                                 "Cannot define or undefine pre-processor symbols after a token in the file");
1534                 }
1535
1536                 //
1537                 // if true, then the code continues processing the code
1538                 // if false, the code stays in a loop until another directive is
1539                 // reached.
1540                 //
1541                 bool handle_preprocessing_directive (bool caller_is_taking)
1542                 {
1543                         string cmd, arg;
1544                         bool region_directive = false;
1545
1546                         get_cmd_arg (out cmd, out arg);
1547
1548                         // Eat any trailing whitespaces and single-line comments
1549                         if (arg.IndexOf ("//") != -1)
1550                                 arg = arg.Substring (0, arg.IndexOf ("//"));
1551                         arg = arg.TrimEnd (' ', '\t');
1552
1553                         //
1554                         // The first group of pre-processing instructions is always processed
1555                         //
1556                         switch (cmd){
1557                         case "pragma":
1558                                 if (RootContext.Version == LanguageVersion.ISO_1) {
1559                                         Report.FeatureIsNotStandardized (Location, "#pragma");
1560                                         return caller_is_taking;
1561                                 }
1562
1563                                 PreProcessPragma (arg);
1564                                 return caller_is_taking;
1565
1566                         case "line":
1567                                 if (!PreProcessLine (arg))
1568                                         Report.Error (
1569                                                 1576, Location,
1570                                                 "Argument to #line directive is missing or invalid");
1571                                 return caller_is_taking;
1572
1573                         case "region":
1574                                 region_directive = true;
1575                                 arg = "true";
1576                                 goto case "if";
1577
1578                         case "endregion":
1579                                 region_directive = true;
1580                                 goto case "endif";
1581
1582                         case "if":
1583                                 if (arg == ""){
1584                                         Error_InvalidDirective ();
1585                                         return true;
1586                                 }
1587                                 bool taking = false;
1588                                 if (ifstack == null)
1589                                         ifstack = new Stack ();
1590
1591                                 if (ifstack.Count == 0){
1592                                         taking = true;
1593                                 } else {
1594                                         int state = (int) ifstack.Peek ();
1595                                         if ((state & TAKING) != 0)
1596                                                 taking = true;
1597                                 }
1598
1599                                 if (eval (arg) && taking){
1600                                         int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;
1601                                         if (region_directive)
1602                                                 push |= REGION;
1603                                         ifstack.Push (push);
1604                                         return true;
1605                                 } else {
1606                                         int push = (taking ? PARENT_TAKING : 0);
1607                                         if (region_directive)
1608                                                 push |= REGION;
1609                                         ifstack.Push (push);
1610                                         return false;
1611                                 }
1612
1613                         case "endif":
1614                                 if (ifstack == null || ifstack.Count == 0){
1615                                         Error_UnexpectedDirective ("no #if for this #endif");
1616                                         return true;
1617                                 } else {
1618                                         int pop = (int) ifstack.Pop ();
1619
1620                                         if (region_directive && ((pop & REGION) == 0))
1621                                                 Report.Error (1027, Location, "#endif directive expected");
1622                                         else if (!region_directive && ((pop & REGION) != 0))
1623                                                 Report.Error (1038, Location, "#endregion directive expected");
1624
1625                                         if (ifstack.Count == 0)
1626                                                 return true;
1627                                         else {
1628                                                 int state = (int) ifstack.Peek ();
1629
1630                                                 if ((state & TAKING) != 0)
1631                                                         return true;
1632                                                 else
1633                                                         return false;
1634                                         }
1635                                 }
1636
1637                         case "elif":
1638                                 if (ifstack == null || ifstack.Count == 0){
1639                                         Error_UnexpectedDirective ("no #if for this #elif");
1640                                         return true;
1641                                 } else {
1642                                         int state = (int) ifstack.Peek ();
1643
1644                                         if ((state & REGION) != 0) {
1645                                                 Report.Error (1038, Location, "#endregion directive expected");
1646                                                 return true;
1647                                         }
1648
1649                                         if ((state & ELSE_SEEN) != 0){
1650                                                 Error_UnexpectedDirective ("#elif not valid after #else");
1651                                                 return true;
1652                                         }
1653
1654                                         if ((state & (TAKEN_BEFORE | TAKING)) != 0)
1655                                                 return false;
1656
1657                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
1658                                                 state = (int) ifstack.Pop ();
1659                                                 ifstack.Push (state | TAKING | TAKEN_BEFORE);
1660                                                 return true;
1661                                         } else
1662                                                 return false;
1663                                 }
1664
1665                         case "else":
1666                                 if (ifstack == null || ifstack.Count == 0){
1667                                         Report.Error (
1668                                                 1028, Location,
1669                                                 "Unexpected processor directive (no #if for this #else)");
1670                                         return true;
1671                                 } else {
1672                                         int state = (int) ifstack.Peek ();
1673
1674                                         if ((state & REGION) != 0) {
1675                                                 Report.Error (1038, Location, "#endregion directive expected");
1676                                                 return true;
1677                                         }
1678
1679                                         if ((state & ELSE_SEEN) != 0){
1680                                                 Error_UnexpectedDirective ("#else within #else");
1681                                                 return true;
1682                                         }
1683
1684                                         ifstack.Pop ();
1685
1686                                         bool ret;
1687                                         if ((state & TAKEN_BEFORE) == 0){
1688                                                 ret = ((state & PARENT_TAKING) != 0);
1689                                         } else
1690                                                 ret = false;
1691
1692                                         if (ret)
1693                                                 state |= TAKING;
1694                                         else
1695                                                 state &= ~TAKING;
1696
1697                                         ifstack.Push (state | ELSE_SEEN);
1698
1699                                         return ret;
1700                                 }
1701                         }
1702
1703                         //
1704                         // These are only processed if we are in a `taking' block
1705                         //
1706                         if (!caller_is_taking)
1707                                 return false;
1708
1709                         switch (cmd){
1710                         case "define":
1711                                 if (any_token_seen){
1712                                         Error_TokensSeen ();
1713                                         return true;
1714                                 }
1715                                 PreProcessDefinition (true, arg);
1716                                 return true;
1717
1718                         case "undef":
1719                                 if (any_token_seen){
1720                                         Error_TokensSeen ();
1721                                         return true;
1722                                 }
1723                                 PreProcessDefinition (false, arg);
1724                                 return true;
1725
1726                         case "error":
1727                                 Report.Error (1029, Location, "#error: '" + arg + "'");
1728                                 return true;
1729
1730                         case "warning":
1731                                 Report.Warning (1030, Location, "#warning: '{0}'", arg);
1732                                 return true;
1733                         }
1734
1735                         Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");
1736                         return true;
1737
1738                 }
1739
1740                 private int consume_string (bool quoted)
1741                 {
1742                         int c;
1743                         string_builder.Length = 0;
1744
1745                         while ((c = getChar ()) != -1){
1746                                 if (c == '"'){
1747                                         if (quoted && peekChar () == '"'){
1748                                                 string_builder.Append ((char) c);
1749                                                 getChar ();
1750                                                 continue;
1751                                         } else {
1752                                                 val = string_builder.ToString ();
1753                                                 return Token.LITERAL_STRING;
1754                                         }
1755                                 }
1756
1757                                 if (c == '\n'){
1758                                         if (!quoted)
1759                                                 Report.Error (1010, Location, "Newline in constant");
1760                                         line++;
1761                                         ref_line++;
1762                                         col = 0;
1763                                 } else
1764                                         col++;
1765
1766                                 if (!quoted){
1767                                         c = escape (c);
1768                                         if (c == -1)
1769                                                 return Token.ERROR;
1770                                 }
1771                                 string_builder.Append ((char) c);
1772                         }
1773
1774                         Report.Error (1039, Location, "Unterminated string literal");
1775                         return Token.EOF;
1776                 }
1777
1778                 private int consume_identifier (int s)
1779                 {
1780                         int res = consume_identifier (s, false);
1781
1782                         if (doc_state == XmlCommentState.Allowed)
1783                                 doc_state = XmlCommentState.NotAllowed;
1784                         switch (res) {
1785                         case Token.USING:
1786                         case Token.NAMESPACE:
1787                                 check_incorrect_doc_comment ();
1788                                 break;
1789                         }
1790
1791                         if (res == Token.PARTIAL) {
1792                                 // Save current position and parse next token.
1793                                 int old = reader.Position;
1794                                 int old_putback = putback_char;
1795
1796                                 putback_char = -1;
1797
1798                                 int next_token = token ();
1799                                 bool ok = (next_token == Token.CLASS) ||
1800                                         (next_token == Token.STRUCT) ||
1801                                         (next_token == Token.INTERFACE);
1802
1803                                 reader.Position = old;
1804                                 putback_char = old_putback;
1805
1806                                 if (ok)
1807                                         return res;
1808                                 else {
1809                                         val = "partial";
1810                                         return Token.IDENTIFIER;
1811                                 }
1812                         }
1813
1814                         return res;
1815                 }
1816
1817                 private int consume_identifier (int s, bool quoted)
1818                 {
1819                         int pos = 1;
1820                         int c;
1821
1822                         id_builder [0] = (char) s;
1823
1824                         while ((c = reader.Read ()) != -1) {
1825                                 if (is_identifier_part_character ((char) c)){
1826                                         if (pos == max_id_size){
1827                                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
1828                                                 return Token.ERROR;
1829                                         }
1830
1831                                         id_builder [pos++] = (char) c;
1832                                         putback_char = -1;
1833                                         col++;
1834                                 } else {
1835                                         putback_char = c;
1836                                         break;
1837                                 }
1838                         }
1839
1840                         //
1841                         // Optimization: avoids doing the keyword lookup
1842                         // on uppercase letters and _
1843                         //
1844                         if (!quoted && (s >= 'a' || s == '_')){
1845                                 int keyword = GetKeyword (id_builder, pos);
1846                                 if (keyword != -1)
1847                                         return keyword;
1848                         }
1849
1850                         //
1851                         // Keep identifiers in an array of hashtables to avoid needless
1852                         // allocations
1853                         //
1854
1855                         if (identifiers [pos] != null) {
1856                                 val = identifiers [pos][id_builder];
1857                                 if (val != null) {
1858                                         return Token.IDENTIFIER;
1859                                 }
1860                         }
1861                         else
1862                                 identifiers [pos] = new CharArrayHashtable (pos);
1863
1864                         val = new String (id_builder, 0, pos);
1865
1866                         char [] chars = new char [pos];
1867                         Array.Copy (id_builder, chars, pos);
1868
1869                         identifiers [pos] [chars] = val;
1870
1871                         return Token.IDENTIFIER;
1872                 }
1873
1874                 public int xtoken ()
1875                 {
1876                         int t;
1877                         bool doread = false;
1878                         int c;
1879
1880                         val = null;
1881                         // optimization: eliminate col and implement #directive semantic correctly.
1882                         for (;(c = getChar ()) != -1; col++) {
1883                                 if (c == ' ')
1884                                         continue;
1885
1886                                 if (c == '\t') {
1887                                         col = (((col + 8) / 8) * 8) - 1;
1888                                         continue;
1889                                 }
1890
1891                                 if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0)
1892                                         continue;
1893
1894                                 if (c == '\r') {
1895                                         if (peekChar () == '\n')
1896                                                 getChar ();
1897
1898                                         line++;
1899                                         ref_line++;
1900                                         col = 0;
1901                                         any_token_seen |= tokens_seen;
1902                                         tokens_seen = false;
1903                                         continue;
1904                                 }
1905
1906                                 // Handle double-slash comments.
1907                                 if (c == '/'){
1908                                         int d = peekChar ();
1909
1910                                         if (d == '/'){
1911                                                 getChar ();
1912                                                 if (RootContext.Documentation != null && peekChar () == '/') {
1913                                                         getChar ();
1914                                                         // Don't allow ////.
1915                                                         if ((d = peekChar ()) != '/') {
1916                                                                 if (doc_state == XmlCommentState.Allowed)
1917                                                                         handle_one_line_xml_comment ();
1918                                                                 else if (doc_state == XmlCommentState.NotAllowed)
1919                                                                         warn_incorrect_doc_comment ();
1920                                                         }
1921                                                 }
1922                                                 while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
1923                                                         col++;
1924                                                 if (d == '\n'){
1925                                                         line++;
1926                                                         ref_line++;
1927                                                         col = 0;
1928                                                 }
1929                                                 any_token_seen |= tokens_seen;
1930                                                 tokens_seen = false;
1931                                                 continue;
1932                                         } else if (d == '*'){
1933                                                 getChar ();
1934                                                 bool docAppend = false;
1935                                                 if (RootContext.Documentation != null && peekChar () == '*') {
1936                                                         getChar ();
1937                                                         // But when it is /**/, just do nothing.
1938                                                         if (peekChar () == '/') {
1939                                                                 getChar ();
1940                                                                 continue;
1941                                                         }
1942                                                         if (doc_state == XmlCommentState.Allowed)
1943                                                                 docAppend = true;
1944                                                         else if (doc_state == XmlCommentState.NotAllowed)
1945                                                                 warn_incorrect_doc_comment ();
1946                                                 }
1947
1948                                                 int currentCommentStart = 0;
1949                                                 if (docAppend) {
1950                                                         currentCommentStart = xml_comment_buffer.Length;
1951                                                         xml_comment_buffer.Append (Environment.NewLine);
1952                                                 }
1953
1954                                                 while ((d = getChar ()) != -1){
1955                                                         if (d == '*' && peekChar () == '/'){
1956                                                                 getChar ();
1957                                                                 col++;
1958                                                                 break;
1959                                                         }
1960                                                         if (docAppend)
1961                                                                 xml_comment_buffer.Append ((char) d);
1962
1963                                                         if (d == '\n'){
1964                                                                 line++;
1965                                                                 ref_line++;
1966                                                                 col = 0;
1967                                                                 any_token_seen |= tokens_seen;
1968                                                                 tokens_seen = false;
1969                                                         }
1970                                                 }
1971                                                 if (docAppend)
1972                                                         update_formatted_doc_comment (currentCommentStart);
1973                                                 continue;
1974                                         }
1975                                         goto is_punct_label;
1976                                 }
1977
1978
1979                                 if (is_identifier_start_character ((char)c)){
1980                                         tokens_seen = true;
1981                                         return consume_identifier (c);
1982                                 }
1983
1984                         is_punct_label:
1985                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
1986                                         tokens_seen = true;
1987                                         if (doread){
1988                                                 getChar ();
1989                                                 col++;
1990                                         }
1991                                         return t;
1992                                 }
1993
1994                                 // white space
1995                                 if (c == '\n'){
1996                                         line++;
1997                                         ref_line++;
1998                                         col = 0;
1999                                         any_token_seen |= tokens_seen;
2000                                         tokens_seen = false;
2001                                         continue;
2002                                 }
2003
2004                                 if (c >= '0' && c <= '9'){
2005                                         tokens_seen = true;
2006                                         return is_number (c);
2007                                 }
2008
2009                                 if (c == '.'){
2010                                         tokens_seen = true;
2011                                         int peek = peekChar ();
2012                                         if (peek >= '0' && peek <= '9')
2013                                                 return is_number (c);
2014                                         return Token.DOT;
2015                                 }
2016
2017                                 /* For now, ignore pre-processor commands */
2018                                 // FIXME: In C# the '#' is not limited to appear
2019                                 // on the first column.
2020                                 if (c == '#' && !tokens_seen){
2021                                         bool cont = true;
2022
2023                                 start_again:
2024
2025                                         cont = handle_preprocessing_directive (cont);
2026
2027                                         if (cont){
2028                                                 col = 0;
2029                                                 continue;
2030                                         }
2031                                         col = 1;
2032
2033                                         bool skipping = false;
2034                                         for (;(c = getChar ()) != -1; col++){
2035                                                 if (c == '\n'){
2036                                                         col = 0;
2037                                                         line++;
2038                                                         ref_line++;
2039                                                         skipping = false;
2040                                                 } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
2041                                                         continue;
2042                                                 else if (c != '#')
2043                                                         skipping = true;
2044                                                 if (c == '#' && !skipping)
2045                                                         goto start_again;
2046                                         }
2047                                         any_token_seen |= tokens_seen;
2048                                         tokens_seen = false;
2049                                         if (c == -1)
2050                                                 Report.Error (1027, Location, "#endif/#endregion expected");
2051                                         continue;
2052                                 }
2053
2054                                 if (c == '"')
2055                                         return consume_string (false);
2056
2057                                 if (c == '\''){
2058                                         c = getChar ();
2059                                         tokens_seen = true;
2060                                         if (c == '\''){
2061                                                 error_details = "Empty character literal";
2062                                                 Report.Error (1011, Location, error_details);
2063                                                 return Token.ERROR;
2064                                         }
2065                                         c = escape (c);
2066                                         if (c == -1)
2067                                                 return Token.ERROR;
2068                                         val = new System.Char ();
2069                                         val = (char) c;
2070                                         c = getChar ();
2071
2072                                         if (c != '\''){
2073                                                 error_details = "Too many characters in character literal";
2074                                                 Report.Error (1012, Location, error_details);
2075
2076                                                 // Try to recover, read until newline or next "'"
2077                                                 while ((c = getChar ()) != -1){
2078                                                         if (c == '\n' || c == '\''){
2079                                                                 line++;
2080                                                                 ref_line++;
2081                                                                 col = 0;
2082                                                                 break;
2083                                                         } else
2084                                                                 col++;
2085
2086                                                 }
2087                                                 return Token.ERROR;
2088                                         }
2089                                         return Token.LITERAL_CHARACTER;
2090                                 }
2091
2092                                 if (c == '@') {
2093                                         c = getChar ();
2094                                         if (c == '"') {
2095                                                 tokens_seen = true;
2096                                                 return consume_string (true);
2097                                         } else if (is_identifier_start_character ((char) c)){
2098                                                 return consume_identifier (c, true);
2099                                         } else {
2100                                                 Report.Error (1033, Location, "'@' must be followed by string constant or identifier");
2101                                         }
2102                                 }
2103
2104                                 if (c == '#') {
2105                                         error_details = "Preprocessor directives must appear as the first non-whitespace " +
2106                                                 "character on a line.";
2107
2108                                         Report.Error (1040, Location, error_details);
2109
2110                                         return Token.ERROR;
2111                                 }
2112
2113                                 error_details = ((char)c).ToString ();
2114
2115                                 return Token.ERROR;
2116                         }
2117
2118                         return Token.EOF;
2119                 }
2120
2121                 //
2122                 // Handles one line xml comment
2123                 //
2124                 private void handle_one_line_xml_comment ()
2125                 {
2126                         int c;
2127                         while ((c = peekChar ()) == ' ')
2128                                 getChar (); // skip heading whitespaces.
2129                         while ((c = peekChar ()) != -1 && c != '\n' && c != '\r') {
2130                                 col++;
2131                                 xml_comment_buffer.Append ((char) getChar ());
2132                         }
2133                         if (c == '\r' || c == '\n')
2134                                 xml_comment_buffer.Append (Environment.NewLine);
2135                 }
2136
2137                 //
2138                 // Remove heading "*" in Javadoc-like xml documentation.
2139                 //
2140                 private void update_formatted_doc_comment (int currentCommentStart)
2141                 {
2142                         int length = xml_comment_buffer.Length - currentCommentStart;
2143                         string [] lines = xml_comment_buffer.ToString (
2144                                 currentCommentStart,
2145                                 length).Replace ("\r", "").Split ('\n');
2146                         // The first line starts with /**, thus it is not target
2147                         // for the format check.
2148                         for (int i = 1; i < lines.Length; i++) {
2149                                 string s = lines [i];
2150                                 int idx = s.IndexOf ('*');
2151                                 string head = null;
2152                                 if (idx < 0) {
2153                                         if (i < lines.Length - 1)
2154                                                 return;
2155                                         head = s;
2156                                 }
2157                                 else
2158                                         head = s.Substring (0, idx);
2159                                 foreach (char c in head)
2160                                         if (c != ' ')
2161                                                 return;
2162                                 lines [i] = s.Substring (idx + 1);
2163                         }
2164                         xml_comment_buffer.Remove (currentCommentStart, length);
2165                         xml_comment_buffer.Insert (
2166                                 currentCommentStart,
2167                                 String.Join (Environment.NewLine, lines));
2168                 }
2169
2170                 //
2171                 // Checks if there was incorrect doc comments and raise
2172                 // warnings.
2173                 //
2174                 public void check_incorrect_doc_comment ()
2175                 {
2176                         if (xml_comment_buffer.Length > 0)
2177                                 warn_incorrect_doc_comment ();
2178                 }
2179
2180                 //
2181                 // Raises a warning when tokenizer found incorrect doccomment
2182                 // markup.
2183                 //
2184                 private void warn_incorrect_doc_comment ()
2185                 {
2186                         doc_state = XmlCommentState.Error;
2187                         // in csc, it is 'XML comment is not placed on a valid
2188                         // language element'. But that does not make sense.
2189                         Report.Warning (1587, 2, Location, "XML comment is placed on an invalid language element which can not accept it.");
2190                 }
2191
2192                 //
2193                 // Consumes the saved xml comment lines (if any)
2194                 // as for current target member or type.
2195                 //
2196                 public string consume_doc_comment ()
2197                 {
2198                         if (xml_comment_buffer.Length > 0) {
2199                                 string ret = xml_comment_buffer.ToString ();
2200                                 xml_comment_buffer.Length = 0;
2201                                 return ret;
2202                         }
2203                         return null;
2204                 }
2205
2206                 public void cleanup ()
2207                 {
2208                         if (ifstack != null && ifstack.Count >= 1) {
2209                                 int state = (int) ifstack.Pop ();
2210                                 if ((state & REGION) != 0)
2211                                         Report.Error (1038, Location, "#endregion directive expected");
2212                                 else
2213                                         Report.Error (1027, "#endif directive expected");
2214                         }
2215
2216                 }
2217         }
2218
2219         //
2220         // Indicates whether it accepts XML documentation or not.
2221         //
2222         public enum XmlCommentState {
2223                 // comment is allowed in this state.
2224                 Allowed,
2225                 // comment is not allowed in this state.
2226                 NotAllowed,
2227                 // once comments appeared when it is NotAllowed, then the
2228                 // state is changed to it, until the state is changed to
2229                 // .Allowed.
2230                 Error
2231         }
2232 }