mcs/gmcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //
   7 // Licensed under the terms of the GNU GPL
   8 //
   9 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
  10 // (C) 2004 Novell, Inc
  11 //
  12
  13 /*
  14  * TODO:
  15  *   Make sure we accept the proper Unicode ranges, per the spec.
  16  *   Report error 1032
  17 */
  18
  19 using System;
  20 using System.Text;
  21 using System.Collections;
  22 using System.IO;
  23 using System.Globalization;
  24 using System.Reflection;
  25
  26 namespace Mono.CSharp
  27 {
  28         /// <summary>
  29         ///    Tokenizer for C# source code.
  30         /// </summary>
  31
  32         public class Tokenizer : yyParser.yyInput
  33         {
  34                 SeekableStreamReader reader;
  35                 public SourceFile ref_name;
  36                 public SourceFile file_name;
  37                 public int ref_line = 1;
  38                 public int line = 1;
  39                 public int col = 1;
  40                 public int current_token;
  41                 bool handle_get_set = false;
  42                 bool handle_remove_add = false;
  43                 bool handle_assembly = false;
  44                 bool handle_constraints = false;
  45                 bool handle_typeof = false;
  46
  47                 //
  48                 // XML documentation buffer. The save point is used to divide
  49                 // comments on types and comments on members.
  50                 //
  51                 StringBuilder xml_comment_buffer;
  52
  53                 //
  54                 // See comment on XmlCommentState enumeration.
  55                 //
  56                 XmlCommentState xmlDocState = XmlCommentState.Allowed;
  57
  58                 //
  59                 // Whether tokens have been seen on this line
  60                 //
  61                 bool tokens_seen = false;
  62
  63                 //
  64                 // Whether a token has been seen on the file
  65                 // This is needed because `define' is not allowed to be used
  66                 // after a token has been seen.
  67                 //
  68                 bool any_token_seen = false;
  69
  70                 static Hashtable tokenValues;
  71
  72                 private static Hashtable TokenValueName
  73                 {
  74                         get {
  75                                 if (tokenValues == null)
  76                                         tokenValues = GetTokenValueNameHash ();
  77
  78                                 return tokenValues;
  79                         }
  80                 }
  81
  82                 private static Hashtable GetTokenValueNameHash ()
  83                 {
  84                         Type t = typeof (Token);
  85                         FieldInfo [] fields = t.GetFields ();
  86                         Hashtable hash = new Hashtable ();
  87                         foreach (FieldInfo field in fields) {
  88                                 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))
  89                                         hash.Add (field.GetValue (null), field.Name);
  90                         }
  91                         return hash;
  92                 }
  93
  94                 //
  95                 // Returns a verbose representation of the current location
  96                 //
  97                 public string location {
  98                         get {
  99                                 string det;
 100
 101                                 if (current_token == Token.ERROR)
 102                                         det = "detail: " + error_details;
 103                                 else
 104                                         det = "";
 105
 106                                 // return "Line:     "+line+" Col: "+col + "\n" +
 107                                 //       "VirtLine: "+ref_line +
 108                                 //       " Token: "+current_token + " " + det;
 109                                 string current_token_name = TokenValueName [current_token] as string;
 110                                 if (current_token_name == null)
 111                                         current_token_name = current_token.ToString ();
 112
 113                                 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,
 114                                                                                        ref_line,
 115                                                                                        col,
 116                                                                                        current_token_name,
 117                                                                                        det);
 118                         }
 119                 }
 120
 121                 public bool PropertyParsing {
 122                         get {
 123                                 return handle_get_set;
 124                         }
 125
 126                         set {
 127                                 handle_get_set = value;
 128                         }
 129                 }
 130
 131                 public bool AssemblyTargetParsing {
 132                         get {
 133                                 return handle_assembly;
 134                         }
 135
 136                         set {
 137                                 handle_assembly = value;
 138                         }
 139                 }
 140
 141                 public bool EventParsing {
 142                         get {
 143                                 return handle_remove_add;
 144                         }
 145
 146                         set {
 147                                 handle_remove_add = value;
 148                         }
 149                 }
 150
 151                 public bool ConstraintsParsing {
 152                         get {
 153                                 return handle_constraints;
 154                         }
 155
 156                         set {
 157                                 handle_constraints = value;
 158                         }
 159                 }
 160
 161                 public bool TypeOfParsing {
 162                         get {
 163                                 return handle_typeof;
 164                         }
 165
 166                         set {
 167                                 handle_typeof = value;
 168                         }
 169                 }
 170
 171                 public XmlCommentState doc_state {
 172                         get { return xmlDocState; }
 173                         set {
 174                                 if (value == XmlCommentState.Allowed) {
 175                                         check_incorrect_doc_comment ();
 176                                         consume_doc_comment ();
 177                                 }
 178                                 xmlDocState = value;
 179                         }
 180                 }
 181
 182                 //
 183                 // Class variables
 184                 //
 185                 static CharArrayHashtable[] keywords;
 186                 static NumberStyles styles;
 187                 static NumberFormatInfo csharp_format_info;
 188
 189                 //
 190                 // Values for the associated token returned
 191                 //
 192                 int putback_char;
 193                 Object val;
 194
 195                 //
 196                 // Pre-processor
 197                 //
 198                 Hashtable defines;
 199
 200                 const int TAKING        = 1;
 201                 const int TAKEN_BEFORE  = 2;
 202                 const int ELSE_SEEN     = 4;
 203                 const int PARENT_TAKING = 8;
 204                 const int REGION        = 16;
 205
 206                 //
 207                 // pre-processor if stack state:
 208                 //
 209                 Stack ifstack;
 210
 211                 static System.Text.StringBuilder string_builder;
 212
 213                 const int max_id_size = 512;
 214                 static char [] id_builder = new char [max_id_size];
 215
 216                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 217
 218                 const int max_number_size = 128;
 219                 static char [] number_builder = new char [max_number_size];
 220                 static int number_pos;
 221
 222                 //
 223                 // Details about the error encoutered by the tokenizer
 224                 //
 225                 string error_details;
 226
 227                 public string error {
 228                         get {
 229                                 return error_details;
 230                         }
 231                 }
 232
 233                 public int Line {
 234                         get {
 235                                 return ref_line;
 236                         }
 237                 }
 238
 239                 public int Col {
 240                         get {
 241                                 return col;
 242                         }
 243                 }
 244
 245                 static void AddKeyword (string kw, int token) {
 246                         if (keywords [kw.Length] == null) {
 247                                 keywords [kw.Length] = new CharArrayHashtable (kw.Length);
 248                         }
 249                         keywords [kw.Length] [kw.ToCharArray ()] = token;
 250                 }
 251
 252                 static void InitTokens ()
 253                 {
 254                         keywords = new CharArrayHashtable [64];
 255
 256                         AddKeyword ("__arglist", Token.ARGLIST);
 257                         AddKeyword ("abstract", Token.ABSTRACT);
 258                         AddKeyword ("as", Token.AS);
 259                         AddKeyword ("add", Token.ADD);
 260                         AddKeyword ("assembly", Token.ASSEMBLY);
 261                         AddKeyword ("base", Token.BASE);
 262                         AddKeyword ("bool", Token.BOOL);
 263                         AddKeyword ("break", Token.BREAK);
 264                         AddKeyword ("byte", Token.BYTE);
 265                         AddKeyword ("case", Token.CASE);
 266                         AddKeyword ("catch", Token.CATCH);
 267                         AddKeyword ("char", Token.CHAR);
 268                         AddKeyword ("checked", Token.CHECKED);
 269                         AddKeyword ("class", Token.CLASS);
 270                         AddKeyword ("const", Token.CONST);
 271                         AddKeyword ("continue", Token.CONTINUE);
 272                         AddKeyword ("decimal", Token.DECIMAL);
 273                         AddKeyword ("default", Token.DEFAULT);
 274                         AddKeyword ("delegate", Token.DELEGATE);
 275                         AddKeyword ("do", Token.DO);
 276                         AddKeyword ("double", Token.DOUBLE);
 277                         AddKeyword ("else", Token.ELSE);
 278                         AddKeyword ("enum", Token.ENUM);
 279                         AddKeyword ("event", Token.EVENT);
 280                         AddKeyword ("explicit", Token.EXPLICIT);
 281                         AddKeyword ("extern", Token.EXTERN);
 282                         AddKeyword ("false", Token.FALSE);
 283                         AddKeyword ("finally", Token.FINALLY);
 284                         AddKeyword ("fixed", Token.FIXED);
 285                         AddKeyword ("float", Token.FLOAT);
 286                         AddKeyword ("for", Token.FOR);
 287                         AddKeyword ("foreach", Token.FOREACH);
 288                         AddKeyword ("goto", Token.GOTO);
 289                         AddKeyword ("get", Token.GET);
 290                         AddKeyword ("if", Token.IF);
 291                         AddKeyword ("implicit", Token.IMPLICIT);
 292                         AddKeyword ("in", Token.IN);
 293                         AddKeyword ("int", Token.INT);
 294                         AddKeyword ("interface", Token.INTERFACE);
 295                         AddKeyword ("internal", Token.INTERNAL);
 296                         AddKeyword ("is", Token.IS);
 297                         AddKeyword ("lock", Token.LOCK);
 298                         AddKeyword ("long", Token.LONG);
 299                         AddKeyword ("namespace", Token.NAMESPACE);
 300                         AddKeyword ("new", Token.NEW);
 301                         AddKeyword ("null", Token.NULL);
 302                         AddKeyword ("object", Token.OBJECT);
 303                         AddKeyword ("operator", Token.OPERATOR);
 304                         AddKeyword ("out", Token.OUT);
 305                         AddKeyword ("override", Token.OVERRIDE);
 306                         AddKeyword ("params", Token.PARAMS);
 307                         AddKeyword ("private", Token.PRIVATE);
 308                         AddKeyword ("protected", Token.PROTECTED);
 309                         AddKeyword ("public", Token.PUBLIC);
 310                         AddKeyword ("readonly", Token.READONLY);
 311                         AddKeyword ("ref", Token.REF);
 312                         AddKeyword ("remove", Token.REMOVE);
 313                         AddKeyword ("return", Token.RETURN);
 314                         AddKeyword ("sbyte", Token.SBYTE);
 315                         AddKeyword ("sealed", Token.SEALED);
 316                         AddKeyword ("set", Token.SET);
 317                         AddKeyword ("short", Token.SHORT);
 318                         AddKeyword ("sizeof", Token.SIZEOF);
 319                         AddKeyword ("stackalloc", Token.STACKALLOC);
 320                         AddKeyword ("static", Token.STATIC);
 321                         AddKeyword ("string", Token.STRING);
 322                         AddKeyword ("struct", Token.STRUCT);
 323                         AddKeyword ("switch", Token.SWITCH);
 324                         AddKeyword ("this", Token.THIS);
 325                         AddKeyword ("throw", Token.THROW);
 326                         AddKeyword ("true", Token.TRUE);
 327                         AddKeyword ("try", Token.TRY);
 328                         AddKeyword ("typeof", Token.TYPEOF);
 329                         AddKeyword ("uint", Token.UINT);
 330                         AddKeyword ("ulong", Token.ULONG);
 331                         AddKeyword ("unchecked", Token.UNCHECKED);
 332                         AddKeyword ("unsafe", Token.UNSAFE);
 333                         AddKeyword ("ushort", Token.USHORT);
 334                         AddKeyword ("using", Token.USING);
 335                         AddKeyword ("virtual", Token.VIRTUAL);
 336                         AddKeyword ("void", Token.VOID);
 337                         AddKeyword ("volatile", Token.VOLATILE);
 338                         AddKeyword ("where", Token.WHERE);
 339                         AddKeyword ("while", Token.WHILE);
 340                         AddKeyword ("partial", Token.PARTIAL);
 341                 }
 342
 343                 //
 344                 // Class initializer
 345                 //
 346                 static Tokenizer ()
 347                 {
 348                         InitTokens ();
 349                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 350                         styles = NumberStyles.Float;
 351
 352                         string_builder = new System.Text.StringBuilder ();
 353                 }
 354
 355                 int GetKeyword (char[] id, int id_len)
 356                 {
 357                         /*
 358                          * Keywords are stored in an array of hashtables grouped by their
 359                          * length.
 360                          */
 361
 362                         if ((id_len >= keywords.Length) || (keywords [id_len] == null))
 363                                 return -1;
 364                         object o = keywords [id_len] [id];
 365
 366                         if (o == null)
 367                                 return -1;
 368
 369                         int res = (int) o;
 370
 371                         if (handle_get_set == false && (res == Token.GET || res == Token.SET))
 372                                 return -1;
 373                         if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
 374                                 return -1;
 375                         if (handle_assembly == false && res == Token.ASSEMBLY)
 376                                 return -1;
 377                         if (handle_constraints == false && res == Token.WHERE)
 378                                 return -1;
 379
 380                         return res;
 381
 382                 }
 383
 384                 public Location Location {
 385                         get {
 386                                 return new Location (ref_line);
 387                         }
 388                 }
 389
 390                 void define (string def)
 391                 {
 392                         if (!RootContext.AllDefines.Contains (def)){
 393                                 RootContext.AllDefines [def] = true;
 394                         }
 395                         if (defines.Contains (def))
 396                                 return;
 397                         defines [def] = true;
 398                 }
 399
 400                 public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)
 401                 {
 402                         this.ref_name = file;
 403                         this.file_name = file;
 404                         reader = input;
 405
 406                         putback_char = -1;
 407
 408                         if (defs != null){
 409                                 defines = new Hashtable ();
 410                                 foreach (string def in defs)
 411                                         define (def);
 412                         }
 413
 414                         xml_comment_buffer = new StringBuilder ();
 415
 416                         //
 417                         // FIXME: This could be `Location.Push' but we have to
 418                         // find out why the MS compiler allows this
 419                         //
 420                         Mono.CSharp.Location.Push (file);
 421                 }
 422
 423                 public static void Cleanup () {
 424                         identifiers = null;
 425                 }
 426
 427                 static bool is_identifier_start_character (char c)
 428                 {
 429                         return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c);
 430                 }
 431
 432                 static bool is_identifier_part_character (char c)
 433                 {
 434                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
 435                 }
 436
 437                 public static bool IsValidIdentifier (string s)
 438                 {
 439                         if (s == null || s.Length == 0)
 440                                 return false;
 441
 442                         if (!is_identifier_start_character (s [0]))
 443                                 return false;
 444
 445                         for (int i = 1; i < s.Length; i ++)
 446                                 if (! is_identifier_part_character (s [i]))
 447                                         return false;
 448
 449                         return true;
 450                 }
 451
 452                 bool parse_generic_dimension (out int dimension)
 453                 {
 454                         dimension = 1;
 455
 456                 again:
 457                         int the_token = token ();
 458                         if (the_token == Token.OP_GENERICS_GT)
 459                                 return true;
 460                         else if (the_token == Token.COMMA) {
 461                                 dimension++;
 462                                 goto again;
 463                         }
 464
 465                         return false;
 466                 }
 467
 468                 bool parse_less_than ()
 469                 {
 470                 start:
 471                         int the_token = token ();
 472                         switch (the_token) {
 473                         case Token.IDENTIFIER:
 474                         case Token.OBJECT:
 475                         case Token.STRING:
 476                         case Token.BOOL:
 477                         case Token.DECIMAL:
 478                         case Token.FLOAT:
 479                         case Token.DOUBLE:
 480                         case Token.SBYTE:
 481                         case Token.BYTE:
 482                         case Token.SHORT:
 483                         case Token.USHORT:
 484                         case Token.INT:
 485                         case Token.UINT:
 486                         case Token.LONG:
 487                         case Token.ULONG:
 488                         case Token.CHAR:
 489                         case Token.VOID:
 490                                 break;
 491
 492                         default:
 493                                 return false;
 494                         }
 495                 again:
 496                         the_token = token ();
 497
 498                         if (the_token == Token.OP_GENERICS_GT)
 499                                 return true;
 500                         else if ((the_token == Token.COMMA) || (the_token == Token.DOT))
 501                                 goto start;
 502                         else if (the_token == Token.OP_GENERICS_LT) {
 503                                 if (!parse_less_than ())
 504                                         return false;
 505                                 goto again;
 506                         } else if (the_token == Token.OPEN_BRACKET) {
 507                         rank_specifiers:
 508                                 the_token = token ();
 509                                 if (the_token == Token.CLOSE_BRACKET)
 510                                         goto again;
 511                                 else if (the_token == Token.COMMA)
 512                                         goto rank_specifiers;
 513                                 return false;
 514                         }
 515
 516                         return false;
 517                 }
 518
 519                 bool parsing_less_than = false;
 520                 int parsing_generic_less_than = 0;
 521
 522                 int is_punct (char c, ref bool doread)
 523                 {
 524                         int d;
 525                         int t;
 526
 527                         doread = false;
 528
 529                         switch (c){
 530                         case '{':
 531                                 return Token.OPEN_BRACE;
 532                         case '}':
 533                                 return Token.CLOSE_BRACE;
 534                         case '[':
 535                                 // To block doccomment inside attribute declaration.
 536                                 if (doc_state == XmlCommentState.Allowed)
 537                                         doc_state = XmlCommentState.NotAllowed;
 538                                 return Token.OPEN_BRACKET;
 539                         case ']':
 540                                 return Token.CLOSE_BRACKET;
 541                         case '(':
 542                                 return Token.OPEN_PARENS;
 543                         case ')': {
 544                                 if (deambiguate_close_parens == 0)
 545                                         return Token.CLOSE_PARENS;
 546
 547                                 --deambiguate_close_parens;
 548
 549                                 // Save current position and parse next token.
 550                                 int old = reader.Position;
 551                                 int new_token = token ();
 552                                 reader.Position = old;
 553                                 putback_char = -1;
 554
 555                                 if (new_token == Token.OPEN_PARENS)
 556                                         return Token.CLOSE_PARENS_OPEN_PARENS;
 557                                 else if (new_token == Token.MINUS)
 558                                         return Token.CLOSE_PARENS_MINUS;
 559                                 else if (IsCastToken (new_token))
 560                                         return Token.CLOSE_PARENS_CAST;
 561                                 else
 562                                         return Token.CLOSE_PARENS_NO_CAST;
 563                         }
 564
 565                         case ',':
 566                                 return Token.COMMA;
 567                         case ':':
 568                                 return Token.COLON;
 569                         case ';':
 570                                 return Token.SEMICOLON;
 571                         case '~':
 572                                 return Token.TILDE;
 573                         case '?':
 574                                 return Token.INTERR;
 575                         }
 576
 577                         if (c == '<') {
 578                                 if (parsing_generic_less_than++ > 0)
 579                                         return Token.OP_GENERICS_LT;
 580
 581                                 int old = reader.Position;
 582                                 if (handle_typeof) {
 583                                         int dimension;
 584                                         if (parse_generic_dimension (out dimension)) {
 585                                                 val = dimension;
 586                                                 return Token.GENERIC_DIMENSION;
 587                                         }
 588                                         reader.Position = old;
 589                                         putback_char = -1;
 590                                 }
 591
 592                                 // Save current position and parse next token.
 593                                 old = reader.Position;
 594                                 bool is_generic_lt = parse_less_than ();
 595                                 reader.Position = old;
 596                                 putback_char = -1;
 597
 598                                 if (is_generic_lt) {
 599                                         parsing_generic_less_than++;
 600                                         return Token.OP_GENERICS_LT;
 601                                 } else
 602                                         parsing_generic_less_than = 0;
 603
 604                                 d = peekChar ();
 605                                 if (d == '<'){
 606                                         getChar ();
 607                                         d = peekChar ();
 608
 609                                         if (d == '='){
 610                                                 doread = true;
 611                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 612                                         }
 613                                         return Token.OP_SHIFT_LEFT;
 614                                 } else if (d == '='){
 615                                         doread = true;
 616                                         return Token.OP_LE;
 617                                 }
 618                                 return Token.OP_LT;
 619                         } else if (c == '>') {
 620                                 if (parsing_generic_less_than > 0) {
 621                                         parsing_generic_less_than--;
 622                                         return Token.OP_GENERICS_GT;
 623                                 }
 624
 625                                 d = peekChar ();
 626                                 if (d == '>'){
 627                                         getChar ();
 628                                         d = peekChar ();
 629
 630                                         if (d == '='){
 631                                                 doread = true;
 632                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 633                                         }
 634                                         return Token.OP_SHIFT_RIGHT;
 635                                 } else if (d == '='){
 636                                         doread = true;
 637                                         return Token.OP_GE;
 638                                 }
 639                                 return Token.OP_GT;
 640                         }
 641
 642                         d = peekChar ();
 643                         if (c == '+'){
 644
 645                                 if (d == '+')
 646                                         t = Token.OP_INC;
 647                                 else if (d == '=')
 648                                         t = Token.OP_ADD_ASSIGN;
 649                                 else
 650                                         return Token.PLUS;
 651                                 doread = true;
 652                                 return t;
 653                         }
 654                         if (c == '-'){
 655                                 if (d == '-')
 656                                         t = Token.OP_DEC;
 657                                 else if (d == '=')
 658                                         t = Token.OP_SUB_ASSIGN;
 659                                 else if (d == '>')
 660                                         t = Token.OP_PTR;
 661                                 else
 662                                         return Token.MINUS;
 663                                 doread = true;
 664                                 return t;
 665                         }
 666
 667                         if (c == '!'){
 668                                 if (d == '='){
 669                                         doread = true;
 670                                         return Token.OP_NE;
 671                                 }
 672                                 return Token.BANG;
 673                         }
 674
 675                         if (c == '='){
 676                                 if (d == '='){
 677                                         doread = true;
 678                                         return Token.OP_EQ;
 679                                 }
 680                                 return Token.ASSIGN;
 681                         }
 682
 683                         if (c == '&'){
 684                                 if (d == '&'){
 685                                         doread = true;
 686                                         return Token.OP_AND;
 687                                 } else if (d == '='){
 688                                         doread = true;
 689                                         return Token.OP_AND_ASSIGN;
 690                                 }
 691                                 return Token.BITWISE_AND;
 692                         }
 693
 694                         if (c == '|'){
 695                                 if (d == '|'){
 696                                         doread = true;
 697                                         return Token.OP_OR;
 698                                 } else if (d == '='){
 699                                         doread = true;
 700                                         return Token.OP_OR_ASSIGN;
 701                                 }
 702                                 return Token.BITWISE_OR;
 703                         }
 704
 705                         if (c == '*'){
 706                                 if (d == '='){
 707                                         doread = true;
 708                                         return Token.OP_MULT_ASSIGN;
 709                                 }
 710                                 return Token.STAR;
 711                         }
 712
 713                         if (c == '/'){
 714                                 if (d == '='){
 715                                         doread = true;
 716                                         return Token.OP_DIV_ASSIGN;
 717                                 }
 718                                 return Token.DIV;
 719                         }
 720
 721                         if (c == '%'){
 722                                 if (d == '='){
 723                                         doread = true;
 724                                         return Token.OP_MOD_ASSIGN;
 725                                 }
 726                                 return Token.PERCENT;
 727                         }
 728
 729                         if (c == '^'){
 730                                 if (d == '='){
 731                                         doread = true;
 732                                         return Token.OP_XOR_ASSIGN;
 733                                 }
 734                                 return Token.CARRET;
 735                         }
 736
 737                         return Token.ERROR;
 738                 }
 739
 740                 int deambiguate_close_parens = 0;
 741
 742                 public void Deambiguate_CloseParens ()
 743                 {
 744                         putback (')');
 745                         deambiguate_close_parens++;
 746                 }
 747
 748                 void Error_NumericConstantTooLong ()
 749                 {
 750                         Report.Error (1021, Location, "Numeric constant too long");
 751                 }
 752
 753                 bool decimal_digits (int c)
 754                 {
 755                         int d;
 756                         bool seen_digits = false;
 757
 758                         if (c != -1){
 759                                 if (number_pos == max_number_size)
 760                                         Error_NumericConstantTooLong ();
 761                                 number_builder [number_pos++] = (char) c;
 762                         }
 763
 764                         //
 765                         // We use peekChar2, because decimal_digits needs to do a
 766                         // 2-character look-ahead (5.ToString for example).
 767                         //
 768                         while ((d = peekChar2 ()) != -1){
 769                                 if (d >= '0' && d <= '9'){
 770                                         if (number_pos == max_number_size)
 771                                                 Error_NumericConstantTooLong ();
 772                                         number_builder [number_pos++] = (char) d;
 773                                         getChar ();
 774                                         seen_digits = true;
 775                                 } else
 776                                         break;
 777                         }
 778
 779                         return seen_digits;
 780                 }
 781
 782                 bool is_hex (int e)
 783                 {
 784                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
 785                 }
 786
 787                 void hex_digits (int c)
 788                 {
 789                         if (c != -1)
 790                                 number_builder [number_pos++] = (char) c;
 791
 792                 }
 793
 794                 int real_type_suffix (int c)
 795                 {
 796                         int t;
 797
 798                         switch (c){
 799                         case 'F': case 'f':
 800                                 t =  Token.LITERAL_FLOAT;
 801                                 break;
 802                         case 'D': case 'd':
 803                                 t = Token.LITERAL_DOUBLE;
 804                                 break;
 805                         case 'M': case 'm':
 806                                  t= Token.LITERAL_DECIMAL;
 807                                 break;
 808                         default:
 809                                 return Token.NONE;
 810                         }
 811                         return t;
 812                 }
 813
 814                 int integer_type_suffix (ulong ul, int c)
 815                 {
 816                         bool is_unsigned = false;
 817                         bool is_long = false;
 818
 819                         if (c != -1){
 820                                 bool scanning = true;
 821                                 do {
 822                                         switch (c){
 823                                         case 'U': case 'u':
 824                                                 if (is_unsigned)
 825                                                         scanning = false;
 826                                                 is_unsigned = true;
 827                                                 getChar ();
 828                                                 break;
 829
 830                                         case 'l':
 831                                                 if (!is_unsigned && (RootContext.WarningLevel >= 4)){
 832                                                         //
 833                                                         // if we have not seen anything in between
 834                                                         // report this error
 835                                                         //
 836                                                         Report.Warning (78, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
 837                                                 }
 838                                                 //
 839                                                 // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
 840                                                 // work around that.
 841                                                 //
 842                                                 //goto case 'L';
 843                                                 if (is_long)
 844                                                         scanning = false;
 845                                                 is_long = true;
 846                                                 getChar ();
 847                                                 break;
 848
 849                                         case 'L':
 850                                                 if (is_long)
 851                                                         scanning = false;
 852                                                 is_long = true;
 853                                                 getChar ();
 854                                                 break;
 855
 856                                         default:
 857                                                 scanning = false;
 858                                                 break;
 859                                         }
 860                                         c = peekChar ();
 861                                 } while (scanning);
 862                         }
 863
 864                         if (is_long && is_unsigned){
 865                                 val = ul;
 866                                 return Token.LITERAL_INTEGER;
 867                         } else if (is_unsigned){
 868                                 // uint if possible, or ulong else.
 869
 870                                 if ((ul & 0xffffffff00000000) == 0)
 871                                         val = (uint) ul;
 872                                 else
 873                                         val = ul;
 874                         } else if (is_long){
 875                                 // long if possible, ulong otherwise
 876                                 if ((ul & 0x8000000000000000) != 0)
 877                                         val = ul;
 878                                 else
 879                                         val = (long) ul;
 880                         } else {
 881                                 // int, uint, long or ulong in that order
 882                                 if ((ul & 0xffffffff00000000) == 0){
 883                                         uint ui = (uint) ul;
 884
 885                                         if ((ui & 0x80000000) != 0)
 886                                                 val = ui;
 887                                         else
 888                                                 val = (int) ui;
 889                                 } else {
 890                                         if ((ul & 0x8000000000000000) != 0)
 891                                                 val = ul;
 892                                         else
 893                                                 val = (long) ul;
 894                                 }
 895                         }
 896                         return Token.LITERAL_INTEGER;
 897                 }
 898
 899                 //
 900                 // given `c' as the next char in the input decide whether
 901                 // we need to convert to a special type, and then choose
 902                 // the best representation for the integer
 903                 //
 904                 int adjust_int (int c)
 905                 {
 906                         try {
 907                                 if (number_pos > 9){
 908                                         ulong ul = (uint) (number_builder [0] - '0');
 909
 910                                         for (int i = 1; i < number_pos; i++){
 911                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
 912                                         }
 913                                         return integer_type_suffix (ul, c);
 914                                 } else {
 915                                         uint ui = (uint) (number_builder [0] - '0');
 916
 917                                         for (int i = 1; i < number_pos; i++){
 918                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
 919                                         }
 920                                         return integer_type_suffix (ui, c);
 921                                 }
 922                         } catch (OverflowException) {
 923                                 error_details = "Integral constant is too large";
 924                                 Report.Error (1021, Location, error_details);
 925                                 val = 0ul;
 926                                 return Token.LITERAL_INTEGER;
 927                         }
 928                         catch (FormatException) {
 929                                 Report.Error (1013, Location, "Invalid number");
 930                                 val = 0ul;
 931                                 return Token.LITERAL_INTEGER;
 932                         }
 933                 }
 934
 935                 int adjust_real (int t)
 936                 {
 937                         string s = new String (number_builder, 0, number_pos);
 938
 939                         switch (t){
 940                         case Token.LITERAL_DECIMAL:
 941                                 try {
 942                                         val = System.Decimal.Parse (s, styles, csharp_format_info);
 943                                 } catch (OverflowException) {
 944                                         val = 0m;
 945                                         error_details = "Floating-point constant is outside the range of the type 'decimal'";
 946                                         Report.Error (594, Location, error_details);
 947                                 }
 948                                 break;
 949                         case Token.LITERAL_FLOAT:
 950                                 try {
 951                                         val = (float) System.Double.Parse (s, styles, csharp_format_info);
 952                                 } catch (OverflowException) {
 953                                         val = 0.0f;
 954                                         error_details = "Floating-point constant is outside the range of the type 'float'";
 955                                         Report.Error (594, Location, error_details);
 956                                 }
 957                                 break;
 958
 959                         case Token.LITERAL_DOUBLE:
 960                         case Token.NONE:
 961                                 t = Token.LITERAL_DOUBLE;
 962                                 try {
 963                                         val = System.Double.Parse (s, styles, csharp_format_info);
 964                                 } catch (OverflowException) {
 965                                         val = 0.0;
 966                                         error_details = "Floating-point constant is outside the range of the type 'double'";
 967                                         Report.Error (594, Location, error_details);
 968                                 }
 969                                 break;
 970                         }
 971                         return t;
 972                 }
 973
 974                 int handle_hex ()
 975                 {
 976                         int d;
 977                         ulong ul;
 978
 979                         getChar ();
 980                         while ((d = peekChar ()) != -1){
 981                                 if (is_hex (d)){
 982                                         number_builder [number_pos++] = (char) d;
 983                                         getChar ();
 984                                 } else
 985                                         break;
 986                         }
 987
 988                         string s = new String (number_builder, 0, number_pos);
 989                         try {
 990                                 if (number_pos <= 8)
 991                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
 992                                 else
 993                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
 994                         } catch (OverflowException){
 995                                 error_details = "Integral constant is too large";
 996                                 Report.Error (1021, Location, error_details);
 997                                 val = 0ul;
 998                                 return Token.LITERAL_INTEGER;
 999                         }
1000
1001                         return integer_type_suffix (ul, peekChar ());
1002                 }
1003
1004                 //
1005                 // Invoked if we know we have .digits or digits
1006                 //
1007                 int is_number (int c)
1008                 {
1009                         bool is_real = false;
1010                         int type;
1011
1012                         number_pos = 0;
1013
1014                         if (c >= '0' && c <= '9'){
1015                                 if (c == '0'){
1016                                         int peek = peekChar ();
1017
1018                                         if (peek == 'x' || peek == 'X')
1019                                                 return handle_hex ();
1020                                 }
1021                                 decimal_digits (c);
1022                                 c = getChar ();
1023                         }
1024
1025                         //
1026                         // We need to handle the case of
1027                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1028                         //
1029                         if (c == '.'){
1030                                 if (decimal_digits ('.')){
1031                                         is_real = true;
1032                                         c = getChar ();
1033                                 } else {
1034                                         putback ('.');
1035                                         number_pos--;
1036                                         return adjust_int (-1);
1037                                 }
1038                         }
1039
1040                         if (c == 'e' || c == 'E'){
1041                                 is_real = true;
1042                                 if (number_pos == max_number_size)
1043                                         Error_NumericConstantTooLong ();
1044                                 number_builder [number_pos++] = 'e';
1045                                 c = getChar ();
1046
1047                                 if (c == '+'){
1048                                         if (number_pos == max_number_size)
1049                                                 Error_NumericConstantTooLong ();
1050                                         number_builder [number_pos++] = '+';
1051                                         c = -1;
1052                                 } else if (c == '-') {
1053                                         if (number_pos == max_number_size)
1054                                                 Error_NumericConstantTooLong ();
1055                                         number_builder [number_pos++] = '-';
1056                                         c = -1;
1057                                 } else {
1058                                         if (number_pos == max_number_size)
1059                                                 Error_NumericConstantTooLong ();
1060                                         number_builder [number_pos++] = '+';
1061                                 }
1062
1063                                 decimal_digits (c);
1064                                 c = getChar ();
1065                         }
1066
1067                         type = real_type_suffix (c);
1068                         if (type == Token.NONE && !is_real){
1069                                 putback (c);
1070                                 return adjust_int (c);
1071                         } else
1072                                 is_real = true;
1073
1074                         if (type == Token.NONE){
1075                                 putback (c);
1076                         }
1077
1078                         if (is_real)
1079                                 return adjust_real (type);
1080
1081                         Console.WriteLine ("This should not be reached");
1082                         throw new Exception ("Is Number should never reach this point");
1083                 }
1084
1085                 //
1086                 // Accepts exactly count (4 or 8) hex, no more no less
1087                 //
1088                 int getHex (int count, out bool error)
1089                 {
1090                         int i;
1091                         int total = 0;
1092                         int c;
1093                         int top = count != -1 ? count : 4;
1094
1095                         getChar ();
1096                         error = false;
1097                         for (i = 0; i < top; i++){
1098                                 c = getChar ();
1099
1100                                 if (c >= '0' && c <= '9')
1101                                         c = (int) c - (int) '0';
1102                                 else if (c >= 'A' && c <= 'F')
1103                                         c = (int) c - (int) 'A' + 10;
1104                                 else if (c >= 'a' && c <= 'f')
1105                                         c = (int) c - (int) 'a' + 10;
1106                                 else {
1107                                         error = true;
1108                                         return 0;
1109                                 }
1110
1111                                 total = (total * 16) + c;
1112                                 if (count == -1){
1113                                         int p = peekChar ();
1114                                         if (p == -1)
1115                                                 break;
1116                                         if (!is_hex ((char)p))
1117                                                 break;
1118                                 }
1119                         }
1120                         return total;
1121                 }
1122
1123                 int escape (int c)
1124                 {
1125                         bool error;
1126                         int d;
1127                         int v;
1128
1129                         d = peekChar ();
1130                         if (c != '\\')
1131                                 return c;
1132
1133                         switch (d){
1134                         case 'a':
1135                                 v = '\a'; break;
1136                         case 'b':
1137                                 v = '\b'; break;
1138                         case 'n':
1139                                 v = '\n'; break;
1140                         case 't':
1141                                 v = '\t'; break;
1142                         case 'v':
1143                                 v = '\v'; break;
1144                         case 'r':
1145                                 v = '\r'; break;
1146                         case '\\':
1147                                 v = '\\'; break;
1148                         case 'f':
1149                                 v = '\f'; break;
1150                         case '0':
1151                                 v = 0; break;
1152                         case '"':
1153                                 v = '"'; break;
1154                         case '\'':
1155                                 v = '\''; break;
1156                         case 'x':
1157                                 v = getHex (-1, out error);
1158                                 if (error)
1159                                         goto default;
1160                                 return v;
1161                         case 'u':
1162                                 v = getHex (4, out error);
1163                                 if (error)
1164                                         goto default;
1165                                 return v;
1166                         case 'U':
1167                                 v = getHex (8, out error);
1168                                 if (error)
1169                                         goto default;
1170                                 return v;
1171                         default:
1172                                 Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);
1173                                 return d;
1174                         }
1175                         getChar ();
1176                         return v;
1177                 }
1178
1179                 int getChar ()
1180                 {
1181                         if (putback_char != -1){
1182                                 int x = putback_char;
1183                                 putback_char = -1;
1184
1185                                 return x;
1186                         }
1187                         return reader.Read ();
1188                 }
1189
1190                 int peekChar ()
1191                 {
1192                         if (putback_char != -1)
1193                                 return putback_char;
1194                         putback_char = reader.Read ();
1195                         return putback_char;
1196                 }
1197
1198                 int peekChar2 ()
1199                 {
1200                         if (putback_char != -1)
1201                                 return putback_char;
1202                         return reader.Peek ();
1203                 }
1204
1205                 void putback (int c)
1206                 {
1207                         if (putback_char != -1){
1208                                 Console.WriteLine ("Col: " + col);
1209                                 Console.WriteLine ("Row: " + line);
1210                                 Console.WriteLine ("Name: " + ref_name.Name);
1211                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1212                                 throw new Exception ("This should not happen putback on putback");
1213                         }
1214                         putback_char = c;
1215                 }
1216
1217                 public bool advance ()
1218                 {
1219                         return peekChar () != -1;
1220                 }
1221
1222                 public Object Value {
1223                         get {
1224                                 return val;
1225                         }
1226                 }
1227
1228                 public Object value ()
1229                 {
1230                         return val;
1231                 }
1232
1233                 bool IsCastToken (int token)
1234                 {
1235                         switch (token) {
1236                         case Token.BANG:
1237                         case Token.TILDE:
1238                         case Token.IDENTIFIER:
1239                         case Token.LITERAL_INTEGER:
1240                         case Token.LITERAL_FLOAT:
1241                         case Token.LITERAL_DOUBLE:
1242                         case Token.LITERAL_DECIMAL:
1243                         case Token.LITERAL_CHARACTER:
1244                         case Token.LITERAL_STRING:
1245                         case Token.BASE:
1246                         case Token.CHECKED:
1247                         case Token.FALSE:
1248                         case Token.FIXED:
1249                         case Token.NEW:
1250                         case Token.NULL:
1251                         case Token.SIZEOF:
1252                         case Token.THIS:
1253                         case Token.THROW:
1254                         case Token.TRUE:
1255                         case Token.TYPEOF:
1256                         case Token.UNCHECKED:
1257                         case Token.UNSAFE:
1258
1259                                 //
1260                                 // These can be part of a member access
1261                                 //
1262                         case Token.INT:
1263                         case Token.UINT:
1264                         case Token.SHORT:
1265                         case Token.USHORT:
1266                         case Token.LONG:
1267                         case Token.ULONG:
1268                         case Token.DOUBLE:
1269                         case Token.FLOAT:
1270                         case Token.CHAR:
1271                                 return true;
1272
1273                         default:
1274                                 return false;
1275                         }
1276                 }
1277
1278                 public int token ()
1279                 {
1280                         current_token = xtoken ();
1281
1282                         if (current_token != Token.DEFAULT)
1283                                 return current_token;
1284
1285                         int c = consume_whitespace ();
1286                         if (c == -1)
1287                                 current_token = Token.ERROR;
1288                         else if (c == '(')
1289                                 current_token = Token.DEFAULT_OPEN_PARENS;
1290                         else
1291                                 putback (c);
1292
1293                         return current_token;
1294                 }
1295
1296                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1297
1298                 void get_cmd_arg (out string cmd, out string arg)
1299                 {
1300                         int c;
1301
1302                         tokens_seen = false;
1303                         arg = "";
1304                         static_cmd_arg.Length = 0;
1305
1306                         // skip over white space
1307                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1308                                 ;
1309
1310                         while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
1311                                 if (is_identifier_part_character ((char) c)){
1312                                         static_cmd_arg.Append ((char) c);
1313                                         c = getChar ();
1314                                 } else {
1315                                         putback (c);
1316                                         break;
1317                                 }
1318                         }
1319
1320                         cmd = static_cmd_arg.ToString ();
1321
1322                         if (c == '\n'){
1323                                 line++;
1324                                 ref_line++;
1325                                 return;
1326                         } else if (c == '\r')
1327                                 col = 0;
1328
1329                         // skip over white space
1330                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1331                                 ;
1332
1333                         if (c == '\n'){
1334                                 line++;
1335                                 ref_line++;
1336                                 return;
1337                         } else if (c == '\r'){
1338                                 col = 0;
1339                                 return;
1340                         }
1341
1342                         static_cmd_arg.Length = 0;
1343                         static_cmd_arg.Append ((char) c);
1344
1345                         while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
1346                                 static_cmd_arg.Append ((char) c);
1347                         }
1348
1349                         if (c == '\n'){
1350                                 line++;
1351                                 ref_line++;
1352                         } else if (c == '\r')
1353                                 col = 0;
1354                         arg = static_cmd_arg.ToString ().Trim ();
1355                 }
1356
1357                 //
1358                 // Handles the #line directive
1359                 //
1360                 bool PreProcessLine (string arg)
1361                 {
1362                         if (arg == "")
1363                                 return false;
1364
1365                         if (arg == "default"){
1366                                 ref_line = line;
1367                                 ref_name = file_name;
1368                                 Location.Push (ref_name);
1369                                 return true;
1370                         } else if (arg == "hidden"){
1371                                 //
1372                                 // We ignore #line hidden
1373                                 //
1374                                 return true;
1375                         }
1376
1377                         try {
1378                                 int pos;
1379
1380                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1381                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1382                                         pos++;
1383
1384                                         char [] quotes = { '\"' };
1385
1386                                         string name = arg.Substring (pos). Trim (quotes);
1387                                         ref_name = Location.LookupFile (name);
1388                                         file_name.HasLineDirective = true;
1389                                         ref_name.HasLineDirective = true;
1390                                         Location.Push (ref_name);
1391                                 } else {
1392                                         ref_line = System.Int32.Parse (arg);
1393                                 }
1394                         } catch {
1395                                 return false;
1396                         }
1397
1398                         return true;
1399                 }
1400
1401                 //
1402                 // Handles #define and #undef
1403                 //
1404                 void PreProcessDefinition (bool is_define, string arg)
1405                 {
1406                         if (arg == "" || arg == "true" || arg == "false"){
1407                                 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
1408                                 return;
1409                         }
1410
1411                         char[] whitespace = { ' ', '\t' };
1412                         if (arg.IndexOfAny (whitespace) != -1){
1413                                 Report.Error (1025, Location, "Single-line comment or end-of-line expected");
1414                                 return;
1415                         }
1416
1417                         if (!is_identifier_start_character (arg [0]))
1418                                 Report.Error (1001, Location, "Identifier expected: " + arg);
1419
1420                         foreach (char c in arg.Substring (1)){
1421                                 if (!is_identifier_part_character (c)){
1422                                         Report.Error (1001, Location, "Identifier expected: " + arg);
1423                                         return;
1424                                 }
1425                         }
1426
1427                         if (is_define){
1428                                 if (defines == null)
1429                                         defines = new Hashtable ();
1430                                 define (arg);
1431                         } else {
1432                                 if (defines == null)
1433                                         return;
1434                                 if (defines.Contains (arg))
1435                                         defines.Remove (arg);
1436                         }
1437                 }
1438
1439                 /// <summary>
1440                 /// Handles #pragma directive
1441                 /// </summary>
1442                 void PreProcessPragma (string arg)
1443                 {
1444                         const string disable = "warning disable";
1445                         const string restore = "warning restore";
1446
1447                         if (arg == disable) {
1448                                 Report.RegisterWarningRegion (Location).WarningDisable (line);
1449                                 return;
1450                         }
1451
1452                         if (arg == restore) {
1453                                 Report.RegisterWarningRegion (Location).WarningEnable (line);
1454                                 return;
1455                         }
1456
1457                         if (arg.StartsWith (disable)) {
1458                                 int[] codes = ParseNumbers (arg.Substring (disable.Length));
1459                                 foreach (int code in codes) {
1460                                         if (code != 0)
1461                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1462                                 }
1463                                 return;
1464                         }
1465
1466                         if (arg.StartsWith (restore)) {
1467                                 int[] codes = ParseNumbers (arg.Substring (restore.Length));
1468                                 foreach (int code in codes) {
1469                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1470                                 }
1471                                 return;
1472                         }
1473
1474                         return;
1475                 }
1476
1477                 int[] ParseNumbers (string text)
1478                 {
1479                         string[] string_array = text.Split (',');
1480                         int[] values = new int [string_array.Length];
1481                         int index = 0;
1482                         foreach (string string_code in string_array) {
1483                                 try {
1484                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1485                                 }
1486                                 catch (FormatException) {
1487                                         Report.Warning (1692, Location, "Invalid number");
1488                                 }
1489                         }
1490                         return values;
1491                 }
1492
1493                 bool eval_val (string s)
1494                 {
1495                         if (s == "true")
1496                                 return true;
1497                         if (s == "false")
1498                                 return false;
1499
1500                         if (defines == null)
1501                                 return false;
1502                         if (defines.Contains (s))
1503                                 return true;
1504
1505                         return false;
1506                 }
1507
1508                 bool pp_primary (ref string s)
1509                 {
1510                         s = s.Trim ();
1511                         int len = s.Length;
1512
1513                         if (len > 0){
1514                                 char c = s [0];
1515
1516                                 if (c == '('){
1517                                         s = s.Substring (1);
1518                                         bool val = pp_expr (ref s);
1519                                         if (s.Length > 0 && s [0] == ')'){
1520                                                 s = s.Substring (1);
1521                                                 return val;
1522                                         }
1523                                         Error_InvalidDirective ();
1524                                         return false;
1525                                 }
1526
1527                                 if (is_identifier_start_character (c)){
1528                                         int j = 1;
1529
1530                                         while (j < len){
1531                                                 c = s [j];
1532
1533                                                 if (is_identifier_part_character (c)){
1534                                                         j++;
1535                                                         continue;
1536                                                 }
1537                                                 bool v = eval_val (s.Substring (0, j));
1538                                                 s = s.Substring (j);
1539                                                 return v;
1540                                         }
1541                                         bool vv = eval_val (s);
1542                                         s = "";
1543                                         return vv;
1544                                 }
1545                         }
1546                         Error_InvalidDirective ();
1547                         return false;
1548                 }
1549
1550                 bool pp_unary (ref string s)
1551                 {
1552                         s = s.Trim ();
1553                         int len = s.Length;
1554
1555                         if (len > 0){
1556                                 if (s [0] == '!'){
1557                                         if (len > 1 && s [1] == '='){
1558                                                 Error_InvalidDirective ();
1559                                                 return false;
1560                                         }
1561                                         s = s.Substring (1);
1562                                         return ! pp_primary (ref s);
1563                                 } else
1564                                         return pp_primary (ref s);
1565                         } else {
1566                                 Error_InvalidDirective ();
1567                                 return false;
1568                         }
1569                 }
1570
1571                 bool pp_eq (ref string s)
1572                 {
1573                         bool va = pp_unary (ref s);
1574
1575                         s = s.Trim ();
1576                         int len = s.Length;
1577                         if (len > 0){
1578                                 if (s [0] == '='){
1579                                         if (len > 2 && s [1] == '='){
1580                                                 s = s.Substring (2);
1581                                                 return va == pp_unary (ref s);
1582                                         } else {
1583                                                 Error_InvalidDirective ();
1584                                                 return false;
1585                                         }
1586                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
1587                                         s = s.Substring (2);
1588
1589                                         return va != pp_unary (ref s);
1590
1591                                 }
1592                         }
1593
1594                         return va;
1595
1596                 }
1597
1598                 bool pp_and (ref string s)
1599                 {
1600                         bool va = pp_eq (ref s);
1601
1602                         s = s.Trim ();
1603                         int len = s.Length;
1604                         if (len > 0){
1605                                 if (s [0] == '&'){
1606                                         if (len > 2 && s [1] == '&'){
1607                                                 s = s.Substring (2);
1608                                                 return (va & pp_eq (ref s));
1609                                         } else {
1610                                                 Error_InvalidDirective ();
1611                                                 return false;
1612                                         }
1613                                 }
1614                         }
1615                         return va;
1616                 }
1617
1618                 //
1619                 // Evaluates an expression for `#if' or `#elif'
1620                 //
1621                 bool pp_expr (ref string s)
1622                 {
1623                         bool va = pp_and (ref s);
1624                         s = s.Trim ();
1625                         int len = s.Length;
1626                         if (len > 0){
1627                                 char c = s [0];
1628
1629                                 if (c == '|'){
1630                                         if (len > 2 && s [1] == '|'){
1631                                                 s = s.Substring (2);
1632                                                 return va | pp_expr (ref s);
1633                                         } else {
1634                                                 Error_InvalidDirective ();
1635                                                 return false;
1636                                         }
1637                                 }
1638                         }
1639
1640                         return va;
1641                 }
1642
1643                 bool eval (string s)
1644                 {
1645                         bool v = pp_expr (ref s);
1646                         s = s.Trim ();
1647                         if (s.Length != 0){
1648                                 Error_InvalidDirective ();
1649                                 return false;
1650                         }
1651
1652                         return v;
1653                 }
1654
1655                 void Error_InvalidDirective ()
1656                 {
1657                         Report.Error (1517, Location, "Invalid pre-processor directive");
1658                 }
1659
1660                 void Error_UnexpectedDirective (string extra)
1661                 {
1662                         Report.Error (
1663                                 1028, Location,
1664                                 "Unexpected processor directive (" + extra + ")");
1665                 }
1666
1667                 void Error_TokensSeen ()
1668                 {
1669                         Report.Error (
1670                                 1032, Location,
1671                                 "Cannot define or undefine pre-processor symbols after a token in the file");
1672                 }
1673
1674                 //
1675                 // if true, then the code continues processing the code
1676                 // if false, the code stays in a loop until another directive is
1677                 // reached.
1678                 //
1679                 bool handle_preprocessing_directive (bool caller_is_taking)
1680                 {
1681                         string cmd, arg;
1682                         bool region_directive = false;
1683
1684                         get_cmd_arg (out cmd, out arg);
1685
1686                         // Eat any trailing whitespaces and single-line comments
1687                         if (arg.IndexOf ("//") != -1)
1688                                 arg = arg.Substring (0, arg.IndexOf ("//"));
1689                         arg = arg.TrimEnd (' ', '\t');
1690
1691                         //
1692                         // The first group of pre-processing instructions is always processed
1693                         //
1694                         switch (cmd){
1695                         case "pragma":
1696                                 if (RootContext.Version == LanguageVersion.ISO_1) {
1697                                         Report.FeatureIsNotStandardized (Location, "#pragma");
1698                                         return caller_is_taking;
1699                                 }
1700
1701                                 PreProcessPragma (arg);
1702                                 return caller_is_taking;
1703
1704                         case "line":
1705                                 if (!PreProcessLine (arg))
1706                                         Report.Error (
1707                                                 1576, Location,
1708                                                 "Argument to #line directive is missing or invalid");
1709                                 return caller_is_taking;
1710
1711                         case "region":
1712                                 region_directive = true;
1713                                 arg = "true";
1714                                 goto case "if";
1715
1716                         case "endregion":
1717                                 region_directive = true;
1718                                 goto case "endif";
1719
1720                         case "if":
1721                                 if (arg == ""){
1722                                         Error_InvalidDirective ();
1723                                         return true;
1724                                 }
1725                                 bool taking = false;
1726                                 if (ifstack == null)
1727                                         ifstack = new Stack ();
1728
1729                                 if (ifstack.Count == 0){
1730                                         taking = true;
1731                                 } else {
1732                                         int state = (int) ifstack.Peek ();
1733                                         if ((state & TAKING) != 0)
1734                                                 taking = true;
1735                                 }
1736
1737                                 if (eval (arg) && taking){
1738                                         int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;
1739                                         if (region_directive)
1740                                                 push |= REGION;
1741                                         ifstack.Push (push);
1742                                         return true;
1743                                 } else {
1744                                         int push = (taking ? PARENT_TAKING : 0);
1745                                         if (region_directive)
1746                                                 push |= REGION;
1747                                         ifstack.Push (push);
1748                                         return false;
1749                                 }
1750
1751                         case "endif":
1752                                 if (ifstack == null || ifstack.Count == 0){
1753                                         Error_UnexpectedDirective ("no #if for this #endif");
1754                                         return true;
1755                                 } else {
1756                                         int pop = (int) ifstack.Pop ();
1757
1758                                         if (region_directive && ((pop & REGION) == 0))
1759                                                 Report.Error (1027, Location, "#endif directive expected");
1760                                         else if (!region_directive && ((pop & REGION) != 0))
1761                                                 Report.Error (1038, Location, "#endregion directive expected");
1762
1763                                         if (ifstack.Count == 0)
1764                                                 return true;
1765                                         else {
1766                                                 int state = (int) ifstack.Peek ();
1767
1768                                                 if ((state & TAKING) != 0)
1769                                                         return true;
1770                                                 else
1771                                                         return false;
1772                                         }
1773                                 }
1774
1775                         case "elif":
1776                                 if (ifstack == null || ifstack.Count == 0){
1777                                         Error_UnexpectedDirective ("no #if for this #elif");
1778                                         return true;
1779                                 } else {
1780                                         int state = (int) ifstack.Peek ();
1781
1782                                         if ((state & REGION) != 0) {
1783                                                 Report.Error (1038, Location, "#endregion directive expected");
1784                                                 return true;
1785                                         }
1786
1787                                         if ((state & ELSE_SEEN) != 0){
1788                                                 Error_UnexpectedDirective ("#elif not valid after #else");
1789                                                 return true;
1790                                         }
1791
1792                                         if ((state & (TAKEN_BEFORE | TAKING)) != 0)
1793                                                 return false;
1794
1795                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
1796                                                 state = (int) ifstack.Pop ();
1797                                                 ifstack.Push (state | TAKING | TAKEN_BEFORE);
1798                                                 return true;
1799                                         } else
1800                                                 return false;
1801                                 }
1802
1803                         case "else":
1804                                 if (ifstack == null || ifstack.Count == 0){
1805                                         Report.Error (
1806                                                 1028, Location,
1807                                                 "Unexpected processor directive (no #if for this #else)");
1808                                         return true;
1809                                 } else {
1810                                         int state = (int) ifstack.Peek ();
1811
1812                                         if ((state & REGION) != 0) {
1813                                                 Report.Error (1038, Location, "#endregion directive expected");
1814                                                 return true;
1815                                         }
1816
1817                                         if ((state & ELSE_SEEN) != 0){
1818                                                 Error_UnexpectedDirective ("#else within #else");
1819                                                 return true;
1820                                         }
1821
1822                                         ifstack.Pop ();
1823
1824                                         bool ret;
1825                                         if ((state & TAKEN_BEFORE) == 0){
1826                                                 ret = ((state & PARENT_TAKING) != 0);
1827                                         } else
1828                                                 ret = false;
1829
1830                                         if (ret)
1831                                                 state |= TAKING;
1832                                         else
1833                                                 state &= ~TAKING;
1834
1835                                         ifstack.Push (state | ELSE_SEEN);
1836
1837                                         return ret;
1838                                 }
1839                         }
1840
1841                         //
1842                         // These are only processed if we are in a `taking' block
1843                         //
1844                         if (!caller_is_taking)
1845                                 return false;
1846
1847                         switch (cmd){
1848                         case "define":
1849                                 if (any_token_seen){
1850                                         Error_TokensSeen ();
1851                                         return true;
1852                                 }
1853                                 PreProcessDefinition (true, arg);
1854                                 return true;
1855
1856                         case "undef":
1857                                 if (any_token_seen){
1858                                         Error_TokensSeen ();
1859                                         return true;
1860                                 }
1861                                 PreProcessDefinition (false, arg);
1862                                 return true;
1863
1864                         case "error":
1865                                 Report.Error (1029, Location, "#error: '" + arg + "'");
1866                                 return true;
1867
1868                         case "warning":
1869                                 Report.Warning (1030, Location, "#warning: '{0}'", arg);
1870                                 return true;
1871                         }
1872
1873                         Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");
1874                         return true;
1875
1876                 }
1877
1878                 private int consume_string (bool quoted)
1879                 {
1880                         int c;
1881                         string_builder.Length = 0;
1882
1883                         while ((c = getChar ()) != -1){
1884                                 if (c == '"'){
1885                                         if (quoted && peekChar () == '"'){
1886                                                 string_builder.Append ((char) c);
1887                                                 getChar ();
1888                                                 continue;
1889                                         } else {
1890                                                 val = string_builder.ToString ();
1891                                                 return Token.LITERAL_STRING;
1892                                         }
1893                                 }
1894
1895                                 if (c == '\n'){
1896                                         if (!quoted)
1897                                                 Report.Error (1010, Location, "Newline in constant");
1898                                         line++;
1899                                         ref_line++;
1900                                         col = 0;
1901                                 } else
1902                                         col++;
1903
1904                                 if (!quoted){
1905                                         c = escape (c);
1906                                         if (c == -1)
1907                                                 return Token.ERROR;
1908                                 }
1909                                 string_builder.Append ((char) c);
1910                         }
1911
1912                         Report.Error (1039, Location, "Unterminated string literal");
1913                         return Token.EOF;
1914                 }
1915
1916                 private int consume_identifier (int s)
1917                 {
1918                         int res = consume_identifier (s, false);
1919
1920                         if (doc_state == XmlCommentState.Allowed)
1921                                 doc_state = XmlCommentState.NotAllowed;
1922                         switch (res) {
1923                         case Token.USING:
1924                         case Token.NAMESPACE:
1925                                 check_incorrect_doc_comment ();
1926                                 break;
1927                         }
1928
1929                         if (res == Token.PARTIAL) {
1930                                 // Save current position and parse next token.
1931                                 int old = reader.Position;
1932                                 int old_putback = putback_char;
1933
1934                                 putback_char = -1;
1935
1936                                 int next_token = token ();
1937                                 bool ok = (next_token == Token.CLASS) ||
1938                                         (next_token == Token.STRUCT) ||
1939                                         (next_token == Token.INTERFACE);
1940
1941                                 reader.Position = old;
1942                                 putback_char = old_putback;
1943
1944                                 if (ok)
1945                                         return res;
1946                                 else {
1947                                         val = "partial";
1948                                         return Token.IDENTIFIER;
1949                                 }
1950                         }
1951
1952                         return res;
1953                 }
1954
1955                 private int consume_identifier (int s, bool quoted)
1956                 {
1957                         int pos = 1;
1958                         int c;
1959
1960                         id_builder [0] = (char) s;
1961
1962                         while ((c = reader.Read ()) != -1) {
1963                                 if (is_identifier_part_character ((char) c)){
1964                                         if (pos == max_id_size){
1965                                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
1966                                                 return Token.ERROR;
1967                                         }
1968
1969                                         id_builder [pos++] = (char) c;
1970                                         putback_char = -1;
1971                                         col++;
1972                                 } else {
1973                                         putback_char = c;
1974                                         break;
1975                                 }
1976                         }
1977
1978                         //
1979                         // Optimization: avoids doing the keyword lookup
1980                         // on uppercase letters and _
1981                         //
1982                         if (!quoted && (s >= 'a' || s == '_')){
1983                                 int keyword = GetKeyword (id_builder, pos);
1984                                 if (keyword != -1)
1985                                 return keyword;
1986                         }
1987
1988                         //
1989                         // Keep identifiers in an array of hashtables to avoid needless
1990                         // allocations
1991                         //
1992
1993                         if (identifiers [pos] != null) {
1994                                 val = identifiers [pos][id_builder];
1995                                 if (val != null) {
1996                                         return Token.IDENTIFIER;
1997                                 }
1998                         }
1999                         else
2000                                 identifiers [pos] = new CharArrayHashtable (pos);
2001
2002                         val = new String (id_builder, 0, pos);
2003
2004                         char [] chars = new char [pos];
2005                         Array.Copy (id_builder, chars, pos);
2006
2007                         identifiers [pos] [chars] = val;
2008
2009                         return Token.IDENTIFIER;
2010                 }
2011
2012                 int consume_whitespace ()
2013                 {
2014                         int t;
2015                         bool doread = false;
2016                         int c;
2017
2018                         // Whether we have seen comments on the current line
2019                         bool comments_seen = false;
2020
2021                         val = null;
2022                         // optimization: eliminate col and implement #directive semantic correctly.
2023                         for (;(c = getChar ()) != -1; col++) {
2024                                 if (c == ' ')
2025                                         continue;
2026
2027                                 if (c == '\t') {
2028                                         col = (((col + 8) / 8) * 8) - 1;
2029                                         continue;
2030                                 }
2031
2032                                 if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0)
2033                                         continue;
2034
2035                                 if (c == '\r') {
2036                                         if (peekChar () == '\n')
2037                                                 getChar ();
2038
2039                                         line++;
2040                                         ref_line++;
2041                                         col = 0;
2042                                         any_token_seen |= tokens_seen;
2043                                         tokens_seen = false;
2044                                         continue;
2045                                 }
2046
2047                                 // Handle double-slash comments.
2048                                 if (c == '/'){
2049                                         int d = peekChar ();
2050
2051                                         if (d == '/'){
2052                                                 getChar ();
2053                                                 if (RootContext.Documentation != null && peekChar () == '/') {
2054                                                         getChar ();
2055                                                         // Don't allow ////.
2056                                                         if ((d = peekChar ()) != '/') {
2057                                                                 if (doc_state == XmlCommentState.Allowed)
2058                                                                         handle_one_line_xml_comment ();
2059                                                                 else if (doc_state == XmlCommentState.NotAllowed)
2060                                                                         warn_incorrect_doc_comment ();
2061                                                         }
2062                                                 }
2063                                                 while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
2064                                                         col++;
2065                                                 if (d == '\n'){
2066                                                         line++;
2067                                                         ref_line++;
2068                                                         col = 0;
2069                                                 }
2070                                                 any_token_seen |= tokens_seen;
2071                                                 tokens_seen = false;
2072                                                 continue;
2073                                         } else if (d == '*'){
2074                                                 getChar ();
2075                                                 bool docAppend = false;
2076                                                 if (RootContext.Documentation != null && peekChar () == '*') {
2077                                                         getChar ();
2078                                                         // But when it is /**/, just do nothing.
2079                                                         if (peekChar () == '/') {
2080                                                                 getChar ();
2081                                                                 continue;
2082                                                         }
2083                                                         if (doc_state == XmlCommentState.Allowed)
2084                                                                 docAppend = true;
2085                                                         else if (doc_state == XmlCommentState.NotAllowed)
2086                                                                 warn_incorrect_doc_comment ();
2087                                                 }
2088
2089                                                 int current_comment_start = 0;
2090                                                 if (docAppend) {
2091                                                         current_comment_start = xml_comment_buffer.Length;
2092                                                         xml_comment_buffer.Append (Environment.NewLine);
2093                                                 }
2094
2095                                                 while ((d = getChar ()) != -1){
2096                                                         if (d == '*' && peekChar () == '/'){
2097                                                                 getChar ();
2098                                                                 col++;
2099                                                                 comments_seen = true;
2100                                                                 break;
2101                                                         }
2102                                                         if (docAppend)
2103                                                                 xml_comment_buffer.Append ((char) d);
2104
2105                                                         if (d == '\n'){
2106                                                                 line++;
2107                                                                 ref_line++;
2108                                                                 col = 0;
2109                                                                 any_token_seen |= tokens_seen;
2110                                                                 tokens_seen = false;
2111                                                         }
2112                                                 }
2113                                                 if (docAppend)
2114                                                         update_formatted_doc_comment (current_comment_start);
2115                                                 continue;
2116                                         }
2117                                         goto is_punct_label;
2118                                 }
2119
2120                         is_punct_label:
2121                                 // white space
2122                                 if (c == '\n'){
2123                                         line++;
2124                                         ref_line++;
2125                                         col = 0;
2126                                         any_token_seen |= tokens_seen;
2127                                         tokens_seen = false;
2128                                         comments_seen = false;
2129                                         continue;
2130                                 }
2131
2132                                 /* For now, ignore pre-processor commands */
2133                                 // FIXME: In C# the '#' is not limited to appear
2134                                 // on the first column.
2135                                 if (c == '#') {
2136                                         bool cont = true;
2137
2138                                         if (tokens_seen || comments_seen) {
2139                                                error_details = "Preprocessor directives must appear as the first non-whitespace " +
2140                                                        "character on a line.";
2141
2142                                                Report.Error (1040, Location, error_details);
2143
2144                                                return Token.ERROR;
2145                                        }
2146
2147                                 start_again:
2148
2149                                         cont = handle_preprocessing_directive (cont);
2150
2151                                         if (cont){
2152                                                 col = 0;
2153                                                 continue;
2154                                         }
2155                                         col = 1;
2156
2157                                         bool skipping = false;
2158                                         for (;(c = getChar ()) != -1; col++){
2159                                                 if (c == '\n'){
2160                                                         col = 0;
2161                                                         line++;
2162                                                         ref_line++;
2163                                                         skipping = false;
2164                                                 } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
2165                                                         continue;
2166                                                 else if (c != '#')
2167                                                         skipping = true;
2168                                                 if (c == '#' && !skipping)
2169                                                         goto start_again;
2170                                         }
2171                                         any_token_seen |= tokens_seen;
2172                                         tokens_seen = false;
2173                                         if (c == -1)
2174                                                 Report.Error (1027, Location, "#endif/#endregion expected");
2175                                         continue;
2176                                 }
2177
2178                                 return c;
2179                         }
2180
2181                         return -1;
2182                 }
2183
2184                 public int xtoken ()
2185                 {
2186                         int t;
2187                         bool doread = false;
2188                         int c;
2189
2190                         val = null;
2191                         // optimization: eliminate col and implement #directive semantic correctly.
2192
2193                         c = consume_whitespace ();
2194                         if (c == -1)
2195                                 return Token.EOF;
2196
2197                         if (is_identifier_start_character ((char)c)){
2198                                 tokens_seen = true;
2199                                         return consume_identifier (c);
2200                         }
2201
2202                 is_punct_label:
2203                         if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
2204                                 tokens_seen = true;
2205                                 if (doread){
2206                                         getChar ();
2207                                         col++;
2208                                 }
2209                                 return t;
2210                         }
2211
2212                         if (c >= '0' && c <= '9'){
2213                                 tokens_seen = true;
2214                                 return is_number (c);
2215                         }
2216
2217                         if (c == '.'){
2218                                 tokens_seen = true;
2219                                 int peek = peekChar ();
2220                                 if (peek >= '0' && peek <= '9')
2221                                         return is_number (c);
2222                                 return Token.DOT;
2223                         }
2224
2225                         if (c == '"')
2226                                 return consume_string (false);
2227
2228                         if (c == '\''){
2229                                 c = getChar ();
2230                                 tokens_seen = true;
2231                                 if (c == '\''){
2232                                         error_details = "Empty character literal";
2233                                         Report.Error (1011, Location, error_details);
2234                                         return Token.ERROR;
2235                                 }
2236                                 c = escape (c);
2237                                 if (c == -1)
2238                                         return Token.ERROR;
2239                                 val = new System.Char ();
2240                                 val = (char) c;
2241                                 c = getChar ();
2242
2243                                 if (c != '\''){
2244                                         error_details = "Too many characters in character literal";
2245                                         Report.Error (1012, Location, error_details);
2246
2247                                         // Try to recover, read until newline or next "'"
2248                                         while ((c = getChar ()) != -1){
2249                                                 if (c == '\n' || c == '\''){
2250                                                         line++;
2251                                                         ref_line++;
2252                                                         col = 0;
2253                                                         break;
2254                                                 } else
2255                                                         col++;
2256                                         }
2257                                         return Token.ERROR;
2258                                 }
2259                                 return Token.LITERAL_CHARACTER;
2260                         }
2261
2262                         if (c == '@') {
2263                                 c = getChar ();
2264                                 if (c == '"') {
2265                                         tokens_seen = true;
2266                                         return consume_string (true);
2267                                 } else if (is_identifier_start_character ((char) c)){
2268                                         return consume_identifier (c, true);
2269                                 } else {
2270                                         Report.Error (1033, Location, "'@' must be followed by string constant or identifier");
2271                                 }
2272                         }
2273
2274                         if (c == '#') {
2275                                 error_details = "Preprocessor directives must appear as the first non-whitespace " +
2276                                         "character on a line.";
2277
2278                                 Report.Error (1040, Location, error_details);
2279
2280                                 return Token.ERROR;
2281                         }
2282
2283                         error_details = ((char)c).ToString ();
2284
2285                         return Token.ERROR;
2286                 }
2287
2288                 //
2289                 // Handles one line xml comment
2290                 //
2291                 private void handle_one_line_xml_comment ()
2292                 {
2293                         int c;
2294                         while ((c = peekChar ()) == ' ')
2295                                 getChar (); // skip heading whitespaces.
2296                         while ((c = peekChar ()) != -1 && c != '\n' && c != '\r') {
2297                                 col++;
2298                                 xml_comment_buffer.Append ((char) getChar ());
2299                         }
2300                         if (c == '\r' || c == '\n')
2301                                 xml_comment_buffer.Append (Environment.NewLine);
2302                 }
2303
2304                 //
2305                 // Remove heading "*" in Javadoc-like xml documentation.
2306                 //
2307                 private void update_formatted_doc_comment (int current_comment_start)
2308                 {
2309                         int length = xml_comment_buffer.Length - current_comment_start;
2310                         string [] lines = xml_comment_buffer.ToString (
2311                                 current_comment_start,
2312                                 length).Replace ("\r", "").Split ('\n');
2313
2314                         // The first line starts with /**, thus it is not target
2315                         // for the format check.
2316                         for (int i = 1; i < lines.Length; i++) {
2317                                 string s = lines [i];
2318                                 int idx = s.IndexOf ('*');
2319                                 string head = null;
2320                                 if (idx < 0) {
2321                                         if (i < lines.Length - 1)
2322                                                 return;
2323                                         head = s;
2324                                 } else
2325                                         head = s.Substring (0, idx);
2326                                 foreach (char c in head)
2327                                         if (c != ' ')
2328                                                 return;
2329                                 lines [i] = s.Substring (idx + 1);
2330                         }
2331                         xml_comment_buffer.Remove (current_comment_start, length);
2332                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
2333                 }
2334
2335                 //
2336                 // Checks if there was incorrect doc comments and raise
2337                 // warnings.
2338                 //
2339                 public void check_incorrect_doc_comment ()
2340                 {
2341                         if (xml_comment_buffer.Length > 0)
2342                                 warn_incorrect_doc_comment ();
2343                 }
2344
2345                 //
2346                 // Raises a warning when tokenizer found incorrect doccomment
2347                 // markup.
2348                 //
2349                 private void warn_incorrect_doc_comment ()
2350                 {
2351                         doc_state = XmlCommentState.Error;
2352                         // in csc, it is 'XML comment is not placed on a valid
2353                         // language element'. But that does not make sense.
2354                         Report.Warning (1587, 2, Location, "XML comment is placed on an invalid language element which can not accept it.");
2355                 }
2356
2357                 //
2358                 // Consumes the saved xml comment lines (if any)
2359                 // as for current target member or type.
2360                 //
2361                 public string consume_doc_comment ()
2362                 {
2363                         if (xml_comment_buffer.Length > 0) {
2364                                 string ret = xml_comment_buffer.ToString ();
2365                                 xml_comment_buffer.Length = 0;
2366                                 return ret;
2367                         }
2368                         return null;
2369                 }
2370
2371                 public void cleanup ()
2372                 {
2373                         if (ifstack != null && ifstack.Count >= 1) {
2374                                 int state = (int) ifstack.Pop ();
2375                                 if ((state & REGION) != 0)
2376                                         Report.Error (1038, Location, "#endregion directive expected");
2377                                 else
2378                                         Report.Error (1027, Location, "#endif directive expected");
2379                         }
2380
2381                 }
2382         }
2383
2384         //
2385         // Indicates whether it accepts XML documentation or not.
2386         //
2387         public enum XmlCommentState {
2388                 // comment is allowed in this state.
2389                 Allowed,
2390                 // comment is not allowed in this state.
2391                 NotAllowed,
2392                 // once comments appeared when it is NotAllowed, then the
2393                 // state is changed to it, until the state is changed to
2394                 // .Allowed.
2395                 Error
2396         }
2397 }
2398