mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Licensed under the terms of the GNU GPL
   9 //
  10 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // (C) 2004 Novell, Inc
  12 //
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections;
  18 using System.IO;
  19 using System.Globalization;
  20 using System.Reflection;
  21
  22 namespace Mono.CSharp
  23 {
  24         /// <summary>
  25         ///    Tokenizer for C# source code.
  26         /// </summary>
  27
  28         public class Tokenizer : yyParser.yyInput
  29         {
  30                 SeekableStreamReader reader;
  31                 SourceFile ref_name;
  32                 SourceFile file_name;
  33                 int ref_line = 1;
  34                 int line = 1;
  35                 int col = 0;
  36                 int previous_col;
  37                 int current_token;
  38                 bool handle_get_set = false;
  39                 bool handle_remove_add = false;
  40                 bool handle_assembly = false;
  41                 bool handle_constraints = false;
  42                 bool handle_typeof = false;
  43                 bool lambda_arguments_parsing;
  44                 Location current_location;
  45                 Location current_comment_location = Location.Null;
  46                 ArrayList escaped_identifiers = new ArrayList ();
  47
  48 #if GMCS_SOURCE
  49                 bool query_parsing;
  50 #endif
  51
  52                 static bool IsLinqEnabled {
  53                         get {
  54                                 return RootContext.Version == LanguageVersion.LINQ;
  55                         }
  56                 }
  57
  58                 //
  59                 // XML documentation buffer. The save point is used to divide
  60                 // comments on types and comments on members.
  61                 //
  62                 StringBuilder xml_comment_buffer;
  63
  64                 //
  65                 // See comment on XmlCommentState enumeration.
  66                 //
  67                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
  68
  69                 //
  70                 // Whether tokens have been seen on this line
  71                 //
  72                 bool tokens_seen = false;
  73
  74                 //
  75                 // Whether a token has been seen on the file
  76                 // This is needed because `define' is not allowed to be used
  77                 // after a token has been seen.
  78                 //
  79                 bool any_token_seen = false;
  80
  81                 static Hashtable token_values;
  82                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
  83
  84                 private static Hashtable TokenValueName
  85                 {
  86                         get {
  87                                 if (token_values == null)
  88                                         token_values = GetTokenValueNameHash ();
  89
  90                                 return token_values;
  91                         }
  92                 }
  93
  94                 private static Hashtable GetTokenValueNameHash ()
  95                 {
  96                         Type t = typeof (Token);
  97                         FieldInfo [] fields = t.GetFields ();
  98                         Hashtable hash = new Hashtable ();
  99                         foreach (FieldInfo field in fields) {
 100                                 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))
 101                                         hash.Add (field.GetValue (null), field.Name);
 102                         }
 103                         return hash;
 104                 }
 105
 106                 //
 107                 // Returns a verbose representation of the current location
 108                 //
 109                 public string location {
 110                         get {
 111                                 string det;
 112
 113                                 if (current_token == Token.ERROR)
 114                                         det = "detail: " + error_details;
 115                                 else
 116                                         det = "";
 117
 118                                 // return "Line:     "+line+" Col: "+col + "\n" +
 119                                 //       "VirtLine: "+ref_line +
 120                                 //       " Token: "+current_token + " " + det;
 121                                 string current_token_name = TokenValueName [current_token] as string;
 122                                 if (current_token_name == null)
 123                                         current_token_name = current_token.ToString ();
 124
 125                                 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,
 126                                                                                        ref_line,
 127                                                                                        col,
 128                                                                                        current_token_name,
 129                                                                                        det);
 130                         }
 131                 }
 132
 133                 public bool PropertyParsing {
 134                         get {
 135                                 return handle_get_set;
 136                         }
 137
 138                         set {
 139                                 handle_get_set = value;
 140                         }
 141                 }
 142
 143                 public bool AssemblyTargetParsing {
 144                         get {
 145                                 return handle_assembly;
 146                         }
 147
 148                         set {
 149                                 handle_assembly = value;
 150                         }
 151                 }
 152
 153                 public bool EventParsing {
 154                         get {
 155                                 return handle_remove_add;
 156                         }
 157
 158                         set {
 159                                 handle_remove_add = value;
 160                         }
 161                 }
 162
 163                 public bool ConstraintsParsing {
 164                         get {
 165                                 return handle_constraints;
 166                         }
 167
 168                         set {
 169                                 handle_constraints = value;
 170                         }
 171                 }
 172
 173                 public bool TypeOfParsing {
 174                         get {
 175                                 return handle_typeof;
 176                         }
 177
 178                         set {
 179                                 handle_typeof = value;
 180                         }
 181                 }
 182
 183                 public XmlCommentState doc_state {
 184                         get { return xml_doc_state; }
 185                         set {
 186                                 if (value == XmlCommentState.Allowed) {
 187                                         check_incorrect_doc_comment ();
 188                                         reset_doc_comment ();
 189                                 }
 190                                 xml_doc_state = value;
 191                         }
 192                 }
 193
 194                 public bool IsEscapedIdentifier (Location loc)
 195                 {
 196                         foreach (LocatedToken lt in escaped_identifiers)
 197                                 if (lt.Location.Equals (loc))
 198                                         return true;
 199                         return false;
 200                 }
 201
 202                 //
 203                 // Class variables
 204                 //
 205                 static CharArrayHashtable[] keywords;
 206                 static Hashtable keyword_strings;
 207                 static NumberStyles styles;
 208                 static NumberFormatInfo csharp_format_info;
 209
 210                 //
 211                 // Values for the associated token returned
 212                 //
 213                 int putback_char;
 214                 Object val;
 215
 216                 //
 217                 // Pre-processor
 218                 //
 219                 Hashtable defines;
 220
 221                 const int TAKING        = 1;
 222                 const int ELSE_SEEN     = 4;
 223                 const int PARENT_TAKING = 8;
 224                 const int REGION        = 16;
 225
 226                 //
 227                 // pre-processor if stack state:
 228                 //
 229                 Stack ifstack;
 230
 231                 static System.Text.StringBuilder string_builder;
 232
 233                 const int max_id_size = 512;
 234                 static char [] id_builder = new char [max_id_size];
 235
 236                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 237
 238                 const int max_number_size = 512;
 239                 static char [] number_builder = new char [max_number_size];
 240                 static int number_pos;
 241
 242                 //
 243                 // Details about the error encoutered by the tokenizer
 244                 //
 245                 string error_details;
 246
 247                 public string error {
 248                         get {
 249                                 return error_details;
 250                         }
 251                 }
 252
 253                 public int Line {
 254                         get {
 255                                 return ref_line;
 256                         }
 257                 }
 258
 259                 public int Col {
 260                         get {
 261                                 return col;
 262                         }
 263                 }
 264
 265                 //
 266                 // This is used when the tokenizer needs to save
 267                 // the current position as it needs to do some parsing
 268                 // on its own to deamiguate a token in behalf of the
 269                 // parser.
 270                 //
 271                 Stack position_stack = new Stack (2);
 272                 class Position {
 273                         public int position;
 274                         public int ref_line;
 275                         public int col;
 276                         public int putback_char;
 277                         public int previous_col;
 278                         public Stack ifstack;
 279                         public int parsing_generic_less_than;
 280                         public int current_token;
 281
 282                         public Position (Tokenizer t)
 283                         {
 284                                 position = t.reader.Position;
 285                                 ref_line = t.ref_line;
 286                                 col = t.col;
 287                                 putback_char = t.putback_char;
 288                                 previous_col = t.previous_col;
 289                                 if (t.ifstack != null && t.ifstack.Count != 0)
 290                                         ifstack = (Stack)t.ifstack.Clone ();
 291                                 parsing_generic_less_than = t.parsing_generic_less_than;
 292                                 current_token = t.current_token;
 293                         }
 294                 }
 295
 296                 public void PushPosition ()
 297                 {
 298                         position_stack.Push (new Position (this));
 299                 }
 300
 301                 public void PopPosition ()
 302                 {
 303                         Position p = (Position) position_stack.Pop ();
 304
 305                         reader.Position = p.position;
 306                         ref_line = p.ref_line;
 307                         col = p.col;
 308                         putback_char = p.putback_char;
 309                         previous_col = p.previous_col;
 310                         ifstack = p.ifstack;
 311                         parsing_generic_less_than = p.parsing_generic_less_than;
 312                         current_token = p.current_token;
 313                 }
 314
 315                 // Do not reset the position, ignore it.
 316                 public void DiscardPosition ()
 317                 {
 318                         position_stack.Pop ();
 319                 }
 320
 321                 static void AddKeyword (string kw, int token)
 322                 {
 323                         keyword_strings.Add (kw, kw);
 324                         if (keywords [kw.Length] == null) {
 325                                 keywords [kw.Length] = new CharArrayHashtable (kw.Length);
 326                         }
 327                         keywords [kw.Length] [kw.ToCharArray ()] = token;
 328                 }
 329
 330                 static void InitTokens ()
 331                 {
 332                         keyword_strings = new Hashtable ();
 333                         keywords = new CharArrayHashtable [64];
 334
 335                         AddKeyword ("__arglist", Token.ARGLIST);
 336                         AddKeyword ("abstract", Token.ABSTRACT);
 337                         AddKeyword ("as", Token.AS);
 338                         AddKeyword ("add", Token.ADD);
 339                         AddKeyword ("assembly", Token.ASSEMBLY);
 340                         AddKeyword ("base", Token.BASE);
 341                         AddKeyword ("bool", Token.BOOL);
 342                         AddKeyword ("break", Token.BREAK);
 343                         AddKeyword ("byte", Token.BYTE);
 344                         AddKeyword ("case", Token.CASE);
 345                         AddKeyword ("catch", Token.CATCH);
 346                         AddKeyword ("char", Token.CHAR);
 347                         AddKeyword ("checked", Token.CHECKED);
 348                         AddKeyword ("class", Token.CLASS);
 349                         AddKeyword ("const", Token.CONST);
 350                         AddKeyword ("continue", Token.CONTINUE);
 351                         AddKeyword ("decimal", Token.DECIMAL);
 352                         AddKeyword ("default", Token.DEFAULT);
 353                         AddKeyword ("delegate", Token.DELEGATE);
 354                         AddKeyword ("do", Token.DO);
 355                         AddKeyword ("double", Token.DOUBLE);
 356                         AddKeyword ("else", Token.ELSE);
 357                         AddKeyword ("enum", Token.ENUM);
 358                         AddKeyword ("event", Token.EVENT);
 359                         AddKeyword ("explicit", Token.EXPLICIT);
 360                         AddKeyword ("extern", Token.EXTERN);
 361                         AddKeyword ("false", Token.FALSE);
 362                         AddKeyword ("finally", Token.FINALLY);
 363                         AddKeyword ("fixed", Token.FIXED);
 364                         AddKeyword ("float", Token.FLOAT);
 365                         AddKeyword ("for", Token.FOR);
 366                         AddKeyword ("foreach", Token.FOREACH);
 367                         AddKeyword ("goto", Token.GOTO);
 368                         AddKeyword ("get", Token.GET);
 369                         AddKeyword ("if", Token.IF);
 370                         AddKeyword ("implicit", Token.IMPLICIT);
 371                         AddKeyword ("in", Token.IN);
 372                         AddKeyword ("int", Token.INT);
 373                         AddKeyword ("interface", Token.INTERFACE);
 374                         AddKeyword ("internal", Token.INTERNAL);
 375                         AddKeyword ("is", Token.IS);
 376                         AddKeyword ("lock", Token.LOCK);
 377                         AddKeyword ("long", Token.LONG);
 378                         AddKeyword ("namespace", Token.NAMESPACE);
 379                         AddKeyword ("new", Token.NEW);
 380                         AddKeyword ("null", Token.NULL);
 381                         AddKeyword ("object", Token.OBJECT);
 382                         AddKeyword ("operator", Token.OPERATOR);
 383                         AddKeyword ("out", Token.OUT);
 384                         AddKeyword ("override", Token.OVERRIDE);
 385                         AddKeyword ("params", Token.PARAMS);
 386                         AddKeyword ("private", Token.PRIVATE);
 387                         AddKeyword ("protected", Token.PROTECTED);
 388                         AddKeyword ("public", Token.PUBLIC);
 389                         AddKeyword ("readonly", Token.READONLY);
 390                         AddKeyword ("ref", Token.REF);
 391                         AddKeyword ("remove", Token.REMOVE);
 392                         AddKeyword ("return", Token.RETURN);
 393                         AddKeyword ("sbyte", Token.SBYTE);
 394                         AddKeyword ("sealed", Token.SEALED);
 395                         AddKeyword ("set", Token.SET);
 396                         AddKeyword ("short", Token.SHORT);
 397                         AddKeyword ("sizeof", Token.SIZEOF);
 398                         AddKeyword ("stackalloc", Token.STACKALLOC);
 399                         AddKeyword ("static", Token.STATIC);
 400                         AddKeyword ("string", Token.STRING);
 401                         AddKeyword ("struct", Token.STRUCT);
 402                         AddKeyword ("switch", Token.SWITCH);
 403                         AddKeyword ("this", Token.THIS);
 404                         AddKeyword ("throw", Token.THROW);
 405                         AddKeyword ("true", Token.TRUE);
 406                         AddKeyword ("try", Token.TRY);
 407                         AddKeyword ("typeof", Token.TYPEOF);
 408                         AddKeyword ("uint", Token.UINT);
 409                         AddKeyword ("ulong", Token.ULONG);
 410                         AddKeyword ("unchecked", Token.UNCHECKED);
 411                         AddKeyword ("unsafe", Token.UNSAFE);
 412                         AddKeyword ("ushort", Token.USHORT);
 413                         AddKeyword ("using", Token.USING);
 414                         AddKeyword ("virtual", Token.VIRTUAL);
 415                         AddKeyword ("void", Token.VOID);
 416                         AddKeyword ("volatile", Token.VOLATILE);
 417                         AddKeyword ("while", Token.WHILE);
 418                         AddKeyword ("partial", Token.PARTIAL);
 419 #if GMCS_SOURCE
 420                         AddKeyword ("where", Token.WHERE);
 421 #endif
 422                 }
 423
 424 #if GMCS_SOURCE
 425                 public static void InitializeLinqKeywords ()
 426                 {
 427                         AddKeyword ("from", Token.FROM);
 428                         AddKeyword ("join", Token.JOIN);
 429                         AddKeyword ("on", Token.ON);
 430                         AddKeyword ("equals", Token.EQUALS);
 431                         AddKeyword ("select", Token.SELECT);
 432                         AddKeyword ("group", Token.GROUP);
 433                         AddKeyword ("by", Token.BY);
 434                         AddKeyword ("let", Token.LET);
 435                         AddKeyword ("orderby", Token.ORDERBY);
 436                         AddKeyword ("ascending", Token.ASCENDING);
 437                         AddKeyword ("descending", Token.DESCENDING);
 438                         AddKeyword ("into", Token.INTO);
 439                 }
 440 #endif
 441
 442                 //
 443                 // Class initializer
 444                 //
 445                 static Tokenizer ()
 446                 {
 447                         Reset ();
 448                 }
 449
 450                 public static void Reset ()
 451                 {
 452                         InitTokens ();
 453                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 454                         styles = NumberStyles.Float;
 455
 456                         string_builder = new System.Text.StringBuilder ();
 457                 }
 458
 459                 int GetKeyword (char[] id, int id_len)
 460                 {
 461                         /*
 462                          * Keywords are stored in an array of hashtables grouped by their
 463                          * length.
 464                          */
 465
 466                         if ((id_len >= keywords.Length) || (keywords [id_len] == null))
 467                                 return -1;
 468                         object o = keywords [id_len] [id];
 469
 470                         if (o == null)
 471                                 return -1;
 472
 473                         int res = (int) o;
 474
 475                         if (handle_get_set == false && (res == Token.GET || res == Token.SET))
 476                                 return -1;
 477                         if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
 478                                 return -1;
 479                         if (handle_assembly == false && res == Token.ASSEMBLY)
 480                                 return -1;
 481 #if GMCS_SOURCE
 482                         if (IsLinqEnabled) {
 483                                 if (res == Token.FROM &&
 484                                         (current_token == Token.ASSIGN || current_token == Token.OPEN_BRACKET ||
 485                                          current_token == Token.RETURN || current_token == Token.IN)) {
 486                                         query_parsing = true;
 487                                         return res;
 488                                 }
 489
 490                                 if (!query_parsing && res > Token.QUERY_FIRST_TOKEN && res < Token.QUERY_LAST_TOKEN)
 491                                         return -1;
 492
 493                                 return res;
 494                         }
 495
 496                         if (!handle_constraints && res == Token.WHERE)
 497                                 return -1;
 498 #endif
 499                         return res;
 500
 501                 }
 502
 503                 public Location Location {
 504                         get { return current_location; }
 505                 }
 506
 507                 void define (string def)
 508                 {
 509                         if (!RootContext.AllDefines.Contains (def)){
 510                                 RootContext.AllDefines [def] = true;
 511                         }
 512                         if (defines.Contains (def))
 513                                 return;
 514                         defines [def] = true;
 515                 }
 516
 517                 public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)
 518                 {
 519                         this.ref_name = file;
 520                         this.file_name = file;
 521                         reader = input;
 522
 523                         putback_char = -1;
 524
 525                         if (defs != null){
 526                                 defines = new Hashtable ();
 527                                 foreach (string def in defs)
 528                                         define (def);
 529                         }
 530
 531                         xml_comment_buffer = new StringBuilder ();
 532
 533                         //
 534                         // FIXME: This could be `Location.Push' but we have to
 535                         // find out why the MS compiler allows this
 536                         //
 537                         Mono.CSharp.Location.Push (file);
 538                 }
 539
 540                 static bool is_identifier_start_character (char c)
 541                 {
 542                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter (c);
 543                 }
 544
 545                 static bool is_identifier_part_character (char c)
 546                 {
 547                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') ||
 548                                 Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 549                 }
 550
 551                 public static bool IsKeyword (string s)
 552                 {
 553                         return keyword_strings [s] != null;
 554                 }
 555
 556                 //
 557                 // Tests whether '(' is beggining of lambda parameters
 558                 //
 559                 bool IsLambdaOpenParens ()
 560                 {
 561                         int ntoken;
 562                         while ((ntoken = xtoken ()) != Token.EOF) {
 563                                 switch (ntoken) {
 564                                         case Token.CLOSE_PARENS:
 565                                                 return xtoken () == Token.ARROW;
 566
 567                                         case Token.STAR:
 568                                         case Token.SEMICOLON:
 569                                         case Token.OPEN_PARENS:
 570                                                 return false;
 571                                 }
 572                         }
 573
 574                         Error_TokenExpected (")");
 575                         return false;
 576                 }
 577
 578                 public static bool IsValidIdentifier (string s)
 579                 {
 580                         if (s == null || s.Length == 0)
 581                                 return false;
 582
 583                         if (!is_identifier_start_character (s [0]))
 584                                 return false;
 585
 586                         for (int i = 1; i < s.Length; i ++)
 587                                 if (! is_identifier_part_character (s [i]))
 588                                         return false;
 589
 590                         return true;
 591                 }
 592
 593                 bool parse_less_than ()
 594                 {
 595                 start:
 596                         int the_token = token ();
 597                         if (the_token == Token.OPEN_BRACKET) {
 598                                 do {
 599                                         the_token = token ();
 600                                 } while (the_token != Token.CLOSE_BRACKET);
 601                                 the_token = token ();
 602                         }
 603                         switch (the_token) {
 604                         case Token.IDENTIFIER:
 605                         case Token.OBJECT:
 606                         case Token.STRING:
 607                         case Token.BOOL:
 608                         case Token.DECIMAL:
 609                         case Token.FLOAT:
 610                         case Token.DOUBLE:
 611                         case Token.SBYTE:
 612                         case Token.BYTE:
 613                         case Token.SHORT:
 614                         case Token.USHORT:
 615                         case Token.INT:
 616                         case Token.UINT:
 617                         case Token.LONG:
 618                         case Token.ULONG:
 619                         case Token.CHAR:
 620                         case Token.VOID:
 621                                 break;
 622
 623                         default:
 624                                 return false;
 625                         }
 626                 again:
 627                         the_token = token ();
 628
 629                         if (the_token == Token.OP_GENERICS_GT)
 630                                 return true;
 631                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
 632                                 goto start;
 633                         else if (the_token == Token.INTERR || the_token == Token.STAR)
 634                                 goto again;
 635                         else if (the_token == Token.OP_GENERICS_LT) {
 636                                 if (!parse_less_than ())
 637                                         return false;
 638                                 goto again;
 639                         } else if (the_token == Token.OPEN_BRACKET) {
 640                         rank_specifiers:
 641                                 the_token = token ();
 642                                 if (the_token == Token.CLOSE_BRACKET)
 643                                         goto again;
 644                                 else if (the_token == Token.COMMA)
 645                                         goto rank_specifiers;
 646                                 return false;
 647                         }
 648
 649                         return false;
 650                 }
 651
 652 #if GMCS_SOURCE
 653                 public void PutbackNullable ()
 654                 {
 655                         if (nullable_pos < 0)
 656                                 throw new Exception ();
 657
 658                         current_token = -1;
 659                         val = null;
 660                         reader.Position = nullable_pos;
 661
 662                         putback_char = '?';
 663                 }
 664
 665                 public void PutbackCloseParens ()
 666                 {
 667                         putback_char = ')';
 668                 }
 669
 670
 671                 int nullable_pos = -1;
 672
 673                 public void CheckNullable (bool is_nullable)
 674                 {
 675                         if (is_nullable)
 676                                 nullable_pos = reader.Position;
 677                         else
 678                                 nullable_pos = -1;
 679                 }
 680
 681                 public bool QueryParsing {
 682                         set {
 683                                 query_parsing = value;
 684                         }
 685                 }
 686
 687                 bool parse_generic_dimension (out int dimension)
 688                 {
 689                         dimension = 1;
 690
 691                 again:
 692                         int the_token = token ();
 693                         if (the_token == Token.OP_GENERICS_GT)
 694                                 return true;
 695                         else if (the_token == Token.COMMA) {
 696                                 dimension++;
 697                                 goto again;
 698                         }
 699
 700                         return false;
 701                 }
 702 #endif
 703
 704                 public int peek_token ()
 705                 {
 706                         int the_token;
 707
 708                         PushPosition ();
 709                         the_token = token ();
 710                         PopPosition ();
 711
 712                         return the_token;
 713                 }
 714
 715                 int parsing_generic_less_than = 0;
 716
 717                 int is_punct (char c, ref bool doread)
 718                 {
 719                         int d;
 720                         int t;
 721
 722                         doread = false;
 723
 724                         switch (c){
 725                         case '{':
 726                                 val = Location;
 727                                 return Token.OPEN_BRACE;
 728                         case '}':
 729                                 val = Location;
 730                                 return Token.CLOSE_BRACE;
 731                         case '[':
 732                                 // To block doccomment inside attribute declaration.
 733                                 if (doc_state == XmlCommentState.Allowed)
 734                                         doc_state = XmlCommentState.NotAllowed;
 735                                 return Token.OPEN_BRACKET;
 736                         case ']':
 737                                 return Token.CLOSE_BRACKET;
 738                         case '(':
 739                                 if (IsLinqEnabled && !lambda_arguments_parsing) {
 740                                         lambda_arguments_parsing = true;
 741                                         PushPosition ();
 742                                         bool lambda_start = IsLambdaOpenParens ();
 743                                         PopPosition ();
 744                                         lambda_arguments_parsing = false;
 745                                         if (lambda_start)
 746                                                 return Token.OPEN_PARENS_LAMBDA;
 747                                 }
 748                                 return Token.OPEN_PARENS;
 749                         case ')': {
 750                                 if (deambiguate_close_parens == 0)
 751                                         return Token.CLOSE_PARENS;
 752
 753                                 --deambiguate_close_parens;
 754
 755                                 PushPosition ();
 756
 757                                 int new_token = xtoken ();
 758
 759                                 PopPosition ();
 760
 761                                 if (new_token == Token.OPEN_PARENS)
 762                                         return Token.CLOSE_PARENS_OPEN_PARENS;
 763                                 else if (new_token == Token.MINUS)
 764                                         return Token.CLOSE_PARENS_MINUS;
 765                                 else if (IsCastToken (new_token))
 766                                         return Token.CLOSE_PARENS_CAST;
 767                                 else
 768                                         return Token.CLOSE_PARENS_NO_CAST;
 769                         }
 770
 771                         case ',':
 772                                 return Token.COMMA;
 773                         case ';':
 774                                 val = Location;
 775                                 return Token.SEMICOLON;
 776                         case '~':
 777                                 val = Location;
 778                                 return Token.TILDE;
 779                         case '?':
 780 #if GMCS_SOURCE
 781                                 d = peek_char ();
 782                                 if (d == '?') {
 783                                         get_char ();
 784                                         return Token.OP_COALESCING;
 785                                 }
 786 #endif
 787                                 return Token.INTERR;
 788                         }
 789 #if GMCS_SOURCE
 790                         if (c == '<') {
 791                                 if (parsing_generic_less_than++ > 0)
 792                                         return Token.OP_GENERICS_LT;
 793
 794                                 if (handle_typeof) {
 795                                         int dimension;
 796                                         PushPosition ();
 797                                         if (parse_generic_dimension (out dimension)) {
 798                                                 val = dimension;
 799                                                 DiscardPosition ();
 800                                                 return Token.GENERIC_DIMENSION;
 801                                         }
 802                                         PopPosition ();
 803                                 }
 804
 805                                 // Save current position and parse next token.
 806                                 PushPosition ();
 807                                 bool is_generic_lt = parse_less_than ();
 808                                 PopPosition ();
 809
 810                                 if (is_generic_lt) {
 811                                         return Token.OP_GENERICS_LT;
 812                                 } else
 813                                         parsing_generic_less_than = 0;
 814
 815                                 d = peek_char ();
 816                                 if (d == '<'){
 817                                         get_char ();
 818                                         d = peek_char ();
 819
 820                                         if (d == '='){
 821                                                 doread = true;
 822                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 823                                         }
 824                                         return Token.OP_SHIFT_LEFT;
 825                                 } else if (d == '='){
 826                                         doread = true;
 827                                         return Token.OP_LE;
 828                                 }
 829                                 return Token.OP_LT;
 830                         } else if (c == '>') {
 831                                 if (parsing_generic_less_than > 0) {
 832                                         parsing_generic_less_than--;
 833                                         return Token.OP_GENERICS_GT;
 834                                 }
 835
 836                                 d = peek_char ();
 837                                 if (d == '>'){
 838                                         get_char ();
 839                                         d = peek_char ();
 840
 841                                         if (d == '='){
 842                                                 doread = true;
 843                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 844                                         }
 845                                         return Token.OP_SHIFT_RIGHT;
 846                                 } else if (d == '='){
 847                                         doread = true;
 848                                         return Token.OP_GE;
 849                                 }
 850                                 return Token.OP_GT;
 851                         }
 852 #endif
 853                         d = peek_char ();
 854                         if (c == '+'){
 855
 856                                 if (d == '+') {
 857                                         val = Location;
 858                                         t = Token.OP_INC;
 859                                 }
 860                                 else if (d == '=')
 861                                         t = Token.OP_ADD_ASSIGN;
 862                                 else {
 863                                         val = Location;
 864                                         return Token.PLUS;
 865                                 }
 866                                 doread = true;
 867                                 return t;
 868                         }
 869                         if (c == '-'){
 870                                 if (d == '-') {
 871                                         val = Location;
 872                                         t = Token.OP_DEC;
 873                                 }
 874                                 else if (d == '=')
 875                                         t = Token.OP_SUB_ASSIGN;
 876                                 else if (d == '>')
 877                                         t = Token.OP_PTR;
 878                                 else {
 879                                         val = Location;
 880                                         return Token.MINUS;
 881                                 }
 882                                 doread = true;
 883                                 return t;
 884                         }
 885
 886                         if (c == '!'){
 887                                 if (d == '='){
 888                                         doread = true;
 889                                         return Token.OP_NE;
 890                                 }
 891                                 val = Location;
 892                                 return Token.BANG;
 893                         }
 894
 895                         if (c == '='){
 896                                 if (d == '='){
 897                                         doread = true;
 898                                         return Token.OP_EQ;
 899                                 }
 900                                 if (d == '>'){
 901                                         doread = true;
 902                                         val = Location;
 903                                         return Token.ARROW;
 904                                 }
 905
 906                                 return Token.ASSIGN;
 907                         }
 908
 909                         if (c == '&'){
 910                                 if (d == '&'){
 911                                         doread = true;
 912                                         return Token.OP_AND;
 913                                 } else if (d == '='){
 914                                         doread = true;
 915                                         return Token.OP_AND_ASSIGN;
 916                                 }
 917                                 val = Location;
 918                                 return Token.BITWISE_AND;
 919                         }
 920
 921                         if (c == '|'){
 922                                 if (d == '|'){
 923                                         doread = true;
 924                                         return Token.OP_OR;
 925                                 } else if (d == '='){
 926                                         doread = true;
 927                                         return Token.OP_OR_ASSIGN;
 928                                 }
 929                                 return Token.BITWISE_OR;
 930                         }
 931
 932                         if (c == '*'){
 933                                 if (d == '='){
 934                                         doread = true;
 935                                         return Token.OP_MULT_ASSIGN;
 936                                 }
 937                                 val = Location;
 938                                 return Token.STAR;
 939                         }
 940
 941                         if (c == '/'){
 942                                 if (d == '='){
 943                                         doread = true;
 944                                         return Token.OP_DIV_ASSIGN;
 945                                 }
 946                                 return Token.DIV;
 947                         }
 948
 949                         if (c == '%'){
 950                                 if (d == '='){
 951                                         doread = true;
 952                                         return Token.OP_MOD_ASSIGN;
 953                                 }
 954                                 return Token.PERCENT;
 955                         }
 956
 957                         if (c == '^'){
 958                                 if (d == '='){
 959                                         doread = true;
 960                                         return Token.OP_XOR_ASSIGN;
 961                                 }
 962                                 return Token.CARRET;
 963                         }
 964
 965 #if !GMCS_SOURCE
 966                         if (c == '<'){
 967                                 if (d == '<'){
 968                                         get_char ();
 969                                         d = peek_char ();
 970
 971                                         if (d == '='){
 972                                                 doread = true;
 973                                                 return Token.OP_SHIFT_LEFT_ASSIGN;
 974                                         }
 975                                         return Token.OP_SHIFT_LEFT;
 976                                 } else if (d == '='){
 977                                         doread = true;
 978                                         return Token.OP_LE;
 979                                 }
 980                                 return Token.OP_LT;
 981                         }
 982
 983                         if (c == '>'){
 984                                 if (d == '>'){
 985                                         get_char ();
 986                                         d = peek_char ();
 987
 988                                         if (d == '='){
 989                                                 doread = true;
 990                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;
 991                                         }
 992                                         return Token.OP_SHIFT_RIGHT;
 993                                 } else if (d == '='){
 994                                         doread = true;
 995                                         return Token.OP_GE;
 996                                 }
 997                                 return Token.OP_GT;
 998                         }
 999 #endif
1000                         if (c == ':'){
1001                                 if (d == ':'){
1002                                         doread = true;
1003                                         return Token.DOUBLE_COLON;
1004                                 }
1005                                 val = Location;
1006                                 return Token.COLON;
1007                         }
1008
1009                         return Token.ERROR;
1010                 }
1011
1012                 int deambiguate_close_parens = 0;
1013
1014                 public void Deambiguate_CloseParens (object expression)
1015                 {
1016                         putback (')');
1017
1018                         // When any binary operation is used we are sure it is not a cast
1019                         if (expression is Binary)
1020                                 return;
1021
1022                         deambiguate_close_parens++;
1023                 }
1024
1025                 bool decimal_digits (int c)
1026                 {
1027                         int d;
1028                         bool seen_digits = false;
1029
1030                         if (c != -1){
1031                                 if (number_pos == max_number_size)
1032                                         Error_NumericConstantTooLong ();
1033                                 number_builder [number_pos++] = (char) c;
1034                         }
1035
1036                         //
1037                         // We use peek_char2, because decimal_digits needs to do a
1038                         // 2-character look-ahead (5.ToString for example).
1039                         //
1040                         while ((d = peek_char2 ()) != -1){
1041                                 if (d >= '0' && d <= '9'){
1042                                         if (number_pos == max_number_size)
1043                                                 Error_NumericConstantTooLong ();
1044                                         number_builder [number_pos++] = (char) d;
1045                                         get_char ();
1046                                         seen_digits = true;
1047                                 } else
1048                                         break;
1049                         }
1050
1051                         return seen_digits;
1052                 }
1053
1054                 static bool is_hex (int e)
1055                 {
1056                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1057                 }
1058
1059                 static int real_type_suffix (int c)
1060                 {
1061                         int t;
1062
1063                         switch (c){
1064                         case 'F': case 'f':
1065                                 t =  Token.LITERAL_FLOAT;
1066                                 break;
1067                         case 'D': case 'd':
1068                                 t = Token.LITERAL_DOUBLE;
1069                                 break;
1070                         case 'M': case 'm':
1071                                  t= Token.LITERAL_DECIMAL;
1072                                 break;
1073                         default:
1074                                 return Token.NONE;
1075                         }
1076                         return t;
1077                 }
1078
1079                 int integer_type_suffix (ulong ul, int c)
1080                 {
1081                         bool is_unsigned = false;
1082                         bool is_long = false;
1083
1084                         if (c != -1){
1085                                 bool scanning = true;
1086                                 do {
1087                                         switch (c){
1088                                         case 'U': case 'u':
1089                                                 if (is_unsigned)
1090                                                         scanning = false;
1091                                                 is_unsigned = true;
1092                                                 get_char ();
1093                                                 break;
1094
1095                                         case 'l':
1096                                                 if (!is_unsigned){
1097                                                         //
1098                                                         // if we have not seen anything in between
1099                                                         // report this error
1100                                                         //
1101                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1102                                                 }
1103                                                 //
1104                                                 // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
1105                                                 // work around that.
1106                                                 //
1107                                                 //goto case 'L';
1108                                                 if (is_long)
1109                                                         scanning = false;
1110                                                 is_long = true;
1111                                                 get_char ();
1112                                                 break;
1113
1114                                         case 'L':
1115                                                 if (is_long)
1116                                                         scanning = false;
1117                                                 is_long = true;
1118                                                 get_char ();
1119                                                 break;
1120
1121                                         default:
1122                                                 scanning = false;
1123                                                 break;
1124                                         }
1125                                         c = peek_char ();
1126                                 } while (scanning);
1127                         }
1128
1129                         if (is_long && is_unsigned){
1130                                 val = ul;
1131                                 return Token.LITERAL_INTEGER;
1132                         } else if (is_unsigned){
1133                                 // uint if possible, or ulong else.
1134
1135                                 if ((ul & 0xffffffff00000000) == 0)
1136                                         val = (uint) ul;
1137                                 else
1138                                         val = ul;
1139                         } else if (is_long){
1140                                 // long if possible, ulong otherwise
1141                                 if ((ul & 0x8000000000000000) != 0)
1142                                         val = ul;
1143                                 else
1144                                         val = (long) ul;
1145                         } else {
1146                                 // int, uint, long or ulong in that order
1147                                 if ((ul & 0xffffffff00000000) == 0){
1148                                         uint ui = (uint) ul;
1149
1150                                         if ((ui & 0x80000000) != 0)
1151                                                 val = ui;
1152                                         else
1153                                                 val = (int) ui;
1154                                 } else {
1155                                         if ((ul & 0x8000000000000000) != 0)
1156                                                 val = ul;
1157                                         else
1158                                                 val = (long) ul;
1159                                 }
1160                         }
1161                         return Token.LITERAL_INTEGER;
1162                 }
1163
1164                 //
1165                 // given `c' as the next char in the input decide whether
1166                 // we need to convert to a special type, and then choose
1167                 // the best representation for the integer
1168                 //
1169                 int adjust_int (int c)
1170                 {
1171                         try {
1172                                 if (number_pos > 9){
1173                                         ulong ul = (uint) (number_builder [0] - '0');
1174
1175                                         for (int i = 1; i < number_pos; i++){
1176                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1177                                         }
1178                                         return integer_type_suffix (ul, c);
1179                                 } else {
1180                                         uint ui = (uint) (number_builder [0] - '0');
1181
1182                                         for (int i = 1; i < number_pos; i++){
1183                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1184                                         }
1185                                         return integer_type_suffix (ui, c);
1186                                 }
1187                         } catch (OverflowException) {
1188                                 error_details = "Integral constant is too large";
1189                                 Report.Error (1021, Location, error_details);
1190                                 val = 0ul;
1191                                 return Token.LITERAL_INTEGER;
1192                         }
1193                         catch (FormatException) {
1194                                 Report.Error (1013, Location, "Invalid number");
1195                                 val = 0ul;
1196                                 return Token.LITERAL_INTEGER;
1197                         }
1198                 }
1199
1200                 int adjust_real (int t)
1201                 {
1202                         string s = new String (number_builder, 0, number_pos);
1203                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1204
1205                         switch (t){
1206                         case Token.LITERAL_DECIMAL:
1207                                 try {
1208                                         val = System.Decimal.Parse (s, styles, csharp_format_info);
1209                                 } catch (OverflowException) {
1210                                         val = 0m;
1211                                         Report.Error (594, Location, error_details, "decimal");
1212                                 }
1213                                 break;
1214                         case Token.LITERAL_FLOAT:
1215                                 try {
1216                                         val = float.Parse (s, styles, csharp_format_info);
1217                                 } catch (OverflowException) {
1218                                         val = 0.0f;
1219                                         Report.Error (594, Location, error_details, "float");
1220                                 }
1221                                 break;
1222
1223                         case Token.LITERAL_DOUBLE:
1224                         case Token.NONE:
1225                                 t = Token.LITERAL_DOUBLE;
1226                                 try {
1227                                         val = System.Double.Parse (s, styles, csharp_format_info);
1228                                 } catch (OverflowException) {
1229                                         val = 0.0;
1230                                         Report.Error (594, Location, error_details, "double");
1231                                 }
1232                                 break;
1233                         }
1234                         return t;
1235                 }
1236
1237                 int handle_hex ()
1238                 {
1239                         int d;
1240                         ulong ul;
1241
1242                         get_char ();
1243                         while ((d = peek_char ()) != -1){
1244                                 if (is_hex (d)){
1245                                         number_builder [number_pos++] = (char) d;
1246                                         get_char ();
1247                                 } else
1248                                         break;
1249                         }
1250
1251                         string s = new String (number_builder, 0, number_pos);
1252                         try {
1253                                 if (number_pos <= 8)
1254                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1255                                 else
1256                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1257                         } catch (OverflowException){
1258                                 error_details = "Integral constant is too large";
1259                                 Report.Error (1021, Location, error_details);
1260                                 val = 0ul;
1261                                 return Token.LITERAL_INTEGER;
1262                         }
1263                         catch (FormatException) {
1264                                 Report.Error (1013, Location, "Invalid number");
1265                                 val = 0ul;
1266                                 return Token.LITERAL_INTEGER;
1267                         }
1268
1269                         return integer_type_suffix (ul, peek_char ());
1270                 }
1271
1272                 //
1273                 // Invoked if we know we have .digits or digits
1274                 //
1275                 int is_number (int c)
1276                 {
1277                         bool is_real = false;
1278                         int type;
1279
1280                         number_pos = 0;
1281
1282                         if (c >= '0' && c <= '9'){
1283                                 if (c == '0'){
1284                                         int peek = peek_char ();
1285
1286                                         if (peek == 'x' || peek == 'X')
1287                                                 return handle_hex ();
1288                                 }
1289                                 decimal_digits (c);
1290                                 c = get_char ();
1291                         }
1292
1293                         //
1294                         // We need to handle the case of
1295                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1296                         //
1297                         if (c == '.'){
1298                                 if (decimal_digits ('.')){
1299                                         is_real = true;
1300                                         c = get_char ();
1301                                 } else {
1302                                         putback ('.');
1303                                         number_pos--;
1304                                         return adjust_int (-1);
1305                                 }
1306                         }
1307
1308                         if (c == 'e' || c == 'E'){
1309                                 is_real = true;
1310                                 if (number_pos == max_number_size)
1311                                         Error_NumericConstantTooLong ();
1312                                 number_builder [number_pos++] = 'e';
1313                                 c = get_char ();
1314
1315                                 if (c == '+'){
1316                                         if (number_pos == max_number_size)
1317                                                 Error_NumericConstantTooLong ();
1318                                         number_builder [number_pos++] = '+';
1319                                         c = -1;
1320                                 } else if (c == '-') {
1321                                         if (number_pos == max_number_size)
1322                                                 Error_NumericConstantTooLong ();
1323                                         number_builder [number_pos++] = '-';
1324                                         c = -1;
1325                                 } else {
1326                                         if (number_pos == max_number_size)
1327                                                 Error_NumericConstantTooLong ();
1328                                         number_builder [number_pos++] = '+';
1329                                 }
1330
1331                                 decimal_digits (c);
1332                                 c = get_char ();
1333                         }
1334
1335                         type = real_type_suffix (c);
1336                         if (type == Token.NONE && !is_real){
1337                                 putback (c);
1338                                 return adjust_int (c);
1339                         } else
1340                                 is_real = true;
1341
1342                         if (type == Token.NONE){
1343                                 putback (c);
1344                         }
1345
1346                         if (is_real)
1347                                 return adjust_real (type);
1348
1349                         Console.WriteLine ("This should not be reached");
1350                         throw new Exception ("Is Number should never reach this point");
1351                 }
1352
1353                 //
1354                 // Accepts exactly count (4 or 8) hex, no more no less
1355                 //
1356                 int getHex (int count, out bool error)
1357                 {
1358                         int i;
1359                         int total = 0;
1360                         int c;
1361                         int top = count != -1 ? count : 4;
1362
1363                         get_char ();
1364                         error = false;
1365                         for (i = 0; i < top; i++){
1366                                 c = get_char ();
1367
1368                                 if (c >= '0' && c <= '9')
1369                                         c = (int) c - (int) '0';
1370                                 else if (c >= 'A' && c <= 'F')
1371                                         c = (int) c - (int) 'A' + 10;
1372                                 else if (c >= 'a' && c <= 'f')
1373                                         c = (int) c - (int) 'a' + 10;
1374                                 else {
1375                                         error = true;
1376                                         return 0;
1377                                 }
1378
1379                                 total = (total * 16) + c;
1380                                 if (count == -1){
1381                                         int p = peek_char ();
1382                                         if (p == -1)
1383                                                 break;
1384                                         if (!is_hex ((char)p))
1385                                                 break;
1386                                 }
1387                         }
1388                         return total;
1389                 }
1390
1391                 int escape (int c)
1392                 {
1393                         bool error;
1394                         int d;
1395                         int v;
1396
1397                         d = peek_char ();
1398                         if (c != '\\')
1399                                 return c;
1400
1401                         switch (d){
1402                         case 'a':
1403                                 v = '\a'; break;
1404                         case 'b':
1405                                 v = '\b'; break;
1406                         case 'n':
1407                                 v = '\n'; break;
1408                         case 't':
1409                                 v = '\t'; break;
1410                         case 'v':
1411                                 v = '\v'; break;
1412                         case 'r':
1413                                 v = '\r'; break;
1414                         case '\\':
1415                                 v = '\\'; break;
1416                         case 'f':
1417                                 v = '\f'; break;
1418                         case '0':
1419                                 v = 0; break;
1420                         case '"':
1421                                 v = '"'; break;
1422                         case '\'':
1423                                 v = '\''; break;
1424                         case 'x':
1425                                 v = getHex (-1, out error);
1426                                 if (error)
1427                                         goto default;
1428                                 return v;
1429                         case 'u':
1430                                 v = getHex (4, out error);
1431                                 if (error)
1432                                         goto default;
1433                                 return v;
1434                         case 'U':
1435                                 v = getHex (8, out error);
1436                                 if (error)
1437                                         goto default;
1438                                 return v;
1439                         default:
1440                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1441                                 return d;
1442                         }
1443                         get_char ();
1444                         return v;
1445                 }
1446
1447                 int get_char ()
1448                 {
1449                         int x;
1450                         if (putback_char != -1) {
1451                                 x = putback_char;
1452                                 putback_char = -1;
1453                         } else
1454                                 x = reader.Read ();
1455                         if (x == '\n') {
1456                                 line++;
1457                                 ref_line++;
1458                                 previous_col = col;
1459                                 col = 0;
1460                         }
1461                         else
1462                                 col++;
1463                         return x;
1464                 }
1465
1466                 int peek_char ()
1467                 {
1468                         if (putback_char != -1)
1469                                 return putback_char;
1470                         putback_char = reader.Read ();
1471                         return putback_char;
1472                 }
1473
1474                 int peek_char2 ()
1475                 {
1476                         if (putback_char != -1)
1477                                 return putback_char;
1478                         return reader.Peek ();
1479                 }
1480
1481                 void putback (int c)
1482                 {
1483                         if (putback_char != -1){
1484                                 Console.WriteLine ("Col: " + col);
1485                                 Console.WriteLine ("Row: " + line);
1486                                 Console.WriteLine ("Name: " + ref_name.Name);
1487                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1488                                 throw new Exception ("This should not happen putback on putback");
1489                         }
1490                         if (c == '\n' || col == 0) {
1491                                 // It won't happen though.
1492                                 line--;
1493                                 ref_line--;
1494                                 col = previous_col;
1495                         }
1496                         else
1497                                 col--;
1498                         putback_char = c;
1499                 }
1500
1501                 public bool advance ()
1502                 {
1503                         return peek_char () != -1;
1504                 }
1505
1506                 public Object Value {
1507                         get {
1508                                 return val;
1509                         }
1510                 }
1511
1512                 public Object value ()
1513                 {
1514                         return val;
1515                 }
1516
1517                 static bool IsCastToken (int token)
1518                 {
1519                         switch (token) {
1520                         case Token.BANG:
1521                         case Token.TILDE:
1522                         case Token.IDENTIFIER:
1523                         case Token.LITERAL_INTEGER:
1524                         case Token.LITERAL_FLOAT:
1525                         case Token.LITERAL_DOUBLE:
1526                         case Token.LITERAL_DECIMAL:
1527                         case Token.LITERAL_CHARACTER:
1528                         case Token.LITERAL_STRING:
1529                         case Token.BASE:
1530                         case Token.CHECKED:
1531                         case Token.DELEGATE:
1532                         case Token.FALSE:
1533                         case Token.FIXED:
1534                         case Token.NEW:
1535                         case Token.NULL:
1536                         case Token.SIZEOF:
1537                         case Token.THIS:
1538                         case Token.THROW:
1539                         case Token.TRUE:
1540                         case Token.TYPEOF:
1541                         case Token.UNCHECKED:
1542                         case Token.UNSAFE:
1543 #if GMCS_SOURCE
1544                         case Token.DEFAULT:
1545 #endif
1546
1547                                 //
1548                                 // These can be part of a member access
1549                                 //
1550                         case Token.INT:
1551                         case Token.UINT:
1552                         case Token.SHORT:
1553                         case Token.USHORT:
1554                         case Token.LONG:
1555                         case Token.ULONG:
1556                         case Token.DOUBLE:
1557                         case Token.FLOAT:
1558                         case Token.CHAR:
1559                                 return true;
1560
1561                         default:
1562                                 return false;
1563                         }
1564                 }
1565
1566                 public int token ()
1567                 {
1568                         current_token = xtoken ();
1569
1570 #if GMCS_SOURCE
1571                         if (current_token != Token.DEFAULT)
1572                                 return current_token;
1573
1574                         PushPosition();
1575                         int c = xtoken();
1576                         if (c == -1)
1577                                 current_token = Token.ERROR;
1578                         else if (c == Token.OPEN_PARENS)
1579                                 current_token = Token.DEFAULT_OPEN_PARENS;
1580                         else if (c == Token.COLON)
1581                                 current_token = Token.DEFAULT_COLON;
1582                         else
1583                                 PopPosition();
1584 #endif
1585                         return current_token;
1586                 }
1587
1588                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1589
1590                 void get_cmd_arg (out string cmd, out string arg)
1591                 {
1592                         int c;
1593
1594                         tokens_seen = false;
1595                         arg = "";
1596                         static_cmd_arg.Length = 0;
1597
1598                         // skip over white space
1599                         while ((c = get_char ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1600                                 ;
1601
1602                         while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
1603                                 if (is_identifier_part_character ((char) c)){
1604                                         static_cmd_arg.Append ((char) c);
1605                                         c = get_char ();
1606                                 } else {
1607                                         putback (c);
1608                                         break;
1609                                 }
1610                         }
1611
1612                         cmd = static_cmd_arg.ToString ();
1613
1614                         if (c == '\n' || c == '\r'){
1615                                 return;
1616                         }
1617
1618                         // skip over white space
1619                         while ((c = get_char ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1620                                 ;
1621
1622                         if (c == '\n'){
1623                                 return;
1624                         } else if (c == '\r'){
1625                                 return;
1626                         } else if (c == -1){
1627                                 arg = "";
1628                                 return;
1629                         }
1630
1631                         static_cmd_arg.Length = 0;
1632                         static_cmd_arg.Append ((char) c);
1633
1634                         while ((c = get_char ()) != -1 && (c != '\n') && (c != '\r')){
1635                                 static_cmd_arg.Append ((char) c);
1636                         }
1637
1638                         arg = static_cmd_arg.ToString ();
1639                 }
1640
1641                 //
1642                 // Handles the #line directive
1643                 //
1644                 bool PreProcessLine (string arg)
1645                 {
1646                         if (arg.Length == 0)
1647                                 return false;
1648
1649                         if (arg == "default"){
1650                                 ref_line = line;
1651                                 ref_name = file_name;
1652                                 Location.Push (ref_name);
1653                                 return true;
1654                         } else if (arg == "hidden"){
1655                                 //
1656                                 // We ignore #line hidden
1657                                 //
1658                                 return true;
1659                         }
1660
1661                         try {
1662                                 int pos;
1663
1664                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1665                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1666                                         pos++;
1667
1668                                         char [] quotes = { '\"' };
1669
1670                                         string name = arg.Substring (pos). Trim (quotes);
1671                                         ref_name = Location.LookupFile (name);
1672                                         file_name.HasLineDirective = true;
1673                                         ref_name.HasLineDirective = true;
1674                                         Location.Push (ref_name);
1675                                 } else {
1676                                         ref_line = System.Int32.Parse (arg);
1677                                 }
1678                         } catch {
1679                                 return false;
1680                         }
1681
1682                         return true;
1683                 }
1684
1685                 //
1686                 // Handles #define and #undef
1687                 //
1688                 void PreProcessDefinition (bool is_define, string arg, bool caller_is_taking)
1689                 {
1690                         if (arg.Length == 0 || arg == "true" || arg == "false"){
1691                                 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
1692                                 return;
1693                         }
1694
1695                         if (arg.IndexOfAny (simple_whitespaces) != -1){
1696                                 Error_EndLineExpected ();
1697                                 return;
1698                         }
1699
1700                         if (!is_identifier_start_character (arg [0]))
1701                                 Report.Error (1001, Location, "Identifier expected: " + arg);
1702
1703                         foreach (char c in arg.Substring (1)){
1704                                 if (!is_identifier_part_character (c)){
1705                                         Report.Error (1001, Location, "Identifier expected: " + arg);
1706                                         return;
1707                                 }
1708                         }
1709
1710                         if (!caller_is_taking)
1711                                 return;
1712
1713                         if (is_define){
1714                                 if (defines == null)
1715                                         defines = new Hashtable ();
1716                                 define (arg);
1717                         } else {
1718                                 if (defines == null)
1719                                         return;
1720                                 if (defines.Contains (arg))
1721                                         defines.Remove (arg);
1722                         }
1723                 }
1724
1725                 /// <summary>
1726                 /// Handles #pragma directive
1727                 /// </summary>
1728                 void PreProcessPragma (string arg)
1729                 {
1730                         const string warning = "warning";
1731                         const string w_disable = "warning disable";
1732                         const string w_restore = "warning restore";
1733
1734                         if (arg == w_disable) {
1735                                 Report.RegisterWarningRegion (Location).WarningDisable (line);
1736                                 return;
1737                         }
1738
1739                         if (arg == w_restore) {
1740                                 Report.RegisterWarningRegion (Location).WarningEnable (line);
1741                                 return;
1742                         }
1743
1744                         if (arg.StartsWith (w_disable)) {
1745                                 int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
1746                                 foreach (int code in codes) {
1747                                         if (code != 0)
1748                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1749                                 }
1750                                 return;
1751                         }
1752
1753                         if (arg.StartsWith (w_restore)) {
1754                                 int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
1755                                 Hashtable w_table = Report.warning_ignore_table;
1756                                 foreach (int code in codes) {
1757                                         if (w_table != null && w_table.Contains (code))
1758                                                 Report.Warning (1635, 1, Location, String.Format ("Cannot restore warning `CS{0:0000}' because it was disabled globally", code));
1759                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1760                                 }
1761                                 return;
1762                         }
1763
1764                         if (arg.StartsWith (warning)) {
1765                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
1766                                 return;
1767                         }
1768
1769                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
1770                 }
1771
1772                 int[] ParseNumbers (string text)
1773                 {
1774                         string[] string_array = text.Split (',');
1775                         int[] values = new int [string_array.Length];
1776                         int index = 0;
1777                         foreach (string string_code in string_array) {
1778                                 try {
1779                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1780                                 }
1781                                 catch (FormatException) {
1782                                         Report.Warning (1692, 1, Location, "Invalid number");
1783                                 }
1784                         }
1785                         return values;
1786                 }
1787
1788                 bool eval_val (string s)
1789                 {
1790                         if (s == "true")
1791                                 return true;
1792                         if (s == "false")
1793                                 return false;
1794
1795                         if (defines == null)
1796                                 return false;
1797                         if (defines.Contains (s))
1798                                 return true;
1799
1800                         return false;
1801                 }
1802
1803                 bool pp_primary (ref string s)
1804                 {
1805                         s = s.Trim ();
1806                         int len = s.Length;
1807
1808                         if (len > 0){
1809                                 char c = s [0];
1810
1811                                 if (c == '('){
1812                                         s = s.Substring (1);
1813                                         bool val = pp_expr (ref s, false);
1814                                         if (s.Length > 0 && s [0] == ')'){
1815                                                 s = s.Substring (1);
1816                                                 return val;
1817                                         }
1818                                         Error_InvalidDirective ();
1819                                         return false;
1820                                 }
1821
1822                                 if (is_identifier_start_character (c)){
1823                                         int j = 1;
1824
1825                                         while (j < len){
1826                                                 c = s [j];
1827
1828                                                 if (is_identifier_part_character (c)){
1829                                                         j++;
1830                                                         continue;
1831                                                 }
1832                                                 bool v = eval_val (s.Substring (0, j));
1833                                                 s = s.Substring (j);
1834                                                 return v;
1835                                         }
1836                                         bool vv = eval_val (s);
1837                                         s = "";
1838                                         return vv;
1839                                 }
1840                         }
1841                         Error_InvalidDirective ();
1842                         return false;
1843                 }
1844
1845                 bool pp_unary (ref string s)
1846                 {
1847                         s = s.Trim ();
1848                         int len = s.Length;
1849
1850                         if (len > 0){
1851                                 if (s [0] == '!'){
1852                                         if (len > 1 && s [1] == '='){
1853                                                 Error_InvalidDirective ();
1854                                                 return false;
1855                                         }
1856                                         s = s.Substring (1);
1857                                         return ! pp_primary (ref s);
1858                                 } else
1859                                         return pp_primary (ref s);
1860                         } else {
1861                                 Error_InvalidDirective ();
1862                                 return false;
1863                         }
1864                 }
1865
1866                 bool pp_eq (ref string s)
1867                 {
1868                         bool va = pp_unary (ref s);
1869
1870                         s = s.Trim ();
1871                         int len = s.Length;
1872                         if (len > 0){
1873                                 if (s [0] == '='){
1874                                         if (len > 2 && s [1] == '='){
1875                                                 s = s.Substring (2);
1876                                                 return va == pp_unary (ref s);
1877                                         } else {
1878                                                 Error_InvalidDirective ();
1879                                                 return false;
1880                                         }
1881                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
1882                                         s = s.Substring (2);
1883
1884                                         return va != pp_unary (ref s);
1885
1886                                 }
1887                         }
1888
1889                         return va;
1890
1891                 }
1892
1893                 bool pp_and (ref string s)
1894                 {
1895                         bool va = pp_eq (ref s);
1896
1897                         s = s.Trim ();
1898                         int len = s.Length;
1899                         if (len > 0){
1900                                 if (s [0] == '&'){
1901                                         if (len > 2 && s [1] == '&'){
1902                                                 s = s.Substring (2);
1903                                                 return (va & pp_and (ref s));
1904                                         } else {
1905                                                 Error_InvalidDirective ();
1906                                                 return false;
1907                                         }
1908                                 }
1909                         }
1910                         return va;
1911                 }
1912
1913                 //
1914                 // Evaluates an expression for `#if' or `#elif'
1915                 //
1916                 bool pp_expr (ref string s, bool isTerm)
1917                 {
1918                         bool va = pp_and (ref s);
1919                         s = s.Trim ();
1920                         int len = s.Length;
1921                         if (len > 0){
1922                                 char c = s [0];
1923
1924                                 if (c == '|'){
1925                                         if (len > 2 && s [1] == '|'){
1926                                                 s = s.Substring (2);
1927                                                 return va | pp_expr (ref s, isTerm);
1928                                         } else {
1929                                                 Error_InvalidDirective ();
1930                                                 return false;
1931                                         }
1932                                 }
1933                                 if (isTerm) {
1934                                         Error_EndLineExpected ();
1935                                         return false;
1936                                 }
1937                         }
1938
1939                         return va;
1940                 }
1941
1942                 bool eval (string s)
1943                 {
1944                         bool v = pp_expr (ref s, true);
1945                         s = s.Trim ();
1946                         if (s.Length != 0){
1947                                 return false;
1948                         }
1949
1950                         return v;
1951                 }
1952
1953                 void Error_NumericConstantTooLong ()
1954                 {
1955                         Report.Error (1021, Location, "Numeric constant too long");
1956                 }
1957
1958                 void Error_InvalidDirective ()
1959                 {
1960                         Report.Error (1517, Location, "Invalid preprocessor directive");
1961                 }
1962
1963                 void Error_UnexpectedDirective (string extra)
1964                 {
1965                         Report.Error (
1966                                 1028, Location,
1967                                 "Unexpected processor directive (" + extra + ")");
1968                 }
1969
1970                 void Error_TokenExpected (string token)
1971                 {
1972                         Report.Error (1026, Location, "Expecting `{0}'", token);
1973                 }
1974
1975                 void Error_TokensSeen ()
1976                 {
1977                         Report.Error (1032, Location,
1978                                 "Cannot define or undefine preprocessor symbols after first token in file");
1979                 }
1980
1981                 void Eror_WrongPreprocessorLocation ()
1982                 {
1983                         Report.Error (1040, Location,
1984                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
1985                 }
1986
1987                 void Error_EndLineExpected ()
1988                 {
1989                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
1990                 }
1991
1992                 //
1993                 // if true, then the code continues processing the code
1994                 // if false, the code stays in a loop until another directive is
1995                 // reached.
1996                 // When caller_is_taking is false we ignore all directives except the ones
1997                 // which can help us to identify where the #if block ends
1998                 bool handle_preprocessing_directive (bool caller_is_taking)
1999                 {
2000                         string cmd, arg;
2001                         bool region_directive = false;
2002
2003                         get_cmd_arg (out cmd, out arg);
2004
2005                         // Eat any trailing whitespaces and single-line comments
2006                         if (arg.IndexOf ("//") != -1)
2007                                 arg = arg.Substring (0, arg.IndexOf ("//"));
2008                         arg = arg.Trim (simple_whitespaces);
2009
2010                         //
2011                         // The first group of pre-processing instructions is always processed
2012                         //
2013                         switch (cmd){
2014                         case "region":
2015                                 region_directive = true;
2016                                 arg = "true";
2017                                 goto case "if";
2018
2019                         case "endregion":
2020                                 if (ifstack == null || ifstack.Count == 0){
2021                                         Error_UnexpectedDirective ("no #region for this #endregion");
2022                                         return true;
2023                                 }
2024                                 int pop = (int) ifstack.Pop ();
2025
2026                                 if ((pop & REGION) == 0)
2027                                         Report.Error (1027, Location, "Expected `#endif' directive");
2028
2029                                 return caller_is_taking;
2030
2031                         case "if":
2032                                 if (ifstack == null)
2033                                         ifstack = new Stack (2);
2034
2035                                 int flags = region_directive ? REGION : 0;
2036                                 if (ifstack.Count == 0){
2037                                         flags |= PARENT_TAKING;
2038                                 } else {
2039                                         int state = (int) ifstack.Peek ();
2040                                         if ((state & TAKING) != 0) {
2041                                                 flags |= PARENT_TAKING;
2042                                         }
2043                                 }
2044
2045                                 if (caller_is_taking && eval (arg)) {
2046                                         ifstack.Push (flags | TAKING);
2047                                         return true;
2048                                 }
2049                                 ifstack.Push (flags);
2050                                 return false;
2051
2052                         case "endif":
2053                                 if (ifstack == null || ifstack.Count == 0){
2054                                         Error_UnexpectedDirective ("no #if for this #endif");
2055                                         return true;
2056                                 } else {
2057                                         pop = (int) ifstack.Pop ();
2058
2059                                         if ((pop & REGION) != 0)
2060                                                 Report.Error (1038, Location, "#endregion directive expected");
2061
2062                                         if (arg.Length != 0) {
2063                                                 Error_EndLineExpected ();
2064                                         }
2065
2066                                         if (ifstack.Count == 0)
2067                                                 return true;
2068
2069                                         int state = (int) ifstack.Peek ();
2070                                         return (state & TAKING) != 0;
2071                                 }
2072
2073                         case "elif":
2074                                 if (ifstack == null || ifstack.Count == 0){
2075                                         Error_UnexpectedDirective ("no #if for this #elif");
2076                                         return true;
2077                                 } else {
2078                                         int state = (int) ifstack.Pop ();
2079
2080                                         if ((state & REGION) != 0) {
2081                                                 Report.Error (1038, Location, "#endregion directive expected");
2082                                                 return true;
2083                                         }
2084
2085                                         if ((state & ELSE_SEEN) != 0){
2086                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2087                                                 return true;
2088                                         }
2089
2090                                         if ((state & TAKING) != 0) {
2091                                                 ifstack.Push (0);
2092                                                 return false;
2093                                         }
2094
2095                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2096                                                 ifstack.Push (state | TAKING);
2097                                                 return true;
2098                                         }
2099
2100                                         ifstack.Push (state);
2101                                         return false;
2102                                 }
2103
2104                         case "else":
2105                                 if (ifstack == null || ifstack.Count == 0){
2106                                         Error_UnexpectedDirective ("no #if for this #else");
2107                                         return true;
2108                                 } else {
2109                                         int state = (int) ifstack.Peek ();
2110
2111                                         if ((state & REGION) != 0) {
2112                                                 Report.Error (1038, Location, "#endregion directive expected");
2113                                                 return true;
2114                                         }
2115
2116                                         if ((state & ELSE_SEEN) != 0){
2117                                                 Error_UnexpectedDirective ("#else within #else");
2118                                                 return true;
2119                                         }
2120
2121                                         ifstack.Pop ();
2122
2123                                         if (arg.Length != 0) {
2124                                                 Error_EndLineExpected ();
2125                                                 return true;
2126                                         }
2127
2128                                         bool ret = false;
2129                                         if ((state & PARENT_TAKING) != 0) {
2130                                                 ret = (state & TAKING) == 0;
2131
2132                                                 if (ret)
2133                                                         state |= TAKING;
2134                                                 else
2135                                                         state &= ~TAKING;
2136                                         }
2137
2138                                         ifstack.Push (state | ELSE_SEEN);
2139
2140                                         return ret;
2141                                 }
2142                                 case "define":
2143                                         if (any_token_seen){
2144                                                 Error_TokensSeen ();
2145                                                 return caller_is_taking;
2146                                         }
2147                                         PreProcessDefinition (true, arg, caller_is_taking);
2148                                         return caller_is_taking;
2149
2150                                 case "undef":
2151                                         if (any_token_seen){
2152                                                 Error_TokensSeen ();
2153                                                 return caller_is_taking;
2154                                         }
2155                                         PreProcessDefinition (false, arg, caller_is_taking);
2156                                         return caller_is_taking;
2157                         }
2158
2159                         //
2160                         // These are only processed if we are in a `taking' block
2161                         //
2162                         if (!caller_is_taking)
2163                                 return false;
2164
2165                         switch (cmd){
2166                         case "error":
2167                                 Report.Error (1029, Location, "#error: '" + arg + "'");
2168                                 return true;
2169
2170                         case "warning":
2171                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2172                                 return true;
2173
2174                         case "pragma":
2175                                 if (RootContext.Version == LanguageVersion.ISO_1) {
2176                                         Report.FeatureIsNotISO1 (Location, "#pragma");
2177                                         return true;
2178                                 }
2179
2180                                 PreProcessPragma (arg);
2181                                 return true;
2182
2183                         case "line":
2184                                 if (!PreProcessLine (arg))
2185                                         Report.Error (
2186                                                 1576, Location,
2187                                                 "The line number specified for #line directive is missing or invalid");
2188                                 return caller_is_taking;
2189                         }
2190
2191                         Report.Error (1024, Location, "Wrong preprocessor directive");
2192                         return true;
2193
2194                 }
2195
2196                 private int consume_string (bool quoted)
2197                 {
2198                         int c;
2199                         string_builder.Length = 0;
2200
2201                         while ((c = get_char ()) != -1){
2202                                 if (c == '"'){
2203                                         if (quoted && peek_char () == '"'){
2204                                                 string_builder.Append ((char) c);
2205                                                 get_char ();
2206                                                 continue;
2207                                         } else {
2208                                                 val = string_builder.ToString ();
2209                                                 return Token.LITERAL_STRING;
2210                                         }
2211                                 }
2212
2213                                 if (c == '\n'){
2214                                         if (!quoted)
2215                                                 Report.Error (1010, Location, "Newline in constant");
2216                                 }
2217
2218                                 if (!quoted){
2219                                         c = escape (c);
2220                                         if (c == -1)
2221                                                 return Token.ERROR;
2222                                 }
2223                                 string_builder.Append ((char) c);
2224                         }
2225
2226                         Report.Error (1039, Location, "Unterminated string literal");
2227                         return Token.EOF;
2228                 }
2229
2230                 private int consume_identifier (int s)
2231                 {
2232                         int res = consume_identifier (s, false);
2233
2234                         if (doc_state == XmlCommentState.Allowed)
2235                                 doc_state = XmlCommentState.NotAllowed;
2236                         switch (res) {
2237                         case Token.USING:
2238                         case Token.NAMESPACE:
2239                                 check_incorrect_doc_comment ();
2240                                 break;
2241                         }
2242
2243                         if (res == Token.PARTIAL) {
2244                                 // Save current position and parse next token.
2245                                 PushPosition ();
2246
2247                                 int next_token = token ();
2248                                 bool ok = (next_token == Token.CLASS) ||
2249                                         (next_token == Token.STRUCT) ||
2250                                         (next_token == Token.INTERFACE);
2251
2252                                 PopPosition ();
2253
2254                                 if (ok) {
2255                                         if (RootContext.Version == LanguageVersion.ISO_1)
2256                                                 Report.FeatureIsNotISO1 (Location, "partial types");
2257
2258                                         return res;
2259                                 }
2260
2261                                 if (next_token < Token.LAST_KEYWORD)
2262                                         Report.Error (267, Location, "The `partial' modifier can be used only immediately before keyword `class', `struct', or `interface'");
2263
2264                                 val = new LocatedToken (Location, "partial");
2265                                 return Token.IDENTIFIER;
2266                         }
2267
2268                         return res;
2269                 }
2270
2271                 private int consume_identifier (int s, bool quoted)
2272                 {
2273                         int pos = 1;
2274                         int c = -1;
2275
2276                         id_builder [0] = (char) s;
2277
2278                         current_location = new Location (ref_line, Col);
2279
2280                         while ((c = get_char ()) != -1) {
2281                         loop:
2282                                 if (is_identifier_part_character ((char) c)){
2283                                         if (pos == max_id_size){
2284                                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2285                                                 return Token.ERROR;
2286                                         }
2287
2288                                         id_builder [pos++] = (char) c;
2289 //                                      putback_char = -1;
2290                                 } else if (c == '\\') {
2291                                         c = escape (c);
2292                                         goto loop;
2293                                 } else {
2294 //                                      putback_char = c;
2295                                         putback (c);
2296                                         break;
2297                                 }
2298                         }
2299
2300                         //
2301                         // Optimization: avoids doing the keyword lookup
2302                         // on uppercase letters and _
2303                         //
2304                         if (!quoted && (s >= 'a' || s == '_')){
2305                                 int keyword = GetKeyword (id_builder, pos);
2306                                 if (keyword != -1) {
2307                                         val = Location;
2308                                         return keyword;
2309                                 }
2310                         }
2311
2312                         //
2313                         // Keep identifiers in an array of hashtables to avoid needless
2314                         // allocations
2315                         //
2316
2317                         if (identifiers [pos] != null) {
2318                                 val = identifiers [pos][id_builder];
2319                                 if (val != null) {
2320                                         val = new LocatedToken (Location, (string) val);
2321                                         if (quoted)
2322                                                 escaped_identifiers.Add (val);
2323                                         return Token.IDENTIFIER;
2324                                 }
2325                         }
2326                         else
2327                                 identifiers [pos] = new CharArrayHashtable (pos);
2328
2329                         val = new String (id_builder, 0, pos);
2330                         if (RootContext.Version == LanguageVersion.ISO_1) {
2331                                 for (int i = 1; i < id_builder.Length; i += 3) {
2332                                         if (id_builder [i] == '_' && (id_builder [i - 1] == '_' || id_builder [i + 1] == '_')) {
2333                                                 Report.Error (1638, Location,
2334                                                         "`{0}': Any identifier with double underscores cannot be used when ISO language version mode is specified", val.ToString ());
2335                                                 break;
2336                                         }
2337                                 }
2338                         }
2339
2340                         char [] chars = new char [pos];
2341                         Array.Copy (id_builder, chars, pos);
2342
2343                         identifiers [pos] [chars] = val;
2344
2345                         val = new LocatedToken (Location, (string) val);
2346                         if (quoted)
2347                                 escaped_identifiers.Add (val);
2348                         return Token.IDENTIFIER;
2349                 }
2350
2351                 public int xtoken ()
2352                 {
2353                         int t;
2354                         bool doread = false;
2355                         int c;
2356
2357                         // Whether we have seen comments on the current line
2358                         bool comments_seen = false;
2359                         val = null;
2360                         for (;(c = get_char ()) != -1;) {
2361                                 if (c == '\t'){
2362                                         col = ((col + 8) / 8) * 8;
2363                                         continue;
2364                                 }
2365
2366                                 if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0 || c == 0)
2367                                         continue;
2368
2369                                 if (c == '\r') {
2370                                         if (peek_char () == '\n')
2371                                                 get_char ();
2372
2373                                         any_token_seen |= tokens_seen;
2374                                         tokens_seen = false;
2375                                         comments_seen = false;
2376                                         continue;
2377                                 }
2378
2379                                 // Handle double-slash comments.
2380                                 if (c == '/'){
2381                                         int d = peek_char ();
2382
2383                                         if (d == '/'){
2384                                                 get_char ();
2385                                                 if (RootContext.Documentation != null && peek_char () == '/') {
2386                                                         get_char ();
2387                                                         // Don't allow ////.
2388                                                         if ((d = peek_char ()) != '/') {
2389                                                                 update_comment_location ();
2390                                                                 if (doc_state == XmlCommentState.Allowed)
2391                                                                         handle_one_line_xml_comment ();
2392                                                                 else if (doc_state == XmlCommentState.NotAllowed)
2393                                                                         warn_incorrect_doc_comment ();
2394                                                         }
2395                                                 }
2396                                                 while ((d = get_char ()) != -1 && (d != '\n') && d != '\r')
2397                                                         if (d == '\n'){
2398                                                         }
2399                                                 any_token_seen |= tokens_seen;
2400                                                 tokens_seen = false;
2401                                                 comments_seen = false;
2402                                                 continue;
2403                                         } else if (d == '*'){
2404                                                 get_char ();
2405                                                 bool docAppend = false;
2406                                                 if (RootContext.Documentation != null && peek_char () == '*') {
2407                                                         get_char ();
2408                                                         update_comment_location ();
2409                                                         // But when it is /**/, just do nothing.
2410                                                         if (peek_char () == '/') {
2411                                                                 get_char ();
2412                                                                 continue;
2413                                                         }
2414                                                         if (doc_state == XmlCommentState.Allowed)
2415                                                                 docAppend = true;
2416                                                         else if (doc_state == XmlCommentState.NotAllowed)
2417                                                                 warn_incorrect_doc_comment ();
2418                                                 }
2419
2420                                                 int current_comment_start = 0;
2421                                                 if (docAppend) {
2422                                                         current_comment_start = xml_comment_buffer.Length;
2423                                                         xml_comment_buffer.Append (Environment.NewLine);
2424                                                 }
2425
2426                                                 Location start_location = Location;
2427
2428                                                 while ((d = get_char ()) != -1){
2429                                                         if (d == '*' && peek_char () == '/'){
2430                                                                 get_char ();
2431                                                                 comments_seen = true;
2432                                                                 break;
2433                                                         }
2434                                                         if (docAppend)
2435                                                                 xml_comment_buffer.Append ((char) d);
2436
2437                                                         if (d == '\n'){
2438                                                                 any_token_seen |= tokens_seen;
2439                                                                 tokens_seen = false;
2440                                                                 //
2441                                                                 // Reset 'comments_seen' just to be consistent.
2442                                                                 // It doesn't matter either way, here.
2443                                                                 //
2444                                                                 comments_seen = false;
2445                                                         }
2446                                                 }
2447                                                 if (!comments_seen)
2448                                                         Report.Error (1035, start_location, "End-of-file found, '*/' expected");
2449
2450                                                 if (docAppend)
2451                                                         update_formatted_doc_comment (current_comment_start);
2452                                                 continue;
2453                                         }
2454                                         goto is_punct_label;
2455                                 }
2456
2457
2458                                 if (c == '\\' || is_identifier_start_character ((char)c)){
2459                                         tokens_seen = true;
2460                                         return consume_identifier (c);
2461                                 }
2462
2463                         is_punct_label:
2464                                 current_location = new Location (ref_line, Col);
2465                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
2466                                         tokens_seen = true;
2467                                         if (doread){
2468                                                 get_char ();
2469                                         }
2470                                         return t;
2471                                 }
2472
2473                                 // white space
2474                                 if (c == '\n'){
2475                                         any_token_seen |= tokens_seen;
2476                                         tokens_seen = false;
2477                                         comments_seen = false;
2478                                         continue;
2479                                 }
2480
2481                                 if (c >= '0' && c <= '9'){
2482                                         tokens_seen = true;
2483                                         return is_number (c);
2484                                 }
2485
2486                                 if (c == '.'){
2487                                         tokens_seen = true;
2488                                         int peek = peek_char ();
2489                                         if (peek >= '0' && peek <= '9')
2490                                                 return is_number (c);
2491                                         return Token.DOT;
2492                                 }
2493
2494                                 if (c == '#') {
2495                                         if (tokens_seen || comments_seen) {
2496                                                 Eror_WrongPreprocessorLocation ();
2497                                                 return Token.ERROR;
2498                                         }
2499
2500                                         if (handle_preprocessing_directive (true))
2501                                                 continue;
2502
2503                                         bool directive_expected = false;
2504                                         while ((c = get_char ()) != -1) {
2505                                                 if (col == 1) {
2506                                                         directive_expected = true;
2507                                                 } else if (!directive_expected) {
2508                                                         // TODO: Implement comment support for disabled code and uncomment this code
2509 //                                                      if (c == '#') {
2510 //                                                              Eror_WrongPreprocessorLocation ();
2511 //                                                              return Token.ERROR;
2512 //                                                      }
2513                                                         continue;
2514                                                 }
2515
2516                                                 if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
2517                                                         continue;
2518
2519                                                 if (c == '#') {
2520                                                         if (handle_preprocessing_directive (false))
2521                                                                 break;
2522                                                 }
2523                                                 directive_expected = false;
2524                                         }
2525
2526                                         if (c != -1) {
2527                                                 tokens_seen = false;
2528                                                 continue;
2529                                         }
2530
2531                                         return Token.EOF;
2532                                 }
2533
2534                                 if (c == '"')
2535                                         return consume_string (false);
2536
2537                                 if (c == '\''){
2538                                         c = get_char ();
2539                                         tokens_seen = true;
2540                                         if (c == '\''){
2541                                                 error_details = "Empty character literal";
2542                                                 Report.Error (1011, Location, error_details);
2543                                                 return Token.ERROR;
2544                                         }
2545                                         if (c == '\r' || c == '\n') {
2546                                                 Report.Error (1010, Location, "Newline in constant");
2547                                                 return Token.ERROR;
2548                                         }
2549                                         c = escape (c);
2550                                         if (c == -1)
2551                                                 return Token.ERROR;
2552                                         val = new System.Char ();
2553                                         val = (char) c;
2554                                         c = get_char ();
2555
2556                                         if (c != '\''){
2557                                                 error_details = "Too many characters in character literal";
2558                                                 Report.Error (1012, Location, error_details);
2559
2560                                                 // Try to recover, read until newline or next "'"
2561                                                 while ((c = get_char ()) != -1){
2562                                                         if (c == '\n'){
2563                                                                 break;
2564                                                         }
2565                                                         else if (c == '\'')
2566                                                                 break;
2567                                                 }
2568                                                 return Token.ERROR;
2569                                         }
2570                                         return Token.LITERAL_CHARACTER;
2571                                 }
2572
2573                                 if (c == '@') {
2574                                         c = get_char ();
2575                                         if (c == '"') {
2576                                                 tokens_seen = true;
2577                                                 return consume_string (true);
2578                                         } else if (is_identifier_start_character ((char) c)){
2579                                                 return consume_identifier (c, true);
2580                                         } else {
2581                                                 Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
2582                                         }
2583                                 }
2584
2585                                 error_details = ((char)c).ToString ();
2586
2587                                 return Token.ERROR;
2588                         }
2589
2590                         return Token.EOF;
2591                 }
2592
2593                 //
2594                 // Handles one line xml comment
2595                 //
2596                 private void handle_one_line_xml_comment ()
2597                 {
2598                         int c;
2599                         while ((c = peek_char ()) == ' ')
2600                                 get_char (); // skip heading whitespaces.
2601                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
2602                                 xml_comment_buffer.Append ((char) get_char ());
2603                         }
2604                         if (c == '\r' || c == '\n')
2605                                 xml_comment_buffer.Append (Environment.NewLine);
2606                 }
2607
2608                 //
2609                 // Remove heading "*" in Javadoc-like xml documentation.
2610                 //
2611                 private void update_formatted_doc_comment (int current_comment_start)
2612                 {
2613                         int length = xml_comment_buffer.Length - current_comment_start;
2614                         string [] lines = xml_comment_buffer.ToString (
2615                                 current_comment_start,
2616                                 length).Replace ("\r", "").Split ('\n');
2617
2618                         // The first line starts with /**, thus it is not target
2619                         // for the format check.
2620                         for (int i = 1; i < lines.Length; i++) {
2621                                 string s = lines [i];
2622                                 int idx = s.IndexOf ('*');
2623                                 string head = null;
2624                                 if (idx < 0) {
2625                                         if (i < lines.Length - 1)
2626                                                 return;
2627                                         head = s;
2628                                 } else
2629                                         head = s.Substring (0, idx);
2630                                 foreach (char c in head)
2631                                         if (c != ' ')
2632                                                 return;
2633                                 lines [i] = s.Substring (idx + 1);
2634                         }
2635                         xml_comment_buffer.Remove (current_comment_start, length);
2636                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
2637                 }
2638
2639                 //
2640                 // Updates current comment location.
2641                 //
2642                 private void update_comment_location ()
2643                 {
2644                         if (current_comment_location.IsNull) {
2645                                 // "-2" is for heading "//" or "/*"
2646                                 current_comment_location =
2647                                         new Location (ref_line, col - 2);
2648                         }
2649                 }
2650
2651                 //
2652                 // Checks if there was incorrect doc comments and raise
2653                 // warnings.
2654                 //
2655                 public void check_incorrect_doc_comment ()
2656                 {
2657                         if (xml_comment_buffer.Length > 0)
2658                                 warn_incorrect_doc_comment ();
2659                 }
2660
2661                 //
2662                 // Raises a warning when tokenizer found incorrect doccomment
2663                 // markup.
2664                 //
2665                 private void warn_incorrect_doc_comment ()
2666                 {
2667                         if (doc_state != XmlCommentState.Error) {
2668                                 doc_state = XmlCommentState.Error;
2669                                 // in csc, it is 'XML comment is not placed on
2670                                 // a valid language element'. But that does not
2671                                 // make sense.
2672                                 Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
2673                         }
2674                 }
2675
2676                 //
2677                 // Consumes the saved xml comment lines (if any)
2678                 // as for current target member or type.
2679                 //
2680                 public string consume_doc_comment ()
2681                 {
2682                         if (xml_comment_buffer.Length > 0) {
2683                                 string ret = xml_comment_buffer.ToString ();
2684                                 reset_doc_comment ();
2685                                 return ret;
2686                         }
2687                         return null;
2688                 }
2689
2690                 void reset_doc_comment ()
2691                 {
2692                         xml_comment_buffer.Length = 0;
2693                         current_comment_location = Location.Null;
2694                 }
2695
2696                 public void cleanup ()
2697                 {
2698                         if (ifstack != null && ifstack.Count >= 1) {
2699                                 current_location = new Location (ref_line, Col);
2700                                 int state = (int) ifstack.Pop ();
2701                                 if ((state & REGION) != 0)
2702                                         Report.Error (1038, Location, "#endregion directive expected");
2703                                 else
2704                                         Report.Error (1027, Location, "Expected `#endif' directive");
2705                         }
2706                 }
2707         }
2708
2709         //
2710         // Indicates whether it accepts XML documentation or not.
2711         //
2712         public enum XmlCommentState {
2713                 // comment is allowed in this state.
2714                 Allowed,
2715                 // comment is not allowed in this state.
2716                 NotAllowed,
2717                 // once comments appeared when it is NotAllowed, then the
2718                 // state is changed to it, until the state is changed to
2719                 // .Allowed.
2720                 Error
2721         }
2722 }
2723