mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 //
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.IO;
  19 using System.Globalization;
  20 using System.Reflection;
  21
  22 namespace Mono.CSharp
  23 {
  24         /// <summary>
  25         ///    Tokenizer for C# source code.
  26         /// </summary>
  27
  28         public class Tokenizer : yyParser.yyInput
  29         {
  30                 class KeywordEntry
  31                 {
  32                         public readonly int Token;
  33                         public KeywordEntry Next;
  34                         public readonly char[] Value;
  35
  36                         public KeywordEntry (string value, int token)
  37                         {
  38                                 this.Value = value.ToCharArray ();
  39                                 this.Token = token;
  40                         }
  41                 }
  42
  43                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  44                 {
  45                         readonly int length;
  46
  47                         public IdentifiersComparer (int length)
  48                         {
  49                                 this.length = length;
  50                         }
  51
  52                         public bool Equals (char[] x, char[] y)
  53                         {
  54                                 for (int i = 0; i < length; ++i)
  55                                         if (x [i] != y [i])
  56                                                 return false;
  57
  58                                 return true;
  59                         }
  60
  61                         public int GetHashCode (char[] obj)
  62                         {
  63                                 int h = 0;
  64                                 for (int i = 0; i < length; ++i)
  65                                         h = (h << 5) - h + obj [i];
  66
  67                                 return h;
  68                         }
  69                 }
  70
  71                 //
  72                 // This class has to be used in the parser only, it reuses token
  73                 // details after each parse
  74                 //
  75                 public class LocatedToken
  76                 {
  77                         int row, column;
  78                         string value;
  79
  80                         static LocatedToken[] buffer;
  81                         static int pos;
  82
  83                         private LocatedToken ()
  84                         {
  85                         }
  86
  87                         public static LocatedToken Create (int row, int column)
  88                         {
  89                                 return Create (null, row, column);
  90                         }
  91
  92                         public static LocatedToken Create (string value, int row, int column)
  93                         {
  94                                 //
  95                                 // TODO: I am not very happy about the logic but it's the best
  96                                 // what I could come up with for now.
  97                                 // Ideally we should be using just tiny buffer (256 elements) which
  98                                 // is enough to hold all details for currect stack and recycle elements
  99                                 // poped from the stack but there is a trick needed to recycle
 100                                 // them properly.
 101                                 //
 102                                 LocatedToken entry;
 103                                 if (pos >= buffer.Length) {
 104                                         entry = new LocatedToken ();
 105                                 } else {
 106                                         entry = buffer [pos];
 107                                         if (entry == null) {
 108                                                 entry = new LocatedToken ();
 109                                                 buffer [pos] = entry;
 110                                         }
 111
 112                                         ++pos;
 113                                 }
 114                                 entry.value = value;
 115                                 entry.row = row;
 116                                 entry.column = column;
 117                                 return entry;
 118                         }
 119
 120                         //
 121                         // Used for token not required by expression evaluator
 122                         //
 123                         public static LocatedToken CreateOptional (int row, int col)
 124                         {
 125 #if false
 126                                 return Create (row, col);
 127 #endif
 128                                 return null;
 129                         }
 130
 131                         public static void Initialize ()
 132                         {
 133                                 if (buffer == null)
 134                                         buffer = new LocatedToken [10000];
 135                                 pos = 0;
 136                         }
 137
 138                         public Location Location {
 139                                 get { return new Location (row, column); }
 140                         }
 141
 142                         public string Value {
 143                                 get { return value; }
 144                         }
 145                 }
 146
 147                 SeekableStreamReader reader;
 148                 SourceFile ref_name;
 149                 CompilationUnit file_name;
 150                 CompilerContext context;
 151                 bool hidden = false;
 152                 int ref_line = 1;
 153                 int line = 1;
 154                 int col = 0;
 155                 int previous_col;
 156                 int current_token;
 157                 bool handle_get_set = false;
 158                 bool handle_remove_add = false;
 159                 bool handle_where = false;
 160                 bool handle_typeof = false;
 161                 bool lambda_arguments_parsing;
 162                 Location current_comment_location = Location.Null;
 163                 List<Location> escaped_identifiers;
 164                 int parsing_generic_less_than;
 165
 166                 //
 167                 // Used mainly for parser optimizations. Some expressions for instance
 168                 // can appear only in block (including initializer, base initializer)
 169                 // scope only
 170                 //
 171                 public int parsing_block;
 172                 internal bool query_parsing;
 173
 174                 //
 175                 // When parsing type only, useful for ambiguous nullable types
 176                 //
 177                 public int parsing_type;
 178
 179                 //
 180                 // Set when parsing generic declaration (type or method header)
 181                 //
 182                 public bool parsing_generic_declaration;
 183
 184                 //
 185                 // The value indicates that we have not reach any declaration or
 186                 // namespace yet
 187                 //
 188                 public int parsing_declaration;
 189
 190                 //
 191                 // The special character to inject on streams to trigger the EXPRESSION_PARSE
 192                 // token to be returned.   It just happens to be a Unicode character that
 193                 // would never be part of a program (can not be an identifier).
 194                 //
 195                 // This character is only tested just before the tokenizer is about to report
 196                 // an error;   So on the regular operation mode, this addition will have no
 197                 // impact on the tokenizer's performance.
 198                 //
 199
 200                 public const int EvalStatementParserCharacter = 0x2190;   // Unicode Left Arrow
 201                 public const int EvalCompilationUnitParserCharacter = 0x2191;  // Unicode Arrow
 202                 public const int EvalUsingDeclarationsParserCharacter = 0x2192;  // Unicode Arrow
 203
 204                 //
 205                 // XML documentation buffer. The save point is used to divide
 206                 // comments on types and comments on members.
 207                 //
 208                 StringBuilder xml_comment_buffer;
 209
 210                 //
 211                 // See comment on XmlCommentState enumeration.
 212                 //
 213                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 214
 215                 //
 216                 // Whether tokens have been seen on this line
 217                 //
 218                 bool tokens_seen = false;
 219
 220                 //
 221                 // Set to true once the GENERATE_COMPLETION token has bee
 222                 // returned.   This helps produce one GENERATE_COMPLETION,
 223                 // as many COMPLETE_COMPLETION as necessary to complete the
 224                 // AST tree and one final EOF.
 225                 //
 226                 bool generated;
 227
 228                 //
 229                 // Whether a token has been seen on the file
 230                 // This is needed because `define' is not allowed to be used
 231                 // after a token has been seen.
 232                 //
 233                 bool any_token_seen = false;
 234
 235                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 236
 237                 public bool PropertyParsing {
 238                         get { return handle_get_set; }
 239                         set { handle_get_set = value; }
 240                 }
 241
 242                 public bool EventParsing {
 243                         get { return handle_remove_add; }
 244                         set { handle_remove_add = value; }
 245                 }
 246
 247                 public bool ConstraintsParsing {
 248                         get { return handle_where; }
 249                         set { handle_where = value; }
 250                 }
 251
 252                 public bool TypeOfParsing {
 253                         get { return handle_typeof; }
 254                         set { handle_typeof = value; }
 255                 }
 256
 257                 public XmlCommentState doc_state {
 258                         get { return xml_doc_state; }
 259                         set {
 260                                 if (value == XmlCommentState.Allowed) {
 261                                         check_incorrect_doc_comment ();
 262                                         reset_doc_comment ();
 263                                 }
 264                                 xml_doc_state = value;
 265                         }
 266                 }
 267
 268                 //
 269                 // This is used to trigger completion generation on the parser
 270                 public bool CompleteOnEOF;
 271
 272                 void AddEscapedIdentifier (Location loc)
 273                 {
 274                         if (escaped_identifiers == null)
 275                                 escaped_identifiers = new List<Location> ();
 276
 277                         escaped_identifiers.Add (loc);
 278                 }
 279
 280                 public bool IsEscapedIdentifier (Location loc)
 281                 {
 282                         if (escaped_identifiers != null) {
 283                                 foreach (Location lt in escaped_identifiers)
 284                                         if (lt.Equals (loc))
 285                                                 return true;
 286                         }
 287
 288                         return false;
 289                 }
 290
 291                 //
 292                 // Class variables
 293                 //
 294                 static KeywordEntry[][] keywords;
 295                 static Dictionary<string, object> keyword_strings;              // TODO: HashSet
 296                 static NumberStyles styles;
 297                 static NumberFormatInfo csharp_format_info;
 298
 299                 //
 300                 // Values for the associated token returned
 301                 //
 302                 internal int putback_char;      // Used by repl only
 303                 object val;
 304
 305                 //
 306                 // Pre-processor
 307                 //
 308                 const int TAKING        = 1;
 309                 const int ELSE_SEEN     = 4;
 310                 const int PARENT_TAKING = 8;
 311                 const int REGION        = 16;
 312
 313                 //
 314                 // pre-processor if stack state:
 315                 //
 316                 Stack<int> ifstack;
 317
 318                 static System.Text.StringBuilder string_builder;
 319
 320                 const int max_id_size = 512;
 321                 static char [] id_builder = new char [max_id_size];
 322
 323                 public static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 324
 325                 const int max_number_size = 512;
 326                 static char [] number_builder = new char [max_number_size];
 327                 static int number_pos;
 328
 329                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
 330
 331                 //
 332                 // Details about the error encoutered by the tokenizer
 333                 //
 334                 string error_details;
 335
 336                 public string error {
 337                         get {
 338                                 return error_details;
 339                         }
 340                 }
 341
 342                 public int Line {
 343                         get {
 344                                 return ref_line;
 345                         }
 346                 }
 347
 348                 //
 349                 // This is used when the tokenizer needs to save
 350                 // the current position as it needs to do some parsing
 351                 // on its own to deamiguate a token in behalf of the
 352                 // parser.
 353                 //
 354                 Stack<Position> position_stack = new Stack<Position> (2);
 355
 356                 class Position {
 357                         public int position;
 358                         public int line;
 359                         public int ref_line;
 360                         public int col;
 361                         public bool hidden;
 362                         public int putback_char;
 363                         public int previous_col;
 364                         public Stack<int> ifstack;
 365                         public int parsing_generic_less_than;
 366                         public int current_token;
 367                         public object val;
 368
 369                         public Position (Tokenizer t)
 370                         {
 371                                 position = t.reader.Position;
 372                                 line = t.line;
 373                                 ref_line = t.ref_line;
 374                                 col = t.col;
 375                                 hidden = t.hidden;
 376                                 putback_char = t.putback_char;
 377                                 previous_col = t.previous_col;
 378                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 379                                         // There is no simple way to clone Stack<T> all
 380                                         // methods reverse the order
 381                                         var clone = t.ifstack.ToArray ();
 382                                         Array.Reverse (clone);
 383                                         ifstack = new Stack<int> (clone);
 384                                 }
 385                                 parsing_generic_less_than = t.parsing_generic_less_than;
 386                                 current_token = t.current_token;
 387                                 val = t.val;
 388                         }
 389                 }
 390
 391                 public void PushPosition ()
 392                 {
 393                         position_stack.Push (new Position (this));
 394                 }
 395
 396                 public void PopPosition ()
 397                 {
 398                         Position p = position_stack.Pop ();
 399
 400                         reader.Position = p.position;
 401                         ref_line = p.ref_line;
 402                         line = p.line;
 403                         col = p.col;
 404                         hidden = p.hidden;
 405                         putback_char = p.putback_char;
 406                         previous_col = p.previous_col;
 407                         ifstack = p.ifstack;
 408                         parsing_generic_less_than = p.parsing_generic_less_than;
 409                         current_token = p.current_token;
 410                         val = p.val;
 411                 }
 412
 413                 // Do not reset the position, ignore it.
 414                 public void DiscardPosition ()
 415                 {
 416                         position_stack.Pop ();
 417                 }
 418
 419                 static void AddKeyword (string kw, int token)
 420                 {
 421                         keyword_strings.Add (kw, null);
 422
 423                         int length = kw.Length;
 424                         if (keywords [length] == null) {
 425                                 keywords [length] = new KeywordEntry ['z' - '_' + 1];
 426                         }
 427
 428                         int char_index = kw [0] - '_';
 429                         KeywordEntry kwe = keywords [length] [char_index];
 430                         if (kwe == null) {
 431                                 keywords [length] [char_index] = new KeywordEntry (kw, token);
 432                                 return;
 433                         }
 434
 435                         while (kwe.Next != null) {
 436                                 kwe = kwe.Next;
 437                         }
 438
 439                         kwe.Next = new KeywordEntry (kw, token);
 440                 }
 441
 442                 static void InitTokens ()
 443                 {
 444                         keyword_strings = new Dictionary<string, object> ();
 445
 446                         // 11 is the length of the longest keyword for now
 447                         keywords = new KeywordEntry [11] [];
 448
 449                         AddKeyword ("__arglist", Token.ARGLIST);
 450                         AddKeyword ("abstract", Token.ABSTRACT);
 451                         AddKeyword ("as", Token.AS);
 452                         AddKeyword ("add", Token.ADD);
 453                         AddKeyword ("base", Token.BASE);
 454                         AddKeyword ("bool", Token.BOOL);
 455                         AddKeyword ("break", Token.BREAK);
 456                         AddKeyword ("byte", Token.BYTE);
 457                         AddKeyword ("case", Token.CASE);
 458                         AddKeyword ("catch", Token.CATCH);
 459                         AddKeyword ("char", Token.CHAR);
 460                         AddKeyword ("checked", Token.CHECKED);
 461                         AddKeyword ("class", Token.CLASS);
 462                         AddKeyword ("const", Token.CONST);
 463                         AddKeyword ("continue", Token.CONTINUE);
 464                         AddKeyword ("decimal", Token.DECIMAL);
 465                         AddKeyword ("default", Token.DEFAULT);
 466                         AddKeyword ("delegate", Token.DELEGATE);
 467                         AddKeyword ("do", Token.DO);
 468                         AddKeyword ("double", Token.DOUBLE);
 469                         AddKeyword ("else", Token.ELSE);
 470                         AddKeyword ("enum", Token.ENUM);
 471                         AddKeyword ("event", Token.EVENT);
 472                         AddKeyword ("explicit", Token.EXPLICIT);
 473                         AddKeyword ("extern", Token.EXTERN);
 474                         AddKeyword ("false", Token.FALSE);
 475                         AddKeyword ("finally", Token.FINALLY);
 476                         AddKeyword ("fixed", Token.FIXED);
 477                         AddKeyword ("float", Token.FLOAT);
 478                         AddKeyword ("for", Token.FOR);
 479                         AddKeyword ("foreach", Token.FOREACH);
 480                         AddKeyword ("goto", Token.GOTO);
 481                         AddKeyword ("get", Token.GET);
 482                         AddKeyword ("if", Token.IF);
 483                         AddKeyword ("implicit", Token.IMPLICIT);
 484                         AddKeyword ("in", Token.IN);
 485                         AddKeyword ("int", Token.INT);
 486                         AddKeyword ("interface", Token.INTERFACE);
 487                         AddKeyword ("internal", Token.INTERNAL);
 488                         AddKeyword ("is", Token.IS);
 489                         AddKeyword ("lock", Token.LOCK);
 490                         AddKeyword ("long", Token.LONG);
 491                         AddKeyword ("namespace", Token.NAMESPACE);
 492                         AddKeyword ("new", Token.NEW);
 493                         AddKeyword ("null", Token.NULL);
 494                         AddKeyword ("object", Token.OBJECT);
 495                         AddKeyword ("operator", Token.OPERATOR);
 496                         AddKeyword ("out", Token.OUT);
 497                         AddKeyword ("override", Token.OVERRIDE);
 498                         AddKeyword ("params", Token.PARAMS);
 499                         AddKeyword ("private", Token.PRIVATE);
 500                         AddKeyword ("protected", Token.PROTECTED);
 501                         AddKeyword ("public", Token.PUBLIC);
 502                         AddKeyword ("readonly", Token.READONLY);
 503                         AddKeyword ("ref", Token.REF);
 504                         AddKeyword ("remove", Token.REMOVE);
 505                         AddKeyword ("return", Token.RETURN);
 506                         AddKeyword ("sbyte", Token.SBYTE);
 507                         AddKeyword ("sealed", Token.SEALED);
 508                         AddKeyword ("set", Token.SET);
 509                         AddKeyword ("short", Token.SHORT);
 510                         AddKeyword ("sizeof", Token.SIZEOF);
 511                         AddKeyword ("stackalloc", Token.STACKALLOC);
 512                         AddKeyword ("static", Token.STATIC);
 513                         AddKeyword ("string", Token.STRING);
 514                         AddKeyword ("struct", Token.STRUCT);
 515                         AddKeyword ("switch", Token.SWITCH);
 516                         AddKeyword ("this", Token.THIS);
 517                         AddKeyword ("throw", Token.THROW);
 518                         AddKeyword ("true", Token.TRUE);
 519                         AddKeyword ("try", Token.TRY);
 520                         AddKeyword ("typeof", Token.TYPEOF);
 521                         AddKeyword ("uint", Token.UINT);
 522                         AddKeyword ("ulong", Token.ULONG);
 523                         AddKeyword ("unchecked", Token.UNCHECKED);
 524                         AddKeyword ("unsafe", Token.UNSAFE);
 525                         AddKeyword ("ushort", Token.USHORT);
 526                         AddKeyword ("using", Token.USING);
 527                         AddKeyword ("virtual", Token.VIRTUAL);
 528                         AddKeyword ("void", Token.VOID);
 529                         AddKeyword ("volatile", Token.VOLATILE);
 530                         AddKeyword ("while", Token.WHILE);
 531                         AddKeyword ("partial", Token.PARTIAL);
 532                         AddKeyword ("where", Token.WHERE);
 533
 534                         // LINQ keywords
 535                         AddKeyword ("from", Token.FROM);
 536                         AddKeyword ("join", Token.JOIN);
 537                         AddKeyword ("on", Token.ON);
 538                         AddKeyword ("equals", Token.EQUALS);
 539                         AddKeyword ("select", Token.SELECT);
 540                         AddKeyword ("group", Token.GROUP);
 541                         AddKeyword ("by", Token.BY);
 542                         AddKeyword ("let", Token.LET);
 543                         AddKeyword ("orderby", Token.ORDERBY);
 544                         AddKeyword ("ascending", Token.ASCENDING);
 545                         AddKeyword ("descending", Token.DESCENDING);
 546                         AddKeyword ("into", Token.INTO);
 547                 }
 548
 549                 //
 550                 // Class initializer
 551                 //
 552                 static Tokenizer ()
 553                 {
 554                         InitTokens ();
 555                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 556                         styles = NumberStyles.Float;
 557
 558                         string_builder = new System.Text.StringBuilder ();
 559                 }
 560
 561                 int GetKeyword (char[] id, int id_len)
 562                 {
 563                         //
 564                         // Keywords are stored in an array of arrays grouped by their
 565                         // length and then by the first character
 566                         //
 567                         if (id_len >= keywords.Length || keywords [id_len] == null)
 568                                 return -1;
 569
 570                         int first_index = id [0] - '_';
 571                         if (first_index > 'z')
 572                                 return -1;
 573
 574                         KeywordEntry kwe = keywords [id_len] [first_index];
 575                         if (kwe == null)
 576                                 return -1;
 577
 578                         int res;
 579                         do {
 580                                 res = kwe.Token;
 581                                 for (int i = 1; i < id_len; ++i) {
 582                                         if (id [i] != kwe.Value [i]) {
 583                                                 res = 0;
 584                                                 kwe = kwe.Next;
 585                                                 break;
 586                                         }
 587                                 }
 588                         } while (res == 0 && kwe != null);
 589
 590                         if (res == 0)
 591                                 return -1;
 592
 593                         int next_token;
 594                         switch (res) {
 595                         case Token.GET:
 596                         case Token.SET:
 597                                 if (!handle_get_set)
 598                                         res = -1;
 599                                 break;
 600                         case Token.REMOVE:
 601                         case Token.ADD:
 602                                 if (!handle_remove_add)
 603                                         res = -1;
 604                                 break;
 605                         case Token.EXTERN:
 606                                 if (parsing_declaration == 0)
 607                                         res = Token.EXTERN_ALIAS;
 608                                 break;
 609                         case Token.DEFAULT:
 610                                 if (peek_token () == Token.COLON) {
 611                                         token ();
 612                                         res = Token.DEFAULT_COLON;
 613                                 }
 614                                 break;
 615                         case Token.WHERE:
 616                                 if (!handle_where && !query_parsing)
 617                                         res = -1;
 618                                 break;
 619                         case Token.FROM:
 620                                 //
 621                                 // A query expression is any expression that starts with `from identifier'
 622                                 // followed by any token except ; , =
 623                                 //
 624                                 if (!query_parsing) {
 625                                         if (lambda_arguments_parsing) {
 626                                                 res = -1;
 627                                                 break;
 628                                         }
 629
 630                                         PushPosition ();
 631                                         // HACK: to disable generics micro-parser, because PushPosition does not
 632                                         // store identifiers array
 633                                         parsing_generic_less_than = 1;
 634                                         switch (xtoken ()) {
 635                                         case Token.IDENTIFIER:
 636                                         case Token.INT:
 637                                         case Token.BOOL:
 638                                         case Token.BYTE:
 639                                         case Token.CHAR:
 640                                         case Token.DECIMAL:
 641                                         case Token.FLOAT:
 642                                         case Token.LONG:
 643                                         case Token.OBJECT:
 644                                         case Token.STRING:
 645                                         case Token.UINT:
 646                                         case Token.ULONG:
 647                                                 next_token = xtoken ();
 648                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 649                                                         goto default;
 650
 651                                                 res = Token.FROM_FIRST;
 652                                                 query_parsing = true;
 653                                                 if (RootContext.Version <= LanguageVersion.ISO_2)
 654                                                         Report.FeatureIsNotAvailable (Location, "query expressions");
 655                                                 break;
 656                                         case Token.VOID:
 657                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 658                                                 break;
 659                                         default:
 660                                                 PopPosition ();
 661                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 662                                                 // which has been overwritten before we grabbed the identifier
 663                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 664                                                 return -1;
 665                                         }
 666                                         PopPosition ();
 667                                 }
 668                                 break;
 669                         case Token.JOIN:
 670                         case Token.ON:
 671                         case Token.EQUALS:
 672                         case Token.SELECT:
 673                         case Token.GROUP:
 674                         case Token.BY:
 675                         case Token.LET:
 676                         case Token.ORDERBY:
 677                         case Token.ASCENDING:
 678                         case Token.DESCENDING:
 679                         case Token.INTO:
 680                                 if (!query_parsing)
 681                                         res = -1;
 682                                 break;
 683
 684                         case Token.USING:
 685                         case Token.NAMESPACE:
 686                                 // TODO: some explanation needed
 687                                 check_incorrect_doc_comment ();
 688                                 break;
 689
 690                         case Token.PARTIAL:
 691                                 if (parsing_block > 0) {
 692                                         res = -1;
 693                                         break;
 694                                 }
 695
 696                                 // Save current position and parse next token.
 697                                 PushPosition ();
 698
 699                                 next_token = token ();
 700                                 bool ok = (next_token == Token.CLASS) ||
 701                                         (next_token == Token.STRUCT) ||
 702                                         (next_token == Token.INTERFACE) ||
 703                                         (next_token == Token.VOID);
 704
 705                                 PopPosition ();
 706
 707                                 if (ok) {
 708                                         if (next_token == Token.VOID) {
 709                                                 if (RootContext.Version == LanguageVersion.ISO_1 ||
 710                                                     RootContext.Version == LanguageVersion.ISO_2)
 711                                                         Report.FeatureIsNotAvailable (Location, "partial methods");
 712                                         } else if (RootContext.Version == LanguageVersion.ISO_1)
 713                                                 Report.FeatureIsNotAvailable (Location, "partial types");
 714
 715                                         return res;
 716                                 }
 717
 718                                 if (next_token < Token.LAST_KEYWORD) {
 719                                         Report.Error (267, Location,
 720                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 721                                         return token ();
 722                                 }
 723
 724                                 res = -1;
 725                                 break;
 726                         }
 727
 728                         return res;
 729                 }
 730
 731                 public Location Location {
 732                         get {
 733                                 return new Location (ref_line, hidden ? -1 : col);
 734                         }
 735                 }
 736
 737                 public Tokenizer (SeekableStreamReader input, CompilationUnit file, CompilerContext ctx)
 738                 {
 739                         this.ref_name = file;
 740                         this.file_name = file;
 741                         this.context = ctx;
 742                         reader = input;
 743
 744                         putback_char = -1;
 745
 746                         xml_comment_buffer = new StringBuilder ();
 747
 748                         //
 749                         // FIXME: This could be `Location.Push' but we have to
 750                         // find out why the MS compiler allows this
 751                         //
 752                         Mono.CSharp.Location.Push (file, file);
 753                 }
 754
 755                 static bool is_identifier_start_character (int c)
 756                 {
 757                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 758                 }
 759
 760                 static bool is_identifier_part_character (char c)
 761                 {
 762                         if (c >= 'a' && c <= 'z')
 763                                 return true;
 764
 765                         if (c >= 'A' && c <= 'Z')
 766                                 return true;
 767
 768                         if (c == '_' || (c >= '0' && c <= '9'))
 769                                 return true;
 770
 771                         if (c < 0x80)
 772                                 return false;
 773
 774                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 775                 }
 776
 777                 public static bool IsKeyword (string s)
 778                 {
 779                         return keyword_strings.ContainsKey (s);
 780                 }
 781
 782                 //
 783                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 784                 //
 785                 int TokenizeOpenParens ()
 786                 {
 787                         int ptoken;
 788                         current_token = -1;
 789
 790                         int bracket_level = 0;
 791                         bool is_type = false;
 792                         bool can_be_type = false;
 793
 794                         while (true) {
 795                                 ptoken = current_token;
 796                                 token ();
 797
 798                                 switch (current_token) {
 799                                 case Token.CLOSE_PARENS:
 800                                         token ();
 801
 802                                         //
 803                                         // Expression inside parens is lambda, (int i) =>
 804                                         //
 805                                         if (current_token == Token.ARROW)
 806                                                 return Token.OPEN_PARENS_LAMBDA;
 807
 808                                         //
 809                                         // Expression inside parens is single type, (int[])
 810                                         //
 811                                         if (is_type)
 812                                                 return Token.OPEN_PARENS_CAST;
 813
 814                                         //
 815                                         // Expression is possible cast, look at next token, (T)null
 816                                         //
 817                                         if (can_be_type) {
 818                                                 switch (current_token) {
 819                                                 case Token.OPEN_PARENS:
 820                                                 case Token.BANG:
 821                                                 case Token.TILDE:
 822                                                 case Token.IDENTIFIER:
 823                                                 case Token.LITERAL:
 824                                                 case Token.BASE:
 825                                                 case Token.CHECKED:
 826                                                 case Token.DELEGATE:
 827                                                 case Token.FALSE:
 828                                                 case Token.FIXED:
 829                                                 case Token.NEW:
 830                                                 case Token.NULL:
 831                                                 case Token.SIZEOF:
 832                                                 case Token.THIS:
 833                                                 case Token.THROW:
 834                                                 case Token.TRUE:
 835                                                 case Token.TYPEOF:
 836                                                 case Token.UNCHECKED:
 837                                                 case Token.UNSAFE:
 838                                                 case Token.DEFAULT:
 839
 840                                                 //
 841                                                 // These can be part of a member access
 842                                                 //
 843                                                 case Token.INT:
 844                                                 case Token.UINT:
 845                                                 case Token.SHORT:
 846                                                 case Token.USHORT:
 847                                                 case Token.LONG:
 848                                                 case Token.ULONG:
 849                                                 case Token.DOUBLE:
 850                                                 case Token.FLOAT:
 851                                                 case Token.CHAR:
 852                                                 case Token.BYTE:
 853                                                 case Token.DECIMAL:
 854                                                 case Token.BOOL:
 855                                                         return Token.OPEN_PARENS_CAST;
 856                                                 }
 857                                         }
 858                                         return Token.OPEN_PARENS;
 859
 860                                 case Token.DOT:
 861                                 case Token.DOUBLE_COLON:
 862                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
 863                                                 goto default;
 864
 865                                         continue;
 866
 867                                 case Token.IDENTIFIER:
 868                                         switch (ptoken) {
 869                                         case Token.DOT:
 870                                         case Token.OP_GENERICS_LT:
 871                                         case Token.COMMA:
 872                                         case Token.DOUBLE_COLON:
 873                                         case -1:
 874                                                 if (bracket_level == 0)
 875                                                         can_be_type = true;
 876                                                 continue;
 877                                         default:
 878                                                 can_be_type = is_type = false;
 879                                                 continue;
 880                                         }
 881
 882                                 case Token.OBJECT:
 883                                 case Token.STRING:
 884                                 case Token.BOOL:
 885                                 case Token.DECIMAL:
 886                                 case Token.FLOAT:
 887                                 case Token.DOUBLE:
 888                                 case Token.SBYTE:
 889                                 case Token.BYTE:
 890                                 case Token.SHORT:
 891                                 case Token.USHORT:
 892                                 case Token.INT:
 893                                 case Token.UINT:
 894                                 case Token.LONG:
 895                                 case Token.ULONG:
 896                                 case Token.CHAR:
 897                                 case Token.VOID:
 898                                         if (bracket_level == 0)
 899                                                 is_type = true;
 900                                         continue;
 901
 902                                 case Token.COMMA:
 903                                         if (bracket_level == 0) {
 904                                                 bracket_level = 100;
 905                                                 can_be_type = is_type = false;
 906                                         }
 907                                         continue;
 908
 909                                 case Token.OP_GENERICS_LT:
 910                                 case Token.OPEN_BRACKET:
 911                                         if (bracket_level++ == 0)
 912                                                 is_type = true;
 913                                         continue;
 914
 915                                 case Token.OP_GENERICS_GT:
 916                                 case Token.CLOSE_BRACKET:
 917                                         --bracket_level;
 918                                         continue;
 919
 920                                 case Token.INTERR_NULLABLE:
 921                                 case Token.STAR:
 922                                         if (bracket_level == 0)
 923                                                 is_type = true;
 924                                         continue;
 925
 926                                 case Token.REF:
 927                                 case Token.OUT:
 928                                         can_be_type = is_type = false;
 929                                         continue;
 930
 931                                 default:
 932                                         return Token.OPEN_PARENS;
 933                                 }
 934                         }
 935                 }
 936
 937                 public static bool IsValidIdentifier (string s)
 938                 {
 939                         if (s == null || s.Length == 0)
 940                                 return false;
 941
 942                         if (!is_identifier_start_character (s [0]))
 943                                 return false;
 944
 945                         for (int i = 1; i < s.Length; i ++)
 946                                 if (! is_identifier_part_character (s [i]))
 947                                         return false;
 948
 949                         return true;
 950                 }
 951
 952                 bool parse_less_than ()
 953                 {
 954                 start:
 955                         int the_token = token ();
 956                         if (the_token == Token.OPEN_BRACKET) {
 957                                 do {
 958                                         the_token = token ();
 959                                 } while (the_token != Token.CLOSE_BRACKET);
 960                                 the_token = token ();
 961                         } else if (the_token == Token.IN || the_token == Token.OUT) {
 962                                 the_token = token ();
 963                         }
 964                         switch (the_token) {
 965                         case Token.IDENTIFIER:
 966                         case Token.OBJECT:
 967                         case Token.STRING:
 968                         case Token.BOOL:
 969                         case Token.DECIMAL:
 970                         case Token.FLOAT:
 971                         case Token.DOUBLE:
 972                         case Token.SBYTE:
 973                         case Token.BYTE:
 974                         case Token.SHORT:
 975                         case Token.USHORT:
 976                         case Token.INT:
 977                         case Token.UINT:
 978                         case Token.LONG:
 979                         case Token.ULONG:
 980                         case Token.CHAR:
 981                         case Token.VOID:
 982                                 break;
 983                         case Token.OP_GENERICS_GT:
 984                                 return true;
 985
 986                         default:
 987                                 return false;
 988                         }
 989                 again:
 990                         the_token = token ();
 991
 992                         if (the_token == Token.OP_GENERICS_GT)
 993                                 return true;
 994                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
 995                                 goto start;
 996                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
 997                                 goto again;
 998                         else if (the_token == Token.OP_GENERICS_LT) {
 999                                 if (!parse_less_than ())
1000                                         return false;
1001                                 goto again;
1002                         } else if (the_token == Token.OPEN_BRACKET) {
1003                         rank_specifiers:
1004                                 the_token = token ();
1005                                 if (the_token == Token.CLOSE_BRACKET)
1006                                         goto again;
1007                                 else if (the_token == Token.COMMA)
1008                                         goto rank_specifiers;
1009                                 return false;
1010                         }
1011
1012                         return false;
1013                 }
1014
1015                 bool parse_generic_dimension (out int dimension)
1016                 {
1017                         dimension = 1;
1018
1019                 again:
1020                         int the_token = token ();
1021                         if (the_token == Token.OP_GENERICS_GT)
1022                                 return true;
1023                         else if (the_token == Token.COMMA) {
1024                                 dimension++;
1025                                 goto again;
1026                         }
1027
1028                         return false;
1029                 }
1030
1031                 public int peek_token ()
1032                 {
1033                         int the_token;
1034
1035                         PushPosition ();
1036                         the_token = token ();
1037                         PopPosition ();
1038
1039                         return the_token;
1040                 }
1041
1042                 //
1043                 // Tonizes `?' using custom disambiguous rules to return one
1044                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1045                 //
1046                 // Tricky expression look like:
1047                 //
1048                 // Foo ? a = x ? b : c;
1049                 //
1050                 int TokenizePossibleNullableType ()
1051                 {
1052                         if (parsing_block == 0 || parsing_type > 0)
1053                                 return Token.INTERR_NULLABLE;
1054
1055                         int d = peek_char ();
1056                         if (d == '?') {
1057                                 get_char ();
1058                                 return Token.OP_COALESCING;
1059                         }
1060
1061                         switch (current_token) {
1062                         case Token.CLOSE_PARENS:
1063                         case Token.TRUE:
1064                         case Token.FALSE:
1065                         case Token.NULL:
1066                         case Token.LITERAL:
1067                                 return Token.INTERR;
1068                         }
1069
1070                         if (d != ' ') {
1071                                 if (d == ',' || d == ';' || d == '>')
1072                                         return Token.INTERR_NULLABLE;
1073                                 if (d == '*' || (d >= '0' && d <= '9'))
1074                                         return Token.INTERR;
1075                         }
1076
1077                         PushPosition ();
1078                         current_token = Token.NONE;
1079                         int next_token;
1080                         switch (xtoken ()) {
1081                         case Token.LITERAL:
1082                         case Token.TRUE:
1083                         case Token.FALSE:
1084                         case Token.NULL:
1085                         case Token.THIS:
1086                         case Token.NEW:
1087                                 next_token = Token.INTERR;
1088                                 break;
1089
1090                         case Token.SEMICOLON:
1091                         case Token.COMMA:
1092                         case Token.CLOSE_PARENS:
1093                         case Token.OPEN_BRACKET:
1094                         case Token.OP_GENERICS_GT:
1095                                 next_token = Token.INTERR_NULLABLE;
1096                                 break;
1097
1098                         default:
1099                                 next_token = -1;
1100                                 break;
1101                         }
1102
1103                         if (next_token == -1) {
1104                                 switch (xtoken ()) {
1105                                 case Token.COMMA:
1106                                 case Token.SEMICOLON:
1107                                 case Token.OPEN_BRACE:
1108                                 case Token.CLOSE_PARENS:
1109                                 case Token.IN:
1110                                         next_token = Token.INTERR_NULLABLE;
1111                                         break;
1112
1113                                 case Token.COLON:
1114                                         next_token = Token.INTERR;
1115                                         break;
1116
1117                                 default:
1118                                         int ntoken;
1119                                         int interrs = 1;
1120                                         int colons = 0;
1121                                         //
1122                                         // All shorcuts failed, do it hard way
1123                                         //
1124                                         while ((ntoken = xtoken ()) != Token.EOF) {
1125                                                 if (ntoken == Token.SEMICOLON)
1126                                                         break;
1127
1128                                                 if (ntoken == Token.COLON) {
1129                                                         if (++colons == interrs)
1130                                                                 break;
1131                                                         continue;
1132                                                 }
1133
1134                                                 if (ntoken == Token.INTERR) {
1135                                                         ++interrs;
1136                                                         continue;
1137                                                 }
1138                                         }
1139
1140                                         next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1141                                         break;
1142                                 }
1143                         }
1144
1145                         PopPosition ();
1146                         return next_token;
1147                 }
1148
1149                 bool decimal_digits (int c)
1150                 {
1151                         int d;
1152                         bool seen_digits = false;
1153
1154                         if (c != -1){
1155                                 if (number_pos == max_number_size)
1156                                         Error_NumericConstantTooLong ();
1157                                 number_builder [number_pos++] = (char) c;
1158                         }
1159
1160                         //
1161                         // We use peek_char2, because decimal_digits needs to do a
1162                         // 2-character look-ahead (5.ToString for example).
1163                         //
1164                         while ((d = peek_char2 ()) != -1){
1165                                 if (d >= '0' && d <= '9'){
1166                                         if (number_pos == max_number_size)
1167                                                 Error_NumericConstantTooLong ();
1168                                         number_builder [number_pos++] = (char) d;
1169                                         get_char ();
1170                                         seen_digits = true;
1171                                 } else
1172                                         break;
1173                         }
1174
1175                         return seen_digits;
1176                 }
1177
1178                 static bool is_hex (int e)
1179                 {
1180                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1181                 }
1182
1183                 static TypeCode real_type_suffix (int c)
1184                 {
1185                         switch (c){
1186                         case 'F': case 'f':
1187                                 return TypeCode.Single;
1188                         case 'D': case 'd':
1189                                 return TypeCode.Double;
1190                         case 'M': case 'm':
1191                                 return TypeCode.Decimal;
1192                         default:
1193                                 return TypeCode.Empty;
1194                         }
1195                 }
1196
1197                 int integer_type_suffix (ulong ul, int c)
1198                 {
1199                         bool is_unsigned = false;
1200                         bool is_long = false;
1201
1202                         if (c != -1){
1203                                 bool scanning = true;
1204                                 do {
1205                                         switch (c){
1206                                         case 'U': case 'u':
1207                                                 if (is_unsigned)
1208                                                         scanning = false;
1209                                                 is_unsigned = true;
1210                                                 get_char ();
1211                                                 break;
1212
1213                                         case 'l':
1214                                                 if (!is_unsigned){
1215                                                         //
1216                                                         // if we have not seen anything in between
1217                                                         // report this error
1218                                                         //
1219                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1220                                                 }
1221
1222                                                 goto case 'L';
1223
1224                                         case 'L':
1225                                                 if (is_long)
1226                                                         scanning = false;
1227                                                 is_long = true;
1228                                                 get_char ();
1229                                                 break;
1230
1231                                         default:
1232                                                 scanning = false;
1233                                                 break;
1234                                         }
1235                                         c = peek_char ();
1236                                 } while (scanning);
1237                         }
1238
1239                         if (is_long && is_unsigned){
1240                                 val = new ULongLiteral (ul, Location);
1241                                 return Token.LITERAL;
1242                         }
1243
1244                         if (is_unsigned){
1245                                 // uint if possible, or ulong else.
1246
1247                                 if ((ul & 0xffffffff00000000) == 0)
1248                                         val = new UIntLiteral ((uint) ul, Location);
1249                                 else
1250                                         val = new ULongLiteral (ul, Location);
1251                         } else if (is_long){
1252                                 // long if possible, ulong otherwise
1253                                 if ((ul & 0x8000000000000000) != 0)
1254                                         val = new ULongLiteral (ul, Location);
1255                                 else
1256                                         val = new LongLiteral ((long) ul, Location);
1257                         } else {
1258                                 // int, uint, long or ulong in that order
1259                                 if ((ul & 0xffffffff00000000) == 0){
1260                                         uint ui = (uint) ul;
1261
1262                                         if ((ui & 0x80000000) != 0)
1263                                                 val = new UIntLiteral (ui, Location);
1264                                         else
1265                                                 val = new IntLiteral ((int) ui, Location);
1266                                 } else {
1267                                         if ((ul & 0x8000000000000000) != 0)
1268                                                 val = new ULongLiteral (ul, Location);
1269                                         else
1270                                                 val = new LongLiteral ((long) ul, Location);
1271                                 }
1272                         }
1273                         return Token.LITERAL;
1274                 }
1275
1276                 //
1277                 // given `c' as the next char in the input decide whether
1278                 // we need to convert to a special type, and then choose
1279                 // the best representation for the integer
1280                 //
1281                 int adjust_int (int c)
1282                 {
1283                         try {
1284                                 if (number_pos > 9){
1285                                         ulong ul = (uint) (number_builder [0] - '0');
1286
1287                                         for (int i = 1; i < number_pos; i++){
1288                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1289                                         }
1290                                         return integer_type_suffix (ul, c);
1291                                 } else {
1292                                         uint ui = (uint) (number_builder [0] - '0');
1293
1294                                         for (int i = 1; i < number_pos; i++){
1295                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1296                                         }
1297                                         return integer_type_suffix (ui, c);
1298                                 }
1299                         } catch (OverflowException) {
1300                                 error_details = "Integral constant is too large";
1301                                 Report.Error (1021, Location, error_details);
1302                                 val = new IntLiteral (0, Location);
1303                                 return Token.LITERAL;
1304                         }
1305                         catch (FormatException) {
1306                                 Report.Error (1013, Location, "Invalid number");
1307                                 val = new IntLiteral (0, Location);
1308                                 return Token.LITERAL;
1309                         }
1310                 }
1311
1312                 int adjust_real (TypeCode t)
1313                 {
1314                         string s = new String (number_builder, 0, number_pos);
1315                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1316
1317                         switch (t){
1318                         case TypeCode.Decimal:
1319                                 try {
1320                                         val = new DecimalLiteral (decimal.Parse (s, styles, csharp_format_info), Location);
1321                                 } catch (OverflowException) {
1322                                         val = new DecimalLiteral (0, Location);
1323                                         Report.Error (594, Location, error_details, "decimal");
1324                                 }
1325                                 break;
1326                         case TypeCode.Single:
1327                                 try {
1328                                         val = new FloatLiteral (float.Parse (s, styles, csharp_format_info), Location);
1329                                 } catch (OverflowException) {
1330                                         val = new FloatLiteral (0, Location);
1331                                         Report.Error (594, Location, error_details, "float");
1332                                 }
1333                                 break;
1334                         default:
1335                                 try {
1336                                         val = new DoubleLiteral (double.Parse (s, styles, csharp_format_info), Location);
1337                                 } catch (OverflowException) {
1338                                         val = new DoubleLiteral (0, Location);
1339                                         Report.Error (594, Location, error_details, "double");
1340                                 }
1341                                 break;
1342                         }
1343
1344                         return Token.LITERAL;
1345                 }
1346
1347                 int handle_hex ()
1348                 {
1349                         int d;
1350                         ulong ul;
1351
1352                         get_char ();
1353                         while ((d = peek_char ()) != -1){
1354                                 if (is_hex (d)){
1355                                         number_builder [number_pos++] = (char) d;
1356                                         get_char ();
1357                                 } else
1358                                         break;
1359                         }
1360
1361                         string s = new String (number_builder, 0, number_pos);
1362                         try {
1363                                 if (number_pos <= 8)
1364                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1365                                 else
1366                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1367                         } catch (OverflowException){
1368                                 error_details = "Integral constant is too large";
1369                                 Report.Error (1021, Location, error_details);
1370                                 val = new IntLiteral (0, Location);
1371                                 return Token.LITERAL;
1372                         }
1373                         catch (FormatException) {
1374                                 Report.Error (1013, Location, "Invalid number");
1375                                 val = new IntLiteral (0, Location);
1376                                 return Token.LITERAL;
1377                         }
1378
1379                         return integer_type_suffix (ul, peek_char ());
1380                 }
1381
1382                 //
1383                 // Invoked if we know we have .digits or digits
1384                 //
1385                 int is_number (int c)
1386                 {
1387                         bool is_real = false;
1388
1389                         number_pos = 0;
1390
1391                         if (c >= '0' && c <= '9'){
1392                                 if (c == '0'){
1393                                         int peek = peek_char ();
1394
1395                                         if (peek == 'x' || peek == 'X')
1396                                                 return handle_hex ();
1397                                 }
1398                                 decimal_digits (c);
1399                                 c = get_char ();
1400                         }
1401
1402                         //
1403                         // We need to handle the case of
1404                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1405                         //
1406                         if (c == '.'){
1407                                 if (decimal_digits ('.')){
1408                                         is_real = true;
1409                                         c = get_char ();
1410                                 } else {
1411                                         putback ('.');
1412                                         number_pos--;
1413                                         return adjust_int (-1);
1414                                 }
1415                         }
1416
1417                         if (c == 'e' || c == 'E'){
1418                                 is_real = true;
1419                                 if (number_pos == max_number_size)
1420                                         Error_NumericConstantTooLong ();
1421                                 number_builder [number_pos++] = 'e';
1422                                 c = get_char ();
1423
1424                                 if (c == '+'){
1425                                         if (number_pos == max_number_size)
1426                                                 Error_NumericConstantTooLong ();
1427                                         number_builder [number_pos++] = '+';
1428                                         c = -1;
1429                                 } else if (c == '-') {
1430                                         if (number_pos == max_number_size)
1431                                                 Error_NumericConstantTooLong ();
1432                                         number_builder [number_pos++] = '-';
1433                                         c = -1;
1434                                 } else {
1435                                         if (number_pos == max_number_size)
1436                                                 Error_NumericConstantTooLong ();
1437                                         number_builder [number_pos++] = '+';
1438                                 }
1439
1440                                 decimal_digits (c);
1441                                 c = get_char ();
1442                         }
1443
1444                         var type = real_type_suffix (c);
1445                         if (type == TypeCode.Empty && !is_real){
1446                                 putback (c);
1447                                 return adjust_int (c);
1448                         }
1449
1450                         is_real = true;
1451
1452                         if (type == TypeCode.Empty){
1453                                 putback (c);
1454                         }
1455
1456                         if (is_real)
1457                                 return adjust_real (type);
1458
1459                         throw new Exception ("Is Number should never reach this point");
1460                 }
1461
1462                 //
1463                 // Accepts exactly count (4 or 8) hex, no more no less
1464                 //
1465                 int getHex (int count, out int surrogate, out bool error)
1466                 {
1467                         int i;
1468                         int total = 0;
1469                         int c;
1470                         int top = count != -1 ? count : 4;
1471
1472                         get_char ();
1473                         error = false;
1474                         surrogate = 0;
1475                         for (i = 0; i < top; i++){
1476                                 c = get_char ();
1477
1478                                 if (c >= '0' && c <= '9')
1479                                         c = (int) c - (int) '0';
1480                                 else if (c >= 'A' && c <= 'F')
1481                                         c = (int) c - (int) 'A' + 10;
1482                                 else if (c >= 'a' && c <= 'f')
1483                                         c = (int) c - (int) 'a' + 10;
1484                                 else {
1485                                         error = true;
1486                                         return 0;
1487                                 }
1488
1489                                 total = (total * 16) + c;
1490                                 if (count == -1){
1491                                         int p = peek_char ();
1492                                         if (p == -1)
1493                                                 break;
1494                                         if (!is_hex ((char)p))
1495                                                 break;
1496                                 }
1497                         }
1498
1499                         if (top == 8) {
1500                                 if (total > 0x0010FFFF) {
1501                                         error = true;
1502                                         return 0;
1503                                 }
1504
1505                                 if (total >= 0x00010000) {
1506                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1507                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1508                                 }
1509                         }
1510
1511                         return total;
1512                 }
1513
1514                 int escape (int c, out int surrogate)
1515                 {
1516                         bool error;
1517                         int d;
1518                         int v;
1519
1520                         d = peek_char ();
1521                         if (c != '\\') {
1522                                 surrogate = 0;
1523                                 return c;
1524                         }
1525
1526                         switch (d){
1527                         case 'a':
1528                                 v = '\a'; break;
1529                         case 'b':
1530                                 v = '\b'; break;
1531                         case 'n':
1532                                 v = '\n'; break;
1533                         case 't':
1534                                 v = '\t'; break;
1535                         case 'v':
1536                                 v = '\v'; break;
1537                         case 'r':
1538                                 v = '\r'; break;
1539                         case '\\':
1540                                 v = '\\'; break;
1541                         case 'f':
1542                                 v = '\f'; break;
1543                         case '0':
1544                                 v = 0; break;
1545                         case '"':
1546                                 v = '"'; break;
1547                         case '\'':
1548                                 v = '\''; break;
1549                         case 'x':
1550                                 v = getHex (-1, out surrogate, out error);
1551                                 if (error)
1552                                         goto default;
1553                                 return v;
1554                         case 'u':
1555                         case 'U':
1556                                 return EscapeUnicode (d, out surrogate);
1557                         default:
1558                                 surrogate = 0;
1559                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1560                                 return d;
1561                         }
1562
1563                         get_char ();
1564                         surrogate = 0;
1565                         return v;
1566                 }
1567
1568                 int EscapeUnicode (int ch, out int surrogate)
1569                 {
1570                         bool error;
1571                         if (ch == 'U') {
1572                                 ch = getHex (8, out surrogate, out error);
1573                         } else {
1574                                 ch = getHex (4, out surrogate, out error);
1575                         }
1576
1577                         if (error)
1578                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1579
1580                         return ch;
1581                 }
1582
1583                 int get_char ()
1584                 {
1585                         int x;
1586                         if (putback_char != -1) {
1587                                 x = putback_char;
1588                                 putback_char = -1;
1589                         } else
1590                                 x = reader.Read ();
1591                         if (x == '\n') {
1592                                 advance_line ();
1593                         } else {
1594                                 col++;
1595                         }
1596                         return x;
1597                 }
1598
1599                 void advance_line ()
1600                 {
1601                         line++;
1602                         ref_line++;
1603                         previous_col = col;
1604                         col = 0;
1605                 }
1606
1607                 int peek_char ()
1608                 {
1609                         if (putback_char == -1)
1610                                 putback_char = reader.Read ();
1611                         return putback_char;
1612                 }
1613
1614                 int peek_char2 ()
1615                 {
1616                         if (putback_char != -1)
1617                                 return putback_char;
1618                         return reader.Peek ();
1619                 }
1620
1621                 void putback (int c)
1622                 {
1623                         if (putback_char != -1){
1624                                 Console.WriteLine ("Col: " + col);
1625                                 Console.WriteLine ("Row: " + line);
1626                                 Console.WriteLine ("Name: " + ref_name.Name);
1627                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1628                                 throw new Exception ("This should not happen putback on putback");
1629                         }
1630                         if (c == '\n' || col == 0) {
1631                                 // It won't happen though.
1632                                 line--;
1633                                 ref_line--;
1634                                 col = previous_col;
1635                         }
1636                         else
1637                                 col--;
1638                         putback_char = c;
1639                 }
1640
1641                 public bool advance ()
1642                 {
1643                         return peek_char () != -1 || CompleteOnEOF;
1644                 }
1645
1646                 public Object Value {
1647                         get {
1648                                 return val;
1649                         }
1650                 }
1651
1652                 public Object value ()
1653                 {
1654                         return val;
1655                 }
1656
1657                 public int token ()
1658                 {
1659                         current_token = xtoken ();
1660                         return current_token;
1661                 }
1662
1663                 void get_cmd_arg (out string cmd, out string arg)
1664                 {
1665                         int c;
1666
1667                         tokens_seen = false;
1668                         arg = "";
1669
1670                         // skip over white space
1671                         do {
1672                                 c = get_char ();
1673                         } while (c == '\r' || c == ' ' || c == '\t');
1674
1675                         static_cmd_arg.Length = 0;
1676                         while (c != -1 && is_identifier_part_character ((char)c)) {
1677                                 static_cmd_arg.Append ((char)c);
1678                                 c = get_char ();
1679                                 if (c == '\\') {
1680                                         int peek = peek_char ();
1681                                         if (peek == 'U' || peek == 'u') {
1682                                                 int surrogate;
1683                                                 c = EscapeUnicode (c, out surrogate);
1684                                                 if (surrogate != 0) {
1685                                                         if (is_identifier_part_character ((char) c))
1686                                                                 static_cmd_arg.Append ((char) c);
1687                                                         c = surrogate;
1688                                                 }
1689                                         }
1690                                 }
1691                         }
1692
1693                         cmd = static_cmd_arg.ToString ();
1694
1695                         // skip over white space
1696                         while (c == '\r' || c == ' ' || c == '\t')
1697                                 c = get_char ();
1698
1699                         static_cmd_arg.Length = 0;
1700                         int has_identifier_argument = 0;
1701
1702                         while (c != -1 && c != '\n' && c != '\r') {
1703                                 if (c == '\\' && has_identifier_argument >= 0) {
1704                                         if (has_identifier_argument != 0 || (cmd == "define" || cmd == "if" || cmd == "elif" || cmd == "undef")) {
1705                                                 has_identifier_argument = 1;
1706
1707                                                 int peek = peek_char ();
1708                                                 if (peek == 'U' || peek == 'u') {
1709                                                         int surrogate;
1710                                                         c = EscapeUnicode (c, out surrogate);
1711                                                         if (surrogate != 0) {
1712                                                                 if (is_identifier_part_character ((char) c))
1713                                                                         static_cmd_arg.Append ((char) c);
1714                                                                 c = surrogate;
1715                                                         }
1716                                                 }
1717                                         } else {
1718                                                 has_identifier_argument = -1;
1719                                         }
1720                                 }
1721                                 static_cmd_arg.Append ((char) c);
1722                                 c = get_char ();
1723                         }
1724
1725                         if (static_cmd_arg.Length != 0)
1726                                 arg = static_cmd_arg.ToString ();
1727                 }
1728
1729                 //
1730                 // Handles the #line directive
1731                 //
1732                 bool PreProcessLine (string arg)
1733                 {
1734                         if (arg.Length == 0)
1735                                 return false;
1736
1737                         if (arg == "default"){
1738                                 ref_line = line;
1739                                 ref_name = file_name;
1740                                 hidden = false;
1741                                 Location.Push (file_name, ref_name);
1742                                 return true;
1743                         } else if (arg == "hidden"){
1744                                 hidden = true;
1745                                 return true;
1746                         }
1747
1748                         try {
1749                                 int pos;
1750
1751                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1752                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1753                                         pos++;
1754
1755                                         char [] quotes = { '\"' };
1756
1757                                         string name = arg.Substring (pos). Trim (quotes);
1758                                         ref_name = Location.LookupFile (file_name, name);
1759                                         file_name.AddFile (ref_name);
1760                                         hidden = false;
1761                                         Location.Push (file_name, ref_name);
1762                                 } else {
1763                                         ref_line = System.Int32.Parse (arg);
1764                                         hidden = false;
1765                                 }
1766                         } catch {
1767                                 return false;
1768                         }
1769
1770                         return true;
1771                 }
1772
1773                 //
1774                 // Handles #define and #undef
1775                 //
1776                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
1777                 {
1778                         if (ident.Length == 0 || ident == "true" || ident == "false"){
1779                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
1780                                 return;
1781                         }
1782
1783                         if (ident.IndexOfAny (simple_whitespaces) != -1){
1784                                 Error_EndLineExpected ();
1785                                 return;
1786                         }
1787
1788                         if (!is_identifier_start_character (ident [0]))
1789                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
1790
1791                         foreach (char c in ident.Substring (1)){
1792                                 if (!is_identifier_part_character (c)){
1793                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
1794                                         return;
1795                                 }
1796                         }
1797
1798                         if (!caller_is_taking)
1799                                 return;
1800
1801                         if (is_define) {
1802                                 //
1803                                 // #define ident
1804                                 //
1805                                 if (RootContext.IsConditionalDefined (ident))
1806                                         return;
1807
1808                                 file_name.AddDefine (ident);
1809                         } else {
1810                                 //
1811                                 // #undef ident
1812                                 //
1813                                 file_name.AddUndefine (ident);
1814                         }
1815                 }
1816
1817                 static byte read_hex (string arg, int pos, out bool error)
1818                 {
1819                         error = false;
1820
1821                         int total;
1822                         char c = arg [pos];
1823
1824                         if ((c >= '0') && (c <= '9'))
1825                                 total = (int) c - (int) '0';
1826                         else if ((c >= 'A') && (c <= 'F'))
1827                                 total = (int) c - (int) 'A' + 10;
1828                         else if ((c >= 'a') && (c <= 'f'))
1829                                 total = (int) c - (int) 'a' + 10;
1830                         else {
1831                                 error = true;
1832                                 return 0;
1833                         }
1834
1835                         total *= 16;
1836                         c = arg [pos+1];
1837
1838                         if ((c >= '0') && (c <= '9'))
1839                                 total += (int) c - (int) '0';
1840                         else if ((c >= 'A') && (c <= 'F'))
1841                                 total += (int) c - (int) 'A' + 10;
1842                         else if ((c >= 'a') && (c <= 'f'))
1843                                 total += (int) c - (int) 'a' + 10;
1844                         else {
1845                                 error = true;
1846                                 return 0;
1847                         }
1848
1849                         return (byte) total;
1850                 }
1851
1852                 /// <summary>
1853                 /// Handles #pragma checksum
1854                 /// </summary>
1855                 bool PreProcessPragmaChecksum (string arg)
1856                 {
1857                         if ((arg [0] != ' ') && (arg [0] != '\t'))
1858                                 return false;
1859
1860                         arg = arg.Trim (simple_whitespaces);
1861                         if ((arg.Length < 2) || (arg [0] != '"'))
1862                                 return false;
1863
1864                         StringBuilder file_sb = new StringBuilder ();
1865
1866                         int pos = 1;
1867                         char ch;
1868                         while ((ch = arg [pos++]) != '"') {
1869                                 if (pos >= arg.Length)
1870                                         return false;
1871
1872                                 if (ch == '\\') {
1873                                         if (pos+1 >= arg.Length)
1874                                                 return false;
1875                                         ch = arg [pos++];
1876                                 }
1877
1878                                 file_sb.Append (ch);
1879                         }
1880
1881                         if ((pos+2 >= arg.Length) || ((arg [pos] != ' ') && (arg [pos] != '\t')))
1882                                 return false;
1883
1884                         arg = arg.Substring (pos).Trim (simple_whitespaces);
1885                         if ((arg.Length < 42) || (arg [0] != '"') || (arg [1] != '{') ||
1886                             (arg [10] != '-') || (arg [15] != '-') || (arg [20] != '-') ||
1887                             (arg [25] != '-') || (arg [38] != '}') || (arg [39] != '"'))
1888                                 return false;
1889
1890                         bool error;
1891                         byte[] guid_bytes = new byte [16];
1892
1893                         for (int i = 0; i < 4; i++) {
1894                                 guid_bytes [i] = read_hex (arg, 2+2*i, out error);
1895                                 if (error)
1896                                         return false;
1897                         }
1898                         for (int i = 0; i < 2; i++) {
1899                                 guid_bytes [i+4] = read_hex (arg, 11+2*i, out error);
1900                                 if (error)
1901                                         return false;
1902                                 guid_bytes [i+6] = read_hex (arg, 16+2*i, out error);
1903                                 if (error)
1904                                         return false;
1905                                 guid_bytes [i+8] = read_hex (arg, 21+2*i, out error);
1906                                 if (error)
1907                                         return false;
1908                         }
1909
1910                         for (int i = 0; i < 6; i++) {
1911                                 guid_bytes [i+10] = read_hex (arg, 26+2*i, out error);
1912                                 if (error)
1913                                         return false;
1914                         }
1915
1916                         arg = arg.Substring (40).Trim (simple_whitespaces);
1917                         if ((arg.Length < 34) || (arg [0] != '"') || (arg [33] != '"'))
1918                                 return false;
1919
1920                         byte[] checksum_bytes = new byte [16];
1921                         for (int i = 0; i < 16; i++) {
1922                                 checksum_bytes [i] = read_hex (arg, 1+2*i, out error);
1923                                 if (error)
1924                                         return false;
1925                         }
1926
1927                         arg = arg.Substring (34).Trim (simple_whitespaces);
1928                         if (arg.Length > 0)
1929                                 return false;
1930
1931                         SourceFile file = Location.LookupFile (file_name, file_sb.ToString ());
1932                         file.SetChecksum (guid_bytes, checksum_bytes);
1933                         ref_name.AutoGenerated = true;
1934                         return true;
1935                 }
1936
1937                 /// <summary>
1938                 /// Handles #pragma directive
1939                 /// </summary>
1940                 void PreProcessPragma (string arg)
1941                 {
1942                         const string warning = "warning";
1943                         const string w_disable = "warning disable";
1944                         const string w_restore = "warning restore";
1945                         const string checksum = "checksum";
1946
1947                         if (arg == w_disable) {
1948                                 Report.RegisterWarningRegion (Location).WarningDisable (Location.Row);
1949                                 return;
1950                         }
1951
1952                         if (arg == w_restore) {
1953                                 Report.RegisterWarningRegion (Location).WarningEnable (Location.Row);
1954                                 return;
1955                         }
1956
1957                         if (arg.StartsWith (w_disable)) {
1958                                 int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
1959                                 foreach (int code in codes) {
1960                                         if (code != 0)
1961                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code, Report);
1962                                 }
1963                                 return;
1964                         }
1965
1966                         if (arg.StartsWith (w_restore)) {
1967                                 int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
1968                                 var w_table = Report.warning_ignore_table;
1969                                 foreach (int code in codes) {
1970                                         if (w_table != null && w_table.ContainsKey (code))
1971                                                 Report.Warning (1635, 1, Location, "Cannot restore warning `CS{0:0000}' because it was disabled globally", code);
1972                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code, Report);
1973                                 }
1974                                 return;
1975                         }
1976
1977                         if (arg.StartsWith (warning)) {
1978                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
1979                                 return;
1980                         }
1981
1982                         if (arg.StartsWith (checksum)) {
1983                                 if (!PreProcessPragmaChecksum (arg.Substring (checksum.Length)))
1984                                         Warning_InvalidPragmaChecksum ();
1985                                 return;
1986                         }
1987
1988                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
1989                 }
1990
1991                 int[] ParseNumbers (string text)
1992                 {
1993                         string[] string_array = text.Split (',');
1994                         int[] values = new int [string_array.Length];
1995                         int index = 0;
1996                         foreach (string string_code in string_array) {
1997                                 try {
1998                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1999                                 }
2000                                 catch (FormatException) {
2001                                         Report.Warning (1692, 1, Location, "Invalid number");
2002                                 }
2003                         }
2004                         return values;
2005                 }
2006
2007                 bool eval_val (string s)
2008                 {
2009                         if (s == "true")
2010                                 return true;
2011                         if (s == "false")
2012                                 return false;
2013
2014                         return file_name.IsConditionalDefined (s);
2015                 }
2016
2017                 bool pp_primary (ref string s)
2018                 {
2019                         s = s.Trim ();
2020                         int len = s.Length;
2021
2022                         if (len > 0){
2023                                 char c = s [0];
2024
2025                                 if (c == '('){
2026                                         s = s.Substring (1);
2027                                         bool val = pp_expr (ref s, false);
2028                                         if (s.Length > 0 && s [0] == ')'){
2029                                                 s = s.Substring (1);
2030                                                 return val;
2031                                         }
2032                                         Error_InvalidDirective ();
2033                                         return false;
2034                                 }
2035
2036                                 if (is_identifier_start_character (c)){
2037                                         int j = 1;
2038
2039                                         while (j < len){
2040                                                 c = s [j];
2041
2042                                                 if (is_identifier_part_character (c)){
2043                                                         j++;
2044                                                         continue;
2045                                                 }
2046                                                 bool v = eval_val (s.Substring (0, j));
2047                                                 s = s.Substring (j);
2048                                                 return v;
2049                                         }
2050                                         bool vv = eval_val (s);
2051                                         s = "";
2052                                         return vv;
2053                                 }
2054                         }
2055                         Error_InvalidDirective ();
2056                         return false;
2057                 }
2058
2059                 bool pp_unary (ref string s)
2060                 {
2061                         s = s.Trim ();
2062                         int len = s.Length;
2063
2064                         if (len > 0){
2065                                 if (s [0] == '!'){
2066                                         if (len > 1 && s [1] == '='){
2067                                                 Error_InvalidDirective ();
2068                                                 return false;
2069                                         }
2070                                         s = s.Substring (1);
2071                                         return ! pp_primary (ref s);
2072                                 } else
2073                                         return pp_primary (ref s);
2074                         } else {
2075                                 Error_InvalidDirective ();
2076                                 return false;
2077                         }
2078                 }
2079
2080                 bool pp_eq (ref string s)
2081                 {
2082                         bool va = pp_unary (ref s);
2083
2084                         s = s.Trim ();
2085                         int len = s.Length;
2086                         if (len > 0){
2087                                 if (s [0] == '='){
2088                                         if (len > 2 && s [1] == '='){
2089                                                 s = s.Substring (2);
2090                                                 return va == pp_unary (ref s);
2091                                         } else {
2092                                                 Error_InvalidDirective ();
2093                                                 return false;
2094                                         }
2095                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2096                                         s = s.Substring (2);
2097
2098                                         return va != pp_unary (ref s);
2099
2100                                 }
2101                         }
2102
2103                         return va;
2104
2105                 }
2106
2107                 bool pp_and (ref string s)
2108                 {
2109                         bool va = pp_eq (ref s);
2110
2111                         s = s.Trim ();
2112                         int len = s.Length;
2113                         if (len > 0){
2114                                 if (s [0] == '&'){
2115                                         if (len > 2 && s [1] == '&'){
2116                                                 s = s.Substring (2);
2117                                                 return (va & pp_and (ref s));
2118                                         } else {
2119                                                 Error_InvalidDirective ();
2120                                                 return false;
2121                                         }
2122                                 }
2123                         }
2124                         return va;
2125                 }
2126
2127                 //
2128                 // Evaluates an expression for `#if' or `#elif'
2129                 //
2130                 bool pp_expr (ref string s, bool isTerm)
2131                 {
2132                         bool va = pp_and (ref s);
2133                         s = s.Trim ();
2134                         int len = s.Length;
2135                         if (len > 0){
2136                                 char c = s [0];
2137
2138                                 if (c == '|'){
2139                                         if (len > 2 && s [1] == '|'){
2140                                                 s = s.Substring (2);
2141                                                 return va | pp_expr (ref s, isTerm);
2142                                         } else {
2143                                                 Error_InvalidDirective ();
2144                                                 return false;
2145                                         }
2146                                 }
2147                                 if (isTerm) {
2148                                         Error_EndLineExpected ();
2149                                         return false;
2150                                 }
2151                         }
2152
2153                         return va;
2154                 }
2155
2156                 bool eval (string s)
2157                 {
2158                         bool v = pp_expr (ref s, true);
2159                         s = s.Trim ();
2160                         if (s.Length != 0){
2161                                 return false;
2162                         }
2163
2164                         return v;
2165                 }
2166
2167                 void Error_NumericConstantTooLong ()
2168                 {
2169                         Report.Error (1021, Location, "Numeric constant too long");
2170                 }
2171
2172                 void Error_InvalidDirective ()
2173                 {
2174                         Report.Error (1517, Location, "Invalid preprocessor directive");
2175                 }
2176
2177                 void Error_UnexpectedDirective (string extra)
2178                 {
2179                         Report.Error (
2180                                 1028, Location,
2181                                 "Unexpected processor directive ({0})", extra);
2182                 }
2183
2184                 void Error_TokensSeen ()
2185                 {
2186                         Report.Error (1032, Location,
2187                                 "Cannot define or undefine preprocessor symbols after first token in file");
2188                 }
2189
2190                 void Eror_WrongPreprocessorLocation ()
2191                 {
2192                         Report.Error (1040, Location,
2193                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2194                 }
2195
2196                 void Error_EndLineExpected ()
2197                 {
2198                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2199                 }
2200
2201                 void Warning_InvalidPragmaChecksum ()
2202                 {
2203                         Report.Warning (1695, 1, Location,
2204                                         "Invalid #pragma checksum syntax; should be " +
2205                                         "#pragma checksum \"filename\" " +
2206                                         "\"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2207                 }
2208                 //
2209                 // if true, then the code continues processing the code
2210                 // if false, the code stays in a loop until another directive is
2211                 // reached.
2212                 // When caller_is_taking is false we ignore all directives except the ones
2213                 // which can help us to identify where the #if block ends
2214                 bool handle_preprocessing_directive (bool caller_is_taking)
2215                 {
2216                         string cmd, arg;
2217                         bool region_directive = false;
2218
2219                         get_cmd_arg (out cmd, out arg);
2220
2221                         // Eat any trailing whitespaces and single-line comments
2222                         if (arg.IndexOf ("//") != -1)
2223                                 arg = arg.Substring (0, arg.IndexOf ("//"));
2224                         arg = arg.Trim (simple_whitespaces);
2225
2226                         //
2227                         // The first group of pre-processing instructions is always processed
2228                         //
2229                         switch (cmd){
2230                         case "region":
2231                                 region_directive = true;
2232                                 arg = "true";
2233                                 goto case "if";
2234
2235                         case "endregion":
2236                                 if (ifstack == null || ifstack.Count == 0){
2237                                         Error_UnexpectedDirective ("no #region for this #endregion");
2238                                         return true;
2239                                 }
2240                                 int pop = ifstack.Pop ();
2241
2242                                 if ((pop & REGION) == 0)
2243                                         Report.Error (1027, Location, "Expected `#endif' directive");
2244
2245                                 return caller_is_taking;
2246
2247                         case "if":
2248                                 if (ifstack == null)
2249                                         ifstack = new Stack<int> (2);
2250
2251                                 int flags = region_directive ? REGION : 0;
2252                                 if (ifstack.Count == 0){
2253                                         flags |= PARENT_TAKING;
2254                                 } else {
2255                                         int state = ifstack.Peek ();
2256                                         if ((state & TAKING) != 0) {
2257                                                 flags |= PARENT_TAKING;
2258                                         }
2259                                 }
2260
2261                                 if (caller_is_taking && eval (arg)) {
2262                                         ifstack.Push (flags | TAKING);
2263                                         return true;
2264                                 }
2265                                 ifstack.Push (flags);
2266                                 return false;
2267
2268                         case "endif":
2269                                 if (ifstack == null || ifstack.Count == 0){
2270                                         Error_UnexpectedDirective ("no #if for this #endif");
2271                                         return true;
2272                                 } else {
2273                                         pop = ifstack.Pop ();
2274
2275                                         if ((pop & REGION) != 0)
2276                                                 Report.Error (1038, Location, "#endregion directive expected");
2277
2278                                         if (arg.Length != 0) {
2279                                                 Error_EndLineExpected ();
2280                                         }
2281
2282                                         if (ifstack.Count == 0)
2283                                                 return true;
2284
2285                                         int state = ifstack.Peek ();
2286                                         return (state & TAKING) != 0;
2287                                 }
2288
2289                         case "elif":
2290                                 if (ifstack == null || ifstack.Count == 0){
2291                                         Error_UnexpectedDirective ("no #if for this #elif");
2292                                         return true;
2293                                 } else {
2294                                         int state = ifstack.Pop ();
2295
2296                                         if ((state & REGION) != 0) {
2297                                                 Report.Error (1038, Location, "#endregion directive expected");
2298                                                 return true;
2299                                         }
2300
2301                                         if ((state & ELSE_SEEN) != 0){
2302                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2303                                                 return true;
2304                                         }
2305
2306                                         if ((state & TAKING) != 0) {
2307                                                 ifstack.Push (0);
2308                                                 return false;
2309                                         }
2310
2311                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2312                                                 ifstack.Push (state | TAKING);
2313                                                 return true;
2314                                         }
2315
2316                                         ifstack.Push (state);
2317                                         return false;
2318                                 }
2319
2320                         case "else":
2321                                 if (ifstack == null || ifstack.Count == 0){
2322                                         Error_UnexpectedDirective ("no #if for this #else");
2323                                         return true;
2324                                 } else {
2325                                         int state = ifstack.Peek ();
2326
2327                                         if ((state & REGION) != 0) {
2328                                                 Report.Error (1038, Location, "#endregion directive expected");
2329                                                 return true;
2330                                         }
2331
2332                                         if ((state & ELSE_SEEN) != 0){
2333                                                 Error_UnexpectedDirective ("#else within #else");
2334                                                 return true;
2335                                         }
2336
2337                                         ifstack.Pop ();
2338
2339                                         if (arg.Length != 0) {
2340                                                 Error_EndLineExpected ();
2341                                                 return true;
2342                                         }
2343
2344                                         bool ret = false;
2345                                         if ((state & PARENT_TAKING) != 0) {
2346                                                 ret = (state & TAKING) == 0;
2347
2348                                                 if (ret)
2349                                                         state |= TAKING;
2350                                                 else
2351                                                         state &= ~TAKING;
2352                                         }
2353
2354                                         ifstack.Push (state | ELSE_SEEN);
2355
2356                                         return ret;
2357                                 }
2358                         case "define":
2359                                 if (any_token_seen){
2360                                         Error_TokensSeen ();
2361                                         return caller_is_taking;
2362                                 }
2363                                 PreProcessDefinition (true, arg, caller_is_taking);
2364                                 return caller_is_taking;
2365
2366                         case "undef":
2367                                 if (any_token_seen){
2368                                         Error_TokensSeen ();
2369                                         return caller_is_taking;
2370                                 }
2371                                 PreProcessDefinition (false, arg, caller_is_taking);
2372                                 return caller_is_taking;
2373                         }
2374
2375                         //
2376                         // These are only processed if we are in a `taking' block
2377                         //
2378                         if (!caller_is_taking)
2379                                 return false;
2380
2381                         switch (cmd){
2382                         case "error":
2383                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2384                                 return true;
2385
2386                         case "warning":
2387                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2388                                 return true;
2389
2390                         case "pragma":
2391                                 if (RootContext.Version == LanguageVersion.ISO_1) {
2392                                         Report.FeatureIsNotAvailable (Location, "#pragma");
2393                                         return true;
2394                                 }
2395
2396                                 PreProcessPragma (arg);
2397                                 return true;
2398
2399                         case "line":
2400                                 if (!PreProcessLine (arg))
2401                                         Report.Error (
2402                                                 1576, Location,
2403                                                 "The line number specified for #line directive is missing or invalid");
2404                                 return caller_is_taking;
2405                         }
2406
2407                         Report.Error (1024, Location, "Wrong preprocessor directive");
2408                         return true;
2409
2410                 }
2411
2412                 private int consume_string (bool quoted)
2413                 {
2414                         int c;
2415                         string_builder.Length = 0;
2416
2417                         while ((c = get_char ()) != -1){
2418                                 if (c == '"'){
2419                                         if (quoted && peek_char () == '"'){
2420                                                 string_builder.Append ((char) c);
2421                                                 get_char ();
2422                                                 continue;
2423                                         } else {
2424                                                 val = new StringLiteral (string_builder.ToString (), Location);
2425                                                 return Token.LITERAL;
2426                                         }
2427                                 }
2428
2429                                 if (c == '\n'){
2430                                         if (!quoted)
2431                                                 Report.Error (1010, Location, "Newline in constant");
2432                                 }
2433
2434                                 if (!quoted){
2435                                         int surrogate;
2436                                         c = escape (c, out surrogate);
2437                                         if (c == -1)
2438                                                 return Token.ERROR;
2439                                         if (surrogate != 0) {
2440                                                 string_builder.Append ((char) c);
2441                                                 c = surrogate;
2442                                         }
2443                                 }
2444                                 string_builder.Append ((char) c);
2445                         }
2446
2447                         Report.Error (1039, Location, "Unterminated string literal");
2448                         return Token.EOF;
2449                 }
2450
2451                 private int consume_identifier (int s)
2452                 {
2453                         int res = consume_identifier (s, false);
2454
2455                         if (doc_state == XmlCommentState.Allowed)
2456                                 doc_state = XmlCommentState.NotAllowed;
2457
2458                         return res;
2459                 }
2460
2461                 int consume_identifier (int c, bool quoted)
2462                 {
2463                         //
2464                         // This method is very performance sensitive. It accounts
2465                         // for approximately 25% of all parser time
2466                         //
2467
2468                         int pos = 0;
2469                         int column = col;
2470
2471                         if (c == '\\') {
2472                                 int surrogate;
2473                                 c = escape (c, out surrogate);
2474                                 if (surrogate != 0) {
2475                                         id_builder [pos++] = (char) c;
2476                                         c = surrogate;
2477                                 }
2478                         }
2479
2480                         id_builder [pos++] = (char) c;
2481
2482                         try {
2483                                 while (true) {
2484                                         c = reader.Read ();
2485
2486                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
2487                                                 id_builder [pos++] = (char) c;
2488                                                 continue;
2489                                         }
2490
2491                                         if (c < 0x80) {
2492                                                 if (c == '\\') {
2493                                                         int surrogate;
2494                                                         c = escape (c, out surrogate);
2495                                                         if (surrogate != 0) {
2496                                                                 if (is_identifier_part_character ((char) c))
2497                                                                         id_builder[pos++] = (char) c;
2498                                                                 c = surrogate;
2499                                                         }
2500
2501                                                         continue;
2502                                                 }
2503                                         } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
2504                                                 id_builder [pos++] = (char) c;
2505                                                 continue;
2506                                         }
2507
2508                                         putback_char = c;
2509                                         break;
2510                                 }
2511                         } catch (IndexOutOfRangeException) {
2512                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2513                                 col += pos - 1;
2514                                 return Token.ERROR;
2515                         }
2516
2517                         col += pos - 1;
2518
2519                         //
2520                         // Optimization: avoids doing the keyword lookup
2521                         // on uppercase letters
2522                         //
2523                         if (id_builder [0] >= '_' && !quoted) {
2524                                 int keyword = GetKeyword (id_builder, pos);
2525                                 if (keyword != -1) {
2526                                         val = LocatedToken.Create (null, ref_line, column);
2527                                         return keyword;
2528                                 }
2529                         }
2530
2531                         //
2532                         // Keep identifiers in an array of hashtables to avoid needless
2533                         // allocations
2534                         //
2535                         var identifiers_group = identifiers [pos];
2536                         string s;
2537                         if (identifiers_group != null) {
2538                                 if (identifiers_group.TryGetValue (id_builder, out s)) {
2539                                         val = LocatedToken.Create (s, ref_line, column);
2540                                         if (quoted)
2541                                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2542                                         return Token.IDENTIFIER;
2543                                 }
2544                         } else {
2545                                 // TODO: this should be number of files dependant
2546                                 // corlib compilation peaks at 1000 and System.Core at 150
2547                                 int capacity = pos > 20 ? 10 : 100;
2548                                 identifiers_group = new Dictionary<char[],string> (capacity, new IdentifiersComparer (pos));
2549                                 identifiers [pos] = identifiers_group;
2550                         }
2551
2552                         char [] chars = new char [pos];
2553                         Array.Copy (id_builder, chars, pos);
2554
2555                         s = new string (id_builder, 0, pos);
2556                         identifiers_group.Add (chars, s);
2557
2558                         val = LocatedToken.Create (s, ref_line, column);
2559                         if (quoted)
2560                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2561
2562                         return Token.IDENTIFIER;
2563                 }
2564
2565                 public int xtoken ()
2566                 {
2567                         int d, c;
2568
2569                         // Whether we have seen comments on the current line
2570                         bool comments_seen = false;
2571                         while ((c = get_char ()) != -1) {
2572                                 switch (c) {
2573                                 case '\t':
2574                                         col = ((col + 8) / 8) * 8;
2575                                         continue;
2576
2577                                 case ' ':
2578                                 case '\f':
2579                                 case '\v':
2580                                 case 0xa0:
2581                                 case 0:
2582                                 case 0xFEFF:    // Ignore BOM anywhere in the file
2583                                         continue;
2584
2585 /*                              This is required for compatibility with .NET
2586                                 case 0xEF:
2587                                         if (peek_char () == 0xBB) {
2588                                                 PushPosition ();
2589                                                 get_char ();
2590                                                 if (get_char () == 0xBF)
2591                                                         continue;
2592                                                 PopPosition ();
2593                                         }
2594                                         break;
2595 */
2596                                 case '\r':
2597                                         if (peek_char () != '\n')
2598                                                 advance_line ();
2599                                         else
2600                                                 get_char ();
2601
2602                                         any_token_seen |= tokens_seen;
2603                                         tokens_seen = false;
2604                                         comments_seen = false;
2605                                         continue;
2606
2607                                 case '\\':
2608                                         tokens_seen = true;
2609                                         return consume_identifier (c);
2610
2611                                 case '{':
2612                                         val = LocatedToken.Create (ref_line, col);
2613                                         return Token.OPEN_BRACE;
2614                                 case '}':
2615                                         val = LocatedToken.Create (ref_line, col);
2616                                         return Token.CLOSE_BRACE;
2617                                 case '[':
2618                                         // To block doccomment inside attribute declaration.
2619                                         if (doc_state == XmlCommentState.Allowed)
2620                                                 doc_state = XmlCommentState.NotAllowed;
2621                                         val = LocatedToken.CreateOptional (ref_line, col);
2622                                         return Token.OPEN_BRACKET;
2623                                 case ']':
2624                                         val = LocatedToken.CreateOptional (ref_line, col);
2625                                         return Token.CLOSE_BRACKET;
2626                                 case '(':
2627                                         val = LocatedToken.Create (ref_line, col);
2628                                         //
2629                                         // An expression versions of parens can appear in block context only
2630                                         //
2631                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
2632
2633                                                 //
2634                                                 // Optmize most common case where we know that parens
2635                                                 // is not special
2636                                                 //
2637                                                 switch (current_token) {
2638                                                 case Token.IDENTIFIER:
2639                                                 case Token.IF:
2640                                                 case Token.FOR:
2641                                                 case Token.FOREACH:
2642                                                 case Token.TYPEOF:
2643                                                 case Token.WHILE:
2644                                                 case Token.USING:
2645                                                 case Token.DEFAULT:
2646                                                 case Token.DELEGATE:
2647                                                 case Token.OP_GENERICS_GT:
2648                                                         return Token.OPEN_PARENS;
2649                                                 }
2650
2651                                                 // Optimize using peek
2652                                                 int xx = peek_char ();
2653                                                 switch (xx) {
2654                                                 case '(':
2655                                                 case '\'':
2656                                                 case '"':
2657                                                 case '0':
2658                                                 case '1':
2659                                                         return Token.OPEN_PARENS;
2660                                                 }
2661
2662                                                 lambda_arguments_parsing = true;
2663                                                 PushPosition ();
2664                                                 d = TokenizeOpenParens ();
2665                                                 PopPosition ();
2666                                                 lambda_arguments_parsing = false;
2667                                                 return d;
2668                                         }
2669
2670                                         return Token.OPEN_PARENS;
2671                                 case ')':
2672                                         val = LocatedToken.CreateOptional (ref_line, col);
2673                                         return Token.CLOSE_PARENS;
2674                                 case ',':
2675                                         val = LocatedToken.CreateOptional (ref_line, col);
2676                                         return Token.COMMA;
2677                                 case ';':
2678                                         val = LocatedToken.CreateOptional (ref_line, col);
2679                                         return Token.SEMICOLON;
2680                                 case '~':
2681                                         val = LocatedToken.Create (ref_line, col);
2682                                         return Token.TILDE;
2683                                 case '?':
2684                                         val = LocatedToken.Create (ref_line, col);
2685                                         return TokenizePossibleNullableType ();
2686                                 case '<':
2687                                         val = LocatedToken.Create (ref_line, col);
2688                                         if (parsing_generic_less_than++ > 0)
2689                                                 return Token.OP_GENERICS_LT;
2690
2691                                         return TokenizeLessThan ();
2692
2693                                 case '>':
2694                                         val = LocatedToken.Create (ref_line, col);
2695                                         d = peek_char ();
2696
2697                                         if (d == '='){
2698                                                 get_char ();
2699                                                 return Token.OP_GE;
2700                                         }
2701
2702                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
2703                                                 parsing_generic_less_than--;
2704                                                 return Token.OP_GENERICS_GT;
2705                                         }
2706
2707                                         if (d == '>') {
2708                                                 get_char ();
2709                                                 d = peek_char ();
2710
2711                                                 if (d == '=') {
2712                                                         get_char ();
2713                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
2714                                                 }
2715                                                 return Token.OP_SHIFT_RIGHT;
2716                                         }
2717
2718                                         return Token.OP_GT;
2719
2720                                 case '+':
2721                                         val = LocatedToken.Create (ref_line, col);
2722                                         d = peek_char ();
2723                                         if (d == '+') {
2724                                                 d = Token.OP_INC;
2725                                         } else if (d == '=') {
2726                                                 d = Token.OP_ADD_ASSIGN;
2727                                         } else {
2728                                                 return Token.PLUS;
2729                                         }
2730                                         get_char ();
2731                                         return d;
2732
2733                                 case '-':
2734                                         val = LocatedToken.Create (ref_line, col);
2735                                         d = peek_char ();
2736                                         if (d == '-') {
2737                                                 d = Token.OP_DEC;
2738                                         } else if (d == '=')
2739                                                 d = Token.OP_SUB_ASSIGN;
2740                                         else if (d == '>')
2741                                                 d = Token.OP_PTR;
2742                                         else {
2743                                                 return Token.MINUS;
2744                                         }
2745                                         get_char ();
2746                                         return d;
2747
2748                                 case '!':
2749                                         val = LocatedToken.Create (ref_line, col);
2750                                         if (peek_char () == '='){
2751                                                 get_char ();
2752                                                 return Token.OP_NE;
2753                                         }
2754                                         return Token.BANG;
2755
2756                                 case '=':
2757                                         val = LocatedToken.Create (ref_line, col);
2758                                         d = peek_char ();
2759                                         if (d == '='){
2760                                                 get_char ();
2761                                                 return Token.OP_EQ;
2762                                         }
2763                                         if (d == '>'){
2764                                                 get_char ();
2765                                                 return Token.ARROW;
2766                                         }
2767
2768                                         return Token.ASSIGN;
2769
2770                                 case '&':
2771                                         val = LocatedToken.Create (ref_line, col);
2772                                         d = peek_char ();
2773                                         if (d == '&'){
2774                                                 get_char ();
2775                                                 return Token.OP_AND;
2776                                         }
2777                                         if (d == '='){
2778                                                 get_char ();
2779                                                 return Token.OP_AND_ASSIGN;
2780                                         }
2781                                         return Token.BITWISE_AND;
2782
2783                                 case '|':
2784                                         val = LocatedToken.Create (ref_line, col);
2785                                         d = peek_char ();
2786                                         if (d == '|'){
2787                                                 get_char ();
2788                                                 return Token.OP_OR;
2789                                         }
2790                                         if (d == '='){
2791                                                 get_char ();
2792                                                 return Token.OP_OR_ASSIGN;
2793                                         }
2794                                         return Token.BITWISE_OR;
2795
2796                                 case '*':
2797                                         val = LocatedToken.Create (ref_line, col);
2798                                         if (peek_char () == '='){
2799                                                 get_char ();
2800                                                 return Token.OP_MULT_ASSIGN;
2801                                         }
2802                                         return Token.STAR;
2803
2804                                 case '/':
2805                                         d = peek_char ();
2806                                         if (d == '='){
2807                                                 val = LocatedToken.Create (ref_line, col);
2808                                                 get_char ();
2809                                                 return Token.OP_DIV_ASSIGN;
2810                                         }
2811
2812                                         // Handle double-slash comments.
2813                                         if (d == '/'){
2814                                                 get_char ();
2815                                                 if (RootContext.Documentation != null && peek_char () == '/') {
2816                                                         get_char ();
2817                                                         // Don't allow ////.
2818                                                         if ((d = peek_char ()) != '/') {
2819                                                                 update_comment_location ();
2820                                                                 if (doc_state == XmlCommentState.Allowed)
2821                                                                         handle_one_line_xml_comment ();
2822                                                                 else if (doc_state == XmlCommentState.NotAllowed)
2823                                                                         warn_incorrect_doc_comment ();
2824                                                         }
2825                                                 }
2826                                                 while ((d = get_char ()) != -1 && (d != '\n') && d != '\r');
2827
2828                                                 any_token_seen |= tokens_seen;
2829                                                 tokens_seen = false;
2830                                                 comments_seen = false;
2831                                                 continue;
2832                                         } else if (d == '*'){
2833                                                 get_char ();
2834                                                 bool docAppend = false;
2835                                                 if (RootContext.Documentation != null && peek_char () == '*') {
2836                                                         get_char ();
2837                                                         update_comment_location ();
2838                                                         // But when it is /**/, just do nothing.
2839                                                         if (peek_char () == '/') {
2840                                                                 get_char ();
2841                                                                 continue;
2842                                                         }
2843                                                         if (doc_state == XmlCommentState.Allowed)
2844                                                                 docAppend = true;
2845                                                         else if (doc_state == XmlCommentState.NotAllowed)
2846                                                                 warn_incorrect_doc_comment ();
2847                                                 }
2848
2849                                                 int current_comment_start = 0;
2850                                                 if (docAppend) {
2851                                                         current_comment_start = xml_comment_buffer.Length;
2852                                                         xml_comment_buffer.Append (Environment.NewLine);
2853                                                 }
2854
2855                                                 while ((d = get_char ()) != -1){
2856                                                         if (d == '*' && peek_char () == '/'){
2857                                                                 get_char ();
2858                                                                 comments_seen = true;
2859                                                                 break;
2860                                                         }
2861                                                         if (docAppend)
2862                                                                 xml_comment_buffer.Append ((char) d);
2863
2864                                                         if (d == '\n'){
2865                                                                 any_token_seen |= tokens_seen;
2866                                                                 tokens_seen = false;
2867                                                                 //
2868                                                                 // Reset 'comments_seen' just to be consistent.
2869                                                                 // It doesn't matter either way, here.
2870                                                                 //
2871                                                                 comments_seen = false;
2872                                                         }
2873                                                 }
2874                                                 if (!comments_seen)
2875                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
2876
2877                                                 if (docAppend)
2878                                                         update_formatted_doc_comment (current_comment_start);
2879                                                 continue;
2880                                         }
2881                                         return Token.DIV;
2882
2883                                 case '%':
2884                                         val = LocatedToken.Create (ref_line, col);
2885                                         if (peek_char () == '='){
2886                                                 get_char ();
2887                                                 return Token.OP_MOD_ASSIGN;
2888                                         }
2889                                         return Token.PERCENT;
2890
2891                                 case '^':
2892                                         val = LocatedToken.Create (ref_line, col);
2893                                         if (peek_char () == '='){
2894                                                 get_char ();
2895                                                 return Token.OP_XOR_ASSIGN;
2896                                         }
2897                                         return Token.CARRET;
2898
2899                                 case ':':
2900                                         val = LocatedToken.Create (ref_line, col);
2901                                         if (peek_char () == ':') {
2902                                                 get_char ();
2903                                                 return Token.DOUBLE_COLON;
2904                                         }
2905                                         return Token.COLON;
2906
2907                                 case '0': case '1': case '2': case '3': case '4':
2908                                 case '5': case '6': case '7': case '8': case '9':
2909                                         tokens_seen = true;
2910                                         return is_number (c);
2911
2912                                 case '\n': // white space
2913                                         any_token_seen |= tokens_seen;
2914                                         tokens_seen = false;
2915                                         comments_seen = false;
2916                                         continue;
2917
2918                                 case '.':
2919                                         tokens_seen = true;
2920                                         d = peek_char ();
2921                                         if (d >= '0' && d <= '9')
2922                                                 return is_number (c);
2923                                         return Token.DOT;
2924
2925                                 case '#':
2926                                         if (tokens_seen || comments_seen) {
2927                                                 Eror_WrongPreprocessorLocation ();
2928                                                 return Token.ERROR;
2929                                         }
2930
2931                                         if (handle_preprocessing_directive (true))
2932                                                 continue;
2933
2934                                         bool directive_expected = false;
2935                                         while ((c = get_char ()) != -1) {
2936                                                 if (col == 1) {
2937                                                         directive_expected = true;
2938                                                 } else if (!directive_expected) {
2939                                                         // TODO: Implement comment support for disabled code and uncomment this code
2940 //                                                      if (c == '#') {
2941 //                                                              Eror_WrongPreprocessorLocation ();
2942 //                                                              return Token.ERROR;
2943 //                                                      }
2944                                                         continue;
2945                                                 }
2946
2947                                                 if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
2948                                                         continue;
2949
2950                                                 if (c == '#') {
2951                                                         if (handle_preprocessing_directive (false))
2952                                                                 break;
2953                                                 }
2954                                                 directive_expected = false;
2955                                         }
2956
2957                                         if (c != -1) {
2958                                                 tokens_seen = false;
2959                                                 continue;
2960                                         }
2961
2962                                         return Token.EOF;
2963
2964                                 case '"':
2965                                         return consume_string (false);
2966
2967                                 case '\'':
2968                                         return TokenizeBackslash ();
2969
2970                                 case '@':
2971                                         c = get_char ();
2972                                         if (c == '"') {
2973                                                 tokens_seen = true;
2974                                                 return consume_string (true);
2975                                         }
2976
2977                                         if (is_identifier_start_character (c)){
2978                                                 return consume_identifier (c, true);
2979                                         }
2980
2981                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
2982                                         return Token.ERROR;
2983
2984                                 case EvalStatementParserCharacter:
2985                                         return Token.EVAL_STATEMENT_PARSER;
2986                                 case EvalCompilationUnitParserCharacter:
2987                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
2988                                 case EvalUsingDeclarationsParserCharacter:
2989                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
2990                                 }
2991
2992                                 if (is_identifier_start_character (c)) {
2993                                         tokens_seen = true;
2994                                         return consume_identifier (c);
2995                                 }
2996
2997                                 error_details = ((char)c).ToString ();
2998                                 return Token.ERROR;
2999                         }
3000
3001                         if (CompleteOnEOF){
3002                                 if (generated)
3003                                         return Token.COMPLETE_COMPLETION;
3004
3005                                 generated = true;
3006                                 return Token.GENERATE_COMPLETION;
3007                         }
3008
3009
3010                         return Token.EOF;
3011                 }
3012
3013                 int TokenizeBackslash ()
3014                 {
3015                         int c = get_char ();
3016                         tokens_seen = true;
3017                         if (c == '\'') {
3018                                 error_details = "Empty character literal";
3019                                 Report.Error (1011, Location, error_details);
3020                                 return Token.ERROR;
3021                         }
3022                         if (c == '\r' || c == '\n') {
3023                                 Report.Error (1010, Location, "Newline in constant");
3024                                 return Token.ERROR;
3025                         }
3026
3027                         int d;
3028                         c = escape (c, out d);
3029                         if (c == -1)
3030                                 return Token.ERROR;
3031                         if (d != 0)
3032                                 throw new NotImplementedException ();
3033
3034                         val = new CharLiteral ((char) c, Location);
3035                         c = get_char ();
3036
3037                         if (c != '\'') {
3038                                 Report.Error (1012, Location, "Too many characters in character literal");
3039
3040                                 // Try to recover, read until newline or next "'"
3041                                 while ((c = get_char ()) != -1) {
3042                                         if (c == '\n' || c == '\'')
3043                                                 break;
3044                                 }
3045                                 return Token.ERROR;
3046                         }
3047
3048                         return Token.LITERAL;
3049                 }
3050
3051                 int TokenizeLessThan ()
3052                 {
3053                         int d;
3054                         if (handle_typeof) {
3055                                 PushPosition ();
3056                                 if (parse_generic_dimension (out d)) {
3057                                         val = d;
3058                                         DiscardPosition ();
3059                                         return Token.GENERIC_DIMENSION;
3060                                 }
3061                                 PopPosition ();
3062                         }
3063
3064                         // Save current position and parse next token.
3065                         PushPosition ();
3066                         if (parse_less_than ()) {
3067                                 if (parsing_generic_declaration && token () != Token.DOT) {
3068                                         d = Token.OP_GENERICS_LT_DECL;
3069                                 } else {
3070                                         d = Token.OP_GENERICS_LT;
3071                                 }
3072                                 PopPosition ();
3073                                 return d;
3074                         }
3075
3076                         PopPosition ();
3077                         parsing_generic_less_than = 0;
3078
3079                         d = peek_char ();
3080                         if (d == '<') {
3081                                 get_char ();
3082                                 d = peek_char ();
3083
3084                                 if (d == '=') {
3085                                         get_char ();
3086                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3087                                 }
3088                                 return Token.OP_SHIFT_LEFT;
3089                         }
3090
3091                         if (d == '=') {
3092                                 get_char ();
3093                                 return Token.OP_LE;
3094                         }
3095                         return Token.OP_LT;
3096                 }
3097
3098                 //
3099                 // Handles one line xml comment
3100                 //
3101                 private void handle_one_line_xml_comment ()
3102                 {
3103                         int c;
3104                         while ((c = peek_char ()) == ' ')
3105                                 get_char (); // skip heading whitespaces.
3106                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3107                                 xml_comment_buffer.Append ((char) get_char ());
3108                         }
3109                         if (c == '\r' || c == '\n')
3110                                 xml_comment_buffer.Append (Environment.NewLine);
3111                 }
3112
3113                 //
3114                 // Remove heading "*" in Javadoc-like xml documentation.
3115                 //
3116                 private void update_formatted_doc_comment (int current_comment_start)
3117                 {
3118                         int length = xml_comment_buffer.Length - current_comment_start;
3119                         string [] lines = xml_comment_buffer.ToString (
3120                                 current_comment_start,
3121                                 length).Replace ("\r", "").Split ('\n');
3122
3123                         // The first line starts with /**, thus it is not target
3124                         // for the format check.
3125                         for (int i = 1; i < lines.Length; i++) {
3126                                 string s = lines [i];
3127                                 int idx = s.IndexOf ('*');
3128                                 string head = null;
3129                                 if (idx < 0) {
3130                                         if (i < lines.Length - 1)
3131                                                 return;
3132                                         head = s;
3133                                 } else
3134                                         head = s.Substring (0, idx);
3135                                 foreach (char c in head)
3136                                         if (c != ' ')
3137                                                 return;
3138                                 lines [i] = s.Substring (idx + 1);
3139                         }
3140                         xml_comment_buffer.Remove (current_comment_start, length);
3141                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3142                 }
3143
3144                 //
3145                 // Updates current comment location.
3146                 //
3147                 private void update_comment_location ()
3148                 {
3149                         if (current_comment_location.IsNull) {
3150                                 // "-2" is for heading "//" or "/*"
3151                                 current_comment_location =
3152                                         new Location (ref_line, hidden ? -1 : col - 2);
3153                         }
3154                 }
3155
3156                 //
3157                 // Checks if there was incorrect doc comments and raise
3158                 // warnings.
3159                 //
3160                 public void check_incorrect_doc_comment ()
3161                 {
3162                         if (xml_comment_buffer.Length > 0)
3163                                 warn_incorrect_doc_comment ();
3164                 }
3165
3166                 //
3167                 // Raises a warning when tokenizer found incorrect doccomment
3168                 // markup.
3169                 //
3170                 private void warn_incorrect_doc_comment ()
3171                 {
3172                         if (doc_state != XmlCommentState.Error) {
3173                                 doc_state = XmlCommentState.Error;
3174                                 // in csc, it is 'XML comment is not placed on
3175                                 // a valid language element'. But that does not
3176                                 // make sense.
3177                                 Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
3178                         }
3179                 }
3180
3181                 //
3182                 // Consumes the saved xml comment lines (if any)
3183                 // as for current target member or type.
3184                 //
3185                 public string consume_doc_comment ()
3186                 {
3187                         if (xml_comment_buffer.Length > 0) {
3188                                 string ret = xml_comment_buffer.ToString ();
3189                                 reset_doc_comment ();
3190                                 return ret;
3191                         }
3192                         return null;
3193                 }
3194
3195                 Report Report {
3196                         get { return context.Report; }
3197                 }
3198
3199                 void reset_doc_comment ()
3200                 {
3201                         xml_comment_buffer.Length = 0;
3202                         current_comment_location = Location.Null;
3203                 }
3204
3205                 public void cleanup ()
3206                 {
3207                         if (ifstack != null && ifstack.Count >= 1) {
3208                                 int state = ifstack.Pop ();
3209                                 if ((state & REGION) != 0)
3210                                         Report.Error (1038, Location, "#endregion directive expected");
3211                                 else
3212                                         Report.Error (1027, Location, "Expected `#endif' directive");
3213                         }
3214                 }
3215         }
3216
3217         //
3218         // Indicates whether it accepts XML documentation or not.
3219         //
3220         public enum XmlCommentState {
3221                 // comment is allowed in this state.
3222                 Allowed,
3223                 // comment is not allowed in this state.
3224                 NotAllowed,
3225                 // once comments appeared when it is NotAllowed, then the
3226                 // state is changed to it, until the state is changed to
3227                 // .Allowed.
3228                 Error
3229         }
3230 }
3231