mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 //
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.IO;
  19 using System.Globalization;
  20 using System.Reflection;
  21
  22 namespace Mono.CSharp
  23 {
  24         /// <summary>
  25         ///    Tokenizer for C# source code.
  26         /// </summary>
  27
  28         public class Tokenizer : yyParser.yyInput
  29         {
  30                 class KeywordEntry
  31                 {
  32                         public readonly int Token;
  33                         public KeywordEntry Next;
  34                         public readonly char[] Value;
  35
  36                         public KeywordEntry (string value, int token)
  37                         {
  38                                 this.Value = value.ToCharArray ();
  39                                 this.Token = token;
  40                         }
  41                 }
  42
  43                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  44                 {
  45                         readonly int length;
  46
  47                         public IdentifiersComparer (int length)
  48                         {
  49                                 this.length = length;
  50                         }
  51
  52                         public bool Equals (char[] x, char[] y)
  53                         {
  54                                 for (int i = 0; i < length; ++i)
  55                                         if (x [i] != y [i])
  56                                                 return false;
  57
  58                                 return true;
  59                         }
  60
  61                         public int GetHashCode (char[] obj)
  62                         {
  63                                 int h = 0;
  64                                 for (int i = 0; i < length; ++i)
  65                                         h = (h << 5) - h + obj [i];
  66
  67                                 return h;
  68                         }
  69                 }
  70
  71                 //
  72                 // This class has to be used in the parser only, it reuses token
  73                 // details after each parse
  74                 //
  75                 public class LocatedToken
  76                 {
  77                         int row, column;
  78                         string value;
  79
  80                         static LocatedToken[] buffer;
  81                         static int pos;
  82
  83                         private LocatedToken ()
  84                         {
  85                         }
  86
  87                         public static LocatedToken Create (int row, int column)
  88                         {
  89                                 return Create (null, row, column);
  90                         }
  91
  92                         public static LocatedToken Create (string value, int row, int column)
  93                         {
  94                                 //
  95                                 // TODO: I am not very happy about the logic but it's the best
  96                                 // what I could come up with for now.
  97                                 // Ideally we should be using just tiny buffer (256 elements) which
  98                                 // is enough to hold all details for currect stack and recycle elements
  99                                 // poped from the stack but there is a trick needed to recycle
 100                                 // them properly.
 101                                 //
 102                                 LocatedToken entry;
 103                                 if (pos >= buffer.Length) {
 104                                         entry = new LocatedToken ();
 105                                 } else {
 106                                         entry = buffer [pos];
 107                                         if (entry == null) {
 108                                                 entry = new LocatedToken ();
 109                                                 buffer [pos] = entry;
 110                                         }
 111
 112                                         ++pos;
 113                                 }
 114                                 entry.value = value;
 115                                 entry.row = row;
 116                                 entry.column = column;
 117                                 return entry;
 118                         }
 119
 120                         public static void Initialize ()
 121                         {
 122                                 if (buffer == null)
 123                                         buffer = new LocatedToken [10000];
 124                                 pos = 0;
 125                         }
 126
 127                         public Location Location {
 128                                 get { return new Location (row, column); }
 129                         }
 130
 131                         public string Value {
 132                                 get { return value; }
 133                         }
 134                 }
 135
 136                 SeekableStreamReader reader;
 137                 SourceFile ref_name;
 138                 CompilationUnit file_name;
 139                 CompilerContext context;
 140                 bool hidden = false;
 141                 int ref_line = 1;
 142                 int line = 1;
 143                 int col = 0;
 144                 int previous_col;
 145                 int current_token;
 146                 bool handle_get_set = false;
 147                 bool handle_remove_add = false;
 148                 bool handle_where = false;
 149                 bool handle_typeof = false;
 150                 bool lambda_arguments_parsing;
 151                 Location current_comment_location = Location.Null;
 152                 List<Location> escaped_identifiers;
 153                 int parsing_generic_less_than;
 154
 155                 //
 156                 // Used mainly for parser optimizations. Some expressions for instance
 157                 // can appear only in block (including initializer, base initializer)
 158                 // scope only
 159                 //
 160                 public int parsing_block;
 161                 internal bool query_parsing;
 162
 163                 //
 164                 // When parsing type only, useful for ambiguous nullable types
 165                 //
 166                 public int parsing_type;
 167
 168                 //
 169                 // Set when parsing generic declaration (type or method header)
 170                 //
 171                 public bool parsing_generic_declaration;
 172
 173                 //
 174                 // The value indicates that we have not reach any declaration or
 175                 // namespace yet
 176                 //
 177                 public int parsing_declaration;
 178
 179                 //
 180                 // The special character to inject on streams to trigger the EXPRESSION_PARSE
 181                 // token to be returned.   It just happens to be a Unicode character that
 182                 // would never be part of a program (can not be an identifier).
 183                 //
 184                 // This character is only tested just before the tokenizer is about to report
 185                 // an error;   So on the regular operation mode, this addition will have no
 186                 // impact on the tokenizer's performance.
 187                 //
 188
 189                 public const int EvalStatementParserCharacter = 0x2190;   // Unicode Left Arrow
 190                 public const int EvalCompilationUnitParserCharacter = 0x2191;  // Unicode Arrow
 191                 public const int EvalUsingDeclarationsParserCharacter = 0x2192;  // Unicode Arrow
 192
 193                 //
 194                 // XML documentation buffer. The save point is used to divide
 195                 // comments on types and comments on members.
 196                 //
 197                 StringBuilder xml_comment_buffer;
 198
 199                 //
 200                 // See comment on XmlCommentState enumeration.
 201                 //
 202                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 203
 204                 //
 205                 // Whether tokens have been seen on this line
 206                 //
 207                 bool tokens_seen = false;
 208
 209                 //
 210                 // Set to true once the GENERATE_COMPLETION token has bee
 211                 // returned.   This helps produce one GENERATE_COMPLETION,
 212                 // as many COMPLETE_COMPLETION as necessary to complete the
 213                 // AST tree and one final EOF.
 214                 //
 215                 bool generated;
 216
 217                 //
 218                 // Whether a token has been seen on the file
 219                 // This is needed because `define' is not allowed to be used
 220                 // after a token has been seen.
 221                 //
 222                 bool any_token_seen = false;
 223
 224                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 225
 226                 public bool PropertyParsing {
 227                         get { return handle_get_set; }
 228                         set { handle_get_set = value; }
 229                 }
 230
 231                 public bool EventParsing {
 232                         get { return handle_remove_add; }
 233                         set { handle_remove_add = value; }
 234                 }
 235
 236                 public bool ConstraintsParsing {
 237                         get { return handle_where; }
 238                         set { handle_where = value; }
 239                 }
 240
 241                 public bool TypeOfParsing {
 242                         get { return handle_typeof; }
 243                         set { handle_typeof = value; }
 244                 }
 245
 246                 public XmlCommentState doc_state {
 247                         get { return xml_doc_state; }
 248                         set {
 249                                 if (value == XmlCommentState.Allowed) {
 250                                         check_incorrect_doc_comment ();
 251                                         reset_doc_comment ();
 252                                 }
 253                                 xml_doc_state = value;
 254                         }
 255                 }
 256
 257                 //
 258                 // This is used to trigger completion generation on the parser
 259                 public bool CompleteOnEOF;
 260
 261                 void AddEscapedIdentifier (Location loc)
 262                 {
 263                         if (escaped_identifiers == null)
 264                                 escaped_identifiers = new List<Location> ();
 265
 266                         escaped_identifiers.Add (loc);
 267                 }
 268
 269                 public bool IsEscapedIdentifier (Location loc)
 270                 {
 271                         if (escaped_identifiers != null) {
 272                                 foreach (Location lt in escaped_identifiers)
 273                                         if (lt.Equals (loc))
 274                                                 return true;
 275                         }
 276
 277                         return false;
 278                 }
 279
 280                 //
 281                 // Class variables
 282                 //
 283                 static KeywordEntry[][] keywords;
 284                 static Dictionary<string, object> keyword_strings;              // TODO: HashSet
 285                 static NumberStyles styles;
 286                 static NumberFormatInfo csharp_format_info;
 287
 288                 //
 289                 // Values for the associated token returned
 290                 //
 291                 internal int putback_char;      // Used by repl only
 292                 object val;
 293
 294                 //
 295                 // Pre-processor
 296                 //
 297                 const int TAKING        = 1;
 298                 const int ELSE_SEEN     = 4;
 299                 const int PARENT_TAKING = 8;
 300                 const int REGION        = 16;
 301
 302                 //
 303                 // pre-processor if stack state:
 304                 //
 305                 Stack<int> ifstack;
 306
 307                 static System.Text.StringBuilder string_builder;
 308
 309                 const int max_id_size = 512;
 310                 static char [] id_builder = new char [max_id_size];
 311
 312                 public static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 313
 314                 const int max_number_size = 512;
 315                 static char [] number_builder = new char [max_number_size];
 316                 static int number_pos;
 317
 318                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
 319
 320                 //
 321                 // Details about the error encoutered by the tokenizer
 322                 //
 323                 string error_details;
 324
 325                 public string error {
 326                         get {
 327                                 return error_details;
 328                         }
 329                 }
 330
 331                 public int Line {
 332                         get {
 333                                 return ref_line;
 334                         }
 335                 }
 336
 337                 //
 338                 // This is used when the tokenizer needs to save
 339                 // the current position as it needs to do some parsing
 340                 // on its own to deamiguate a token in behalf of the
 341                 // parser.
 342                 //
 343                 Stack<Position> position_stack = new Stack<Position> (2);
 344
 345                 class Position {
 346                         public int position;
 347                         public int line;
 348                         public int ref_line;
 349                         public int col;
 350                         public bool hidden;
 351                         public int putback_char;
 352                         public int previous_col;
 353                         public Stack<int> ifstack;
 354                         public int parsing_generic_less_than;
 355                         public int current_token;
 356                         public object val;
 357
 358                         public Position (Tokenizer t)
 359                         {
 360                                 position = t.reader.Position;
 361                                 line = t.line;
 362                                 ref_line = t.ref_line;
 363                                 col = t.col;
 364                                 hidden = t.hidden;
 365                                 putback_char = t.putback_char;
 366                                 previous_col = t.previous_col;
 367                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 368                                         // There is no simple way to clone Stack<T> all
 369                                         // methods reverse the order
 370                                         var clone = t.ifstack.ToArray ();
 371                                         Array.Reverse (clone);
 372                                         ifstack = new Stack<int> (clone);
 373                                 }
 374                                 parsing_generic_less_than = t.parsing_generic_less_than;
 375                                 current_token = t.current_token;
 376                                 val = t.val;
 377                         }
 378                 }
 379
 380                 public void PushPosition ()
 381                 {
 382                         position_stack.Push (new Position (this));
 383                 }
 384
 385                 public void PopPosition ()
 386                 {
 387                         Position p = position_stack.Pop ();
 388
 389                         reader.Position = p.position;
 390                         ref_line = p.ref_line;
 391                         line = p.line;
 392                         col = p.col;
 393                         hidden = p.hidden;
 394                         putback_char = p.putback_char;
 395                         previous_col = p.previous_col;
 396                         ifstack = p.ifstack;
 397                         parsing_generic_less_than = p.parsing_generic_less_than;
 398                         current_token = p.current_token;
 399                         val = p.val;
 400                 }
 401
 402                 // Do not reset the position, ignore it.
 403                 public void DiscardPosition ()
 404                 {
 405                         position_stack.Pop ();
 406                 }
 407
 408                 static void AddKeyword (string kw, int token)
 409                 {
 410                         keyword_strings.Add (kw, null);
 411
 412                         int length = kw.Length;
 413                         if (keywords [length] == null) {
 414                                 keywords [length] = new KeywordEntry ['z' - '_' + 1];
 415                         }
 416
 417                         int char_index = kw [0] - '_';
 418                         KeywordEntry kwe = keywords [length] [char_index];
 419                         if (kwe == null) {
 420                                 keywords [length] [char_index] = new KeywordEntry (kw, token);
 421                                 return;
 422                         }
 423
 424                         while (kwe.Next != null) {
 425                                 kwe = kwe.Next;
 426                         }
 427
 428                         kwe.Next = new KeywordEntry (kw, token);
 429                 }
 430
 431                 static void InitTokens ()
 432                 {
 433                         keyword_strings = new Dictionary<string, object> ();
 434
 435                         // 11 is the length of the longest keyword for now
 436                         keywords = new KeywordEntry [11] [];
 437
 438                         AddKeyword ("__arglist", Token.ARGLIST);
 439                         AddKeyword ("abstract", Token.ABSTRACT);
 440                         AddKeyword ("as", Token.AS);
 441                         AddKeyword ("add", Token.ADD);
 442                         AddKeyword ("base", Token.BASE);
 443                         AddKeyword ("bool", Token.BOOL);
 444                         AddKeyword ("break", Token.BREAK);
 445                         AddKeyword ("byte", Token.BYTE);
 446                         AddKeyword ("case", Token.CASE);
 447                         AddKeyword ("catch", Token.CATCH);
 448                         AddKeyword ("char", Token.CHAR);
 449                         AddKeyword ("checked", Token.CHECKED);
 450                         AddKeyword ("class", Token.CLASS);
 451                         AddKeyword ("const", Token.CONST);
 452                         AddKeyword ("continue", Token.CONTINUE);
 453                         AddKeyword ("decimal", Token.DECIMAL);
 454                         AddKeyword ("default", Token.DEFAULT);
 455                         AddKeyword ("delegate", Token.DELEGATE);
 456                         AddKeyword ("do", Token.DO);
 457                         AddKeyword ("double", Token.DOUBLE);
 458                         AddKeyword ("else", Token.ELSE);
 459                         AddKeyword ("enum", Token.ENUM);
 460                         AddKeyword ("event", Token.EVENT);
 461                         AddKeyword ("explicit", Token.EXPLICIT);
 462                         AddKeyword ("extern", Token.EXTERN);
 463                         AddKeyword ("false", Token.FALSE);
 464                         AddKeyword ("finally", Token.FINALLY);
 465                         AddKeyword ("fixed", Token.FIXED);
 466                         AddKeyword ("float", Token.FLOAT);
 467                         AddKeyword ("for", Token.FOR);
 468                         AddKeyword ("foreach", Token.FOREACH);
 469                         AddKeyword ("goto", Token.GOTO);
 470                         AddKeyword ("get", Token.GET);
 471                         AddKeyword ("if", Token.IF);
 472                         AddKeyword ("implicit", Token.IMPLICIT);
 473                         AddKeyword ("in", Token.IN);
 474                         AddKeyword ("int", Token.INT);
 475                         AddKeyword ("interface", Token.INTERFACE);
 476                         AddKeyword ("internal", Token.INTERNAL);
 477                         AddKeyword ("is", Token.IS);
 478                         AddKeyword ("lock", Token.LOCK);
 479                         AddKeyword ("long", Token.LONG);
 480                         AddKeyword ("namespace", Token.NAMESPACE);
 481                         AddKeyword ("new", Token.NEW);
 482                         AddKeyword ("null", Token.NULL);
 483                         AddKeyword ("object", Token.OBJECT);
 484                         AddKeyword ("operator", Token.OPERATOR);
 485                         AddKeyword ("out", Token.OUT);
 486                         AddKeyword ("override", Token.OVERRIDE);
 487                         AddKeyword ("params", Token.PARAMS);
 488                         AddKeyword ("private", Token.PRIVATE);
 489                         AddKeyword ("protected", Token.PROTECTED);
 490                         AddKeyword ("public", Token.PUBLIC);
 491                         AddKeyword ("readonly", Token.READONLY);
 492                         AddKeyword ("ref", Token.REF);
 493                         AddKeyword ("remove", Token.REMOVE);
 494                         AddKeyword ("return", Token.RETURN);
 495                         AddKeyword ("sbyte", Token.SBYTE);
 496                         AddKeyword ("sealed", Token.SEALED);
 497                         AddKeyword ("set", Token.SET);
 498                         AddKeyword ("short", Token.SHORT);
 499                         AddKeyword ("sizeof", Token.SIZEOF);
 500                         AddKeyword ("stackalloc", Token.STACKALLOC);
 501                         AddKeyword ("static", Token.STATIC);
 502                         AddKeyword ("string", Token.STRING);
 503                         AddKeyword ("struct", Token.STRUCT);
 504                         AddKeyword ("switch", Token.SWITCH);
 505                         AddKeyword ("this", Token.THIS);
 506                         AddKeyword ("throw", Token.THROW);
 507                         AddKeyword ("true", Token.TRUE);
 508                         AddKeyword ("try", Token.TRY);
 509                         AddKeyword ("typeof", Token.TYPEOF);
 510                         AddKeyword ("uint", Token.UINT);
 511                         AddKeyword ("ulong", Token.ULONG);
 512                         AddKeyword ("unchecked", Token.UNCHECKED);
 513                         AddKeyword ("unsafe", Token.UNSAFE);
 514                         AddKeyword ("ushort", Token.USHORT);
 515                         AddKeyword ("using", Token.USING);
 516                         AddKeyword ("virtual", Token.VIRTUAL);
 517                         AddKeyword ("void", Token.VOID);
 518                         AddKeyword ("volatile", Token.VOLATILE);
 519                         AddKeyword ("while", Token.WHILE);
 520                         AddKeyword ("partial", Token.PARTIAL);
 521                         AddKeyword ("where", Token.WHERE);
 522
 523                         // LINQ keywords
 524                         AddKeyword ("from", Token.FROM);
 525                         AddKeyword ("join", Token.JOIN);
 526                         AddKeyword ("on", Token.ON);
 527                         AddKeyword ("equals", Token.EQUALS);
 528                         AddKeyword ("select", Token.SELECT);
 529                         AddKeyword ("group", Token.GROUP);
 530                         AddKeyword ("by", Token.BY);
 531                         AddKeyword ("let", Token.LET);
 532                         AddKeyword ("orderby", Token.ORDERBY);
 533                         AddKeyword ("ascending", Token.ASCENDING);
 534                         AddKeyword ("descending", Token.DESCENDING);
 535                         AddKeyword ("into", Token.INTO);
 536                 }
 537
 538                 //
 539                 // Class initializer
 540                 //
 541                 static Tokenizer ()
 542                 {
 543                         InitTokens ();
 544                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 545                         styles = NumberStyles.Float;
 546
 547                         string_builder = new System.Text.StringBuilder ();
 548                 }
 549
 550                 int GetKeyword (char[] id, int id_len)
 551                 {
 552                         //
 553                         // Keywords are stored in an array of arrays grouped by their
 554                         // length and then by the first character
 555                         //
 556                         if (id_len >= keywords.Length || keywords [id_len] == null)
 557                                 return -1;
 558
 559                         int first_index = id [0] - '_';
 560                         if (first_index > 'z')
 561                                 return -1;
 562
 563                         KeywordEntry kwe = keywords [id_len] [first_index];
 564                         if (kwe == null)
 565                                 return -1;
 566
 567                         int res;
 568                         do {
 569                                 res = kwe.Token;
 570                                 for (int i = 1; i < id_len; ++i) {
 571                                         if (id [i] != kwe.Value [i]) {
 572                                                 res = 0;
 573                                                 kwe = kwe.Next;
 574                                                 break;
 575                                         }
 576                                 }
 577                         } while (res == 0 && kwe != null);
 578
 579                         if (res == 0)
 580                                 return -1;
 581
 582                         int next_token;
 583                         switch (res) {
 584                         case Token.GET:
 585                         case Token.SET:
 586                                 if (!handle_get_set)
 587                                         res = -1;
 588                                 break;
 589                         case Token.REMOVE:
 590                         case Token.ADD:
 591                                 if (!handle_remove_add)
 592                                         res = -1;
 593                                 break;
 594                         case Token.EXTERN:
 595                                 if (parsing_declaration == 0)
 596                                         res = Token.EXTERN_ALIAS;
 597                                 break;
 598                         case Token.DEFAULT:
 599                                 if (peek_token () == Token.COLON) {
 600                                         token ();
 601                                         res = Token.DEFAULT_COLON;
 602                                 }
 603                                 break;
 604                         case Token.WHERE:
 605                                 if (!handle_where && !query_parsing)
 606                                         res = -1;
 607                                 break;
 608                         case Token.FROM:
 609                                 //
 610                                 // A query expression is any expression that starts with `from identifier'
 611                                 // followed by any token except ; , =
 612                                 //
 613                                 if (!query_parsing) {
 614                                         if (lambda_arguments_parsing) {
 615                                                 res = -1;
 616                                                 break;
 617                                         }
 618
 619                                         PushPosition ();
 620                                         // HACK: to disable generics micro-parser, because PushPosition does not
 621                                         // store identifiers array
 622                                         parsing_generic_less_than = 1;
 623                                         switch (xtoken ()) {
 624                                         case Token.IDENTIFIER:
 625                                         case Token.INT:
 626                                         case Token.BOOL:
 627                                         case Token.BYTE:
 628                                         case Token.CHAR:
 629                                         case Token.DECIMAL:
 630                                         case Token.FLOAT:
 631                                         case Token.LONG:
 632                                         case Token.OBJECT:
 633                                         case Token.STRING:
 634                                         case Token.UINT:
 635                                         case Token.ULONG:
 636                                                 next_token = xtoken ();
 637                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 638                                                         goto default;
 639
 640                                                 res = Token.FROM_FIRST;
 641                                                 query_parsing = true;
 642                                                 if (RootContext.Version <= LanguageVersion.ISO_2)
 643                                                         Report.FeatureIsNotAvailable (Location, "query expressions");
 644                                                 break;
 645                                         case Token.VOID:
 646                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 647                                                 break;
 648                                         default:
 649                                                 PopPosition ();
 650                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 651                                                 // which has been overwritten before we grabbed the identifier
 652                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 653                                                 return -1;
 654                                         }
 655                                         PopPosition ();
 656                                 }
 657                                 break;
 658                         case Token.JOIN:
 659                         case Token.ON:
 660                         case Token.EQUALS:
 661                         case Token.SELECT:
 662                         case Token.GROUP:
 663                         case Token.BY:
 664                         case Token.LET:
 665                         case Token.ORDERBY:
 666                         case Token.ASCENDING:
 667                         case Token.DESCENDING:
 668                         case Token.INTO:
 669                                 if (!query_parsing)
 670                                         res = -1;
 671                                 break;
 672
 673                         case Token.USING:
 674                         case Token.NAMESPACE:
 675                                 // TODO: some explanation needed
 676                                 check_incorrect_doc_comment ();
 677                                 break;
 678
 679                         case Token.PARTIAL:
 680                                 if (parsing_block > 0) {
 681                                         res = -1;
 682                                         break;
 683                                 }
 684
 685                                 // Save current position and parse next token.
 686                                 PushPosition ();
 687
 688                                 next_token = token ();
 689                                 bool ok = (next_token == Token.CLASS) ||
 690                                         (next_token == Token.STRUCT) ||
 691                                         (next_token == Token.INTERFACE) ||
 692                                         (next_token == Token.VOID);
 693
 694                                 PopPosition ();
 695
 696                                 if (ok) {
 697                                         if (next_token == Token.VOID) {
 698                                                 if (RootContext.Version == LanguageVersion.ISO_1 ||
 699                                                     RootContext.Version == LanguageVersion.ISO_2)
 700                                                         Report.FeatureIsNotAvailable (Location, "partial methods");
 701                                         } else if (RootContext.Version == LanguageVersion.ISO_1)
 702                                                 Report.FeatureIsNotAvailable (Location, "partial types");
 703
 704                                         return res;
 705                                 }
 706
 707                                 if (next_token < Token.LAST_KEYWORD) {
 708                                         Report.Error (267, Location,
 709                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 710                                         return token ();
 711                                 }
 712
 713                                 res = -1;
 714                                 break;
 715                         }
 716
 717                         return res;
 718                 }
 719
 720                 public Location Location {
 721                         get {
 722                                 return new Location (ref_line, hidden ? -1 : col);
 723                         }
 724                 }
 725
 726                 public Tokenizer (SeekableStreamReader input, CompilationUnit file, CompilerContext ctx)
 727                 {
 728                         this.ref_name = file;
 729                         this.file_name = file;
 730                         this.context = ctx;
 731                         reader = input;
 732
 733                         putback_char = -1;
 734
 735                         xml_comment_buffer = new StringBuilder ();
 736
 737                         //
 738                         // FIXME: This could be `Location.Push' but we have to
 739                         // find out why the MS compiler allows this
 740                         //
 741                         Mono.CSharp.Location.Push (file, file);
 742                 }
 743
 744                 static bool is_identifier_start_character (int c)
 745                 {
 746                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 747                 }
 748
 749                 static bool is_identifier_part_character (char c)
 750                 {
 751                         if (c >= 'a' && c <= 'z')
 752                                 return true;
 753
 754                         if (c >= 'A' && c <= 'Z')
 755                                 return true;
 756
 757                         if (c == '_' || (c >= '0' && c <= '9'))
 758                                 return true;
 759
 760                         if (c < 0x80)
 761                                 return false;
 762
 763                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 764                 }
 765
 766                 public static bool IsKeyword (string s)
 767                 {
 768                         return keyword_strings.ContainsKey (s);
 769                 }
 770
 771                 //
 772                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 773                 //
 774                 int TokenizeOpenParens ()
 775                 {
 776                         int ptoken;
 777                         current_token = -1;
 778
 779                         int bracket_level = 0;
 780                         bool is_type = false;
 781                         bool can_be_type = false;
 782
 783                         while (true) {
 784                                 ptoken = current_token;
 785                                 token ();
 786
 787                                 switch (current_token) {
 788                                 case Token.CLOSE_PARENS:
 789                                         token ();
 790
 791                                         //
 792                                         // Expression inside parens is lambda, (int i) =>
 793                                         //
 794                                         if (current_token == Token.ARROW)
 795                                                 return Token.OPEN_PARENS_LAMBDA;
 796
 797                                         //
 798                                         // Expression inside parens is single type, (int[])
 799                                         //
 800                                         if (is_type)
 801                                                 return Token.OPEN_PARENS_CAST;
 802
 803                                         //
 804                                         // Expression is possible cast, look at next token, (T)null
 805                                         //
 806                                         if (can_be_type) {
 807                                                 switch (current_token) {
 808                                                 case Token.OPEN_PARENS:
 809                                                 case Token.BANG:
 810                                                 case Token.TILDE:
 811                                                 case Token.IDENTIFIER:
 812                                                 case Token.LITERAL:
 813                                                 case Token.BASE:
 814                                                 case Token.CHECKED:
 815                                                 case Token.DELEGATE:
 816                                                 case Token.FALSE:
 817                                                 case Token.FIXED:
 818                                                 case Token.NEW:
 819                                                 case Token.NULL:
 820                                                 case Token.SIZEOF:
 821                                                 case Token.THIS:
 822                                                 case Token.THROW:
 823                                                 case Token.TRUE:
 824                                                 case Token.TYPEOF:
 825                                                 case Token.UNCHECKED:
 826                                                 case Token.UNSAFE:
 827                                                 case Token.DEFAULT:
 828
 829                                                 //
 830                                                 // These can be part of a member access
 831                                                 //
 832                                                 case Token.INT:
 833                                                 case Token.UINT:
 834                                                 case Token.SHORT:
 835                                                 case Token.USHORT:
 836                                                 case Token.LONG:
 837                                                 case Token.ULONG:
 838                                                 case Token.DOUBLE:
 839                                                 case Token.FLOAT:
 840                                                 case Token.CHAR:
 841                                                 case Token.BYTE:
 842                                                 case Token.DECIMAL:
 843                                                 case Token.BOOL:
 844                                                         return Token.OPEN_PARENS_CAST;
 845                                                 }
 846                                         }
 847                                         return Token.OPEN_PARENS;
 848
 849                                 case Token.DOT:
 850                                 case Token.DOUBLE_COLON:
 851                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
 852                                                 goto default;
 853
 854                                         continue;
 855
 856                                 case Token.IDENTIFIER:
 857                                         switch (ptoken) {
 858                                         case Token.DOT:
 859                                         case Token.OP_GENERICS_LT:
 860                                         case Token.COMMA:
 861                                         case Token.DOUBLE_COLON:
 862                                         case -1:
 863                                                 if (bracket_level == 0)
 864                                                         can_be_type = true;
 865                                                 continue;
 866                                         default:
 867                                                 can_be_type = is_type = false;
 868                                                 continue;
 869                                         }
 870
 871                                 case Token.OBJECT:
 872                                 case Token.STRING:
 873                                 case Token.BOOL:
 874                                 case Token.DECIMAL:
 875                                 case Token.FLOAT:
 876                                 case Token.DOUBLE:
 877                                 case Token.SBYTE:
 878                                 case Token.BYTE:
 879                                 case Token.SHORT:
 880                                 case Token.USHORT:
 881                                 case Token.INT:
 882                                 case Token.UINT:
 883                                 case Token.LONG:
 884                                 case Token.ULONG:
 885                                 case Token.CHAR:
 886                                 case Token.VOID:
 887                                         if (bracket_level == 0)
 888                                                 is_type = true;
 889                                         continue;
 890
 891                                 case Token.COMMA:
 892                                         if (bracket_level == 0) {
 893                                                 bracket_level = 100;
 894                                                 can_be_type = is_type = false;
 895                                         }
 896                                         continue;
 897
 898                                 case Token.OP_GENERICS_LT:
 899                                 case Token.OPEN_BRACKET:
 900                                         if (bracket_level++ == 0)
 901                                                 is_type = true;
 902                                         continue;
 903
 904                                 case Token.OP_GENERICS_GT:
 905                                 case Token.CLOSE_BRACKET:
 906                                         --bracket_level;
 907                                         continue;
 908
 909                                 case Token.INTERR_NULLABLE:
 910                                 case Token.STAR:
 911                                         if (bracket_level == 0)
 912                                                 is_type = true;
 913                                         continue;
 914
 915                                 case Token.REF:
 916                                 case Token.OUT:
 917                                         can_be_type = is_type = false;
 918                                         continue;
 919
 920                                 default:
 921                                         return Token.OPEN_PARENS;
 922                                 }
 923                         }
 924                 }
 925
 926                 public static bool IsValidIdentifier (string s)
 927                 {
 928                         if (s == null || s.Length == 0)
 929                                 return false;
 930
 931                         if (!is_identifier_start_character (s [0]))
 932                                 return false;
 933
 934                         for (int i = 1; i < s.Length; i ++)
 935                                 if (! is_identifier_part_character (s [i]))
 936                                         return false;
 937
 938                         return true;
 939                 }
 940
 941                 bool parse_less_than ()
 942                 {
 943                 start:
 944                         int the_token = token ();
 945                         if (the_token == Token.OPEN_BRACKET) {
 946                                 do {
 947                                         the_token = token ();
 948                                 } while (the_token != Token.CLOSE_BRACKET);
 949                                 the_token = token ();
 950                         } else if (the_token == Token.IN || the_token == Token.OUT) {
 951                                 the_token = token ();
 952                         }
 953                         switch (the_token) {
 954                         case Token.IDENTIFIER:
 955                         case Token.OBJECT:
 956                         case Token.STRING:
 957                         case Token.BOOL:
 958                         case Token.DECIMAL:
 959                         case Token.FLOAT:
 960                         case Token.DOUBLE:
 961                         case Token.SBYTE:
 962                         case Token.BYTE:
 963                         case Token.SHORT:
 964                         case Token.USHORT:
 965                         case Token.INT:
 966                         case Token.UINT:
 967                         case Token.LONG:
 968                         case Token.ULONG:
 969                         case Token.CHAR:
 970                         case Token.VOID:
 971                                 break;
 972                         case Token.OP_GENERICS_GT:
 973                                 return true;
 974
 975                         default:
 976                                 return false;
 977                         }
 978                 again:
 979                         the_token = token ();
 980
 981                         if (the_token == Token.OP_GENERICS_GT)
 982                                 return true;
 983                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
 984                                 goto start;
 985                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
 986                                 goto again;
 987                         else if (the_token == Token.OP_GENERICS_LT) {
 988                                 if (!parse_less_than ())
 989                                         return false;
 990                                 goto again;
 991                         } else if (the_token == Token.OPEN_BRACKET) {
 992                         rank_specifiers:
 993                                 the_token = token ();
 994                                 if (the_token == Token.CLOSE_BRACKET)
 995                                         goto again;
 996                                 else if (the_token == Token.COMMA)
 997                                         goto rank_specifiers;
 998                                 return false;
 999                         }
1000
1001                         return false;
1002                 }
1003
1004                 bool parse_generic_dimension (out int dimension)
1005                 {
1006                         dimension = 1;
1007
1008                 again:
1009                         int the_token = token ();
1010                         if (the_token == Token.OP_GENERICS_GT)
1011                                 return true;
1012                         else if (the_token == Token.COMMA) {
1013                                 dimension++;
1014                                 goto again;
1015                         }
1016
1017                         return false;
1018                 }
1019
1020                 public int peek_token ()
1021                 {
1022                         int the_token;
1023
1024                         PushPosition ();
1025                         the_token = token ();
1026                         PopPosition ();
1027
1028                         return the_token;
1029                 }
1030
1031                 //
1032                 // Tonizes `?' using custom disambiguous rules to return one
1033                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1034                 //
1035                 // Tricky expression look like:
1036                 //
1037                 // Foo ? a = x ? b : c;
1038                 //
1039                 int TokenizePossibleNullableType ()
1040                 {
1041                         if (parsing_block == 0 || parsing_type > 0)
1042                                 return Token.INTERR_NULLABLE;
1043
1044                         int d = peek_char ();
1045                         if (d == '?') {
1046                                 get_char ();
1047                                 return Token.OP_COALESCING;
1048                         }
1049
1050                         switch (current_token) {
1051                         case Token.CLOSE_PARENS:
1052                         case Token.TRUE:
1053                         case Token.FALSE:
1054                         case Token.NULL:
1055                         case Token.LITERAL:
1056                                 return Token.INTERR;
1057                         }
1058
1059                         if (d != ' ') {
1060                                 if (d == ',' || d == ';' || d == '>')
1061                                         return Token.INTERR_NULLABLE;
1062                                 if (d == '*' || (d >= '0' && d <= '9'))
1063                                         return Token.INTERR;
1064                         }
1065
1066                         PushPosition ();
1067                         current_token = Token.NONE;
1068                         int next_token;
1069                         switch (xtoken ()) {
1070                         case Token.LITERAL:
1071                         case Token.TRUE:
1072                         case Token.FALSE:
1073                         case Token.NULL:
1074                         case Token.THIS:
1075                         case Token.NEW:
1076                                 next_token = Token.INTERR;
1077                                 break;
1078
1079                         case Token.SEMICOLON:
1080                         case Token.COMMA:
1081                         case Token.CLOSE_PARENS:
1082                         case Token.OPEN_BRACKET:
1083                         case Token.OP_GENERICS_GT:
1084                                 next_token = Token.INTERR_NULLABLE;
1085                                 break;
1086
1087                         default:
1088                                 next_token = -1;
1089                                 break;
1090                         }
1091
1092                         if (next_token == -1) {
1093                                 switch (xtoken ()) {
1094                                 case Token.COMMA:
1095                                 case Token.SEMICOLON:
1096                                 case Token.OPEN_BRACE:
1097                                 case Token.CLOSE_PARENS:
1098                                 case Token.IN:
1099                                         next_token = Token.INTERR_NULLABLE;
1100                                         break;
1101
1102                                 case Token.COLON:
1103                                         next_token = Token.INTERR;
1104                                         break;
1105
1106                                 default:
1107                                         int ntoken;
1108                                         int interrs = 1;
1109                                         int colons = 0;
1110                                         //
1111                                         // All shorcuts failed, do it hard way
1112                                         //
1113                                         while ((ntoken = xtoken ()) != Token.EOF) {
1114                                                 if (ntoken == Token.SEMICOLON)
1115                                                         break;
1116
1117                                                 if (ntoken == Token.COLON) {
1118                                                         if (++colons == interrs)
1119                                                                 break;
1120                                                         continue;
1121                                                 }
1122
1123                                                 if (ntoken == Token.INTERR) {
1124                                                         ++interrs;
1125                                                         continue;
1126                                                 }
1127                                         }
1128
1129                                         next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1130                                         break;
1131                                 }
1132                         }
1133
1134                         PopPosition ();
1135                         return next_token;
1136                 }
1137
1138                 bool decimal_digits (int c)
1139                 {
1140                         int d;
1141                         bool seen_digits = false;
1142
1143                         if (c != -1){
1144                                 if (number_pos == max_number_size)
1145                                         Error_NumericConstantTooLong ();
1146                                 number_builder [number_pos++] = (char) c;
1147                         }
1148
1149                         //
1150                         // We use peek_char2, because decimal_digits needs to do a
1151                         // 2-character look-ahead (5.ToString for example).
1152                         //
1153                         while ((d = peek_char2 ()) != -1){
1154                                 if (d >= '0' && d <= '9'){
1155                                         if (number_pos == max_number_size)
1156                                                 Error_NumericConstantTooLong ();
1157                                         number_builder [number_pos++] = (char) d;
1158                                         get_char ();
1159                                         seen_digits = true;
1160                                 } else
1161                                         break;
1162                         }
1163
1164                         return seen_digits;
1165                 }
1166
1167                 static bool is_hex (int e)
1168                 {
1169                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1170                 }
1171
1172                 static TypeCode real_type_suffix (int c)
1173                 {
1174                         switch (c){
1175                         case 'F': case 'f':
1176                                 return TypeCode.Single;
1177                         case 'D': case 'd':
1178                                 return TypeCode.Double;
1179                         case 'M': case 'm':
1180                                 return TypeCode.Decimal;
1181                         default:
1182                                 return TypeCode.Empty;
1183                         }
1184                 }
1185
1186                 int integer_type_suffix (ulong ul, int c)
1187                 {
1188                         bool is_unsigned = false;
1189                         bool is_long = false;
1190
1191                         if (c != -1){
1192                                 bool scanning = true;
1193                                 do {
1194                                         switch (c){
1195                                         case 'U': case 'u':
1196                                                 if (is_unsigned)
1197                                                         scanning = false;
1198                                                 is_unsigned = true;
1199                                                 get_char ();
1200                                                 break;
1201
1202                                         case 'l':
1203                                                 if (!is_unsigned){
1204                                                         //
1205                                                         // if we have not seen anything in between
1206                                                         // report this error
1207                                                         //
1208                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1209                                                 }
1210
1211                                                 goto case 'L';
1212
1213                                         case 'L':
1214                                                 if (is_long)
1215                                                         scanning = false;
1216                                                 is_long = true;
1217                                                 get_char ();
1218                                                 break;
1219
1220                                         default:
1221                                                 scanning = false;
1222                                                 break;
1223                                         }
1224                                         c = peek_char ();
1225                                 } while (scanning);
1226                         }
1227
1228                         if (is_long && is_unsigned){
1229                                 val = new ULongLiteral (ul, Location);
1230                                 return Token.LITERAL;
1231                         }
1232
1233                         if (is_unsigned){
1234                                 // uint if possible, or ulong else.
1235
1236                                 if ((ul & 0xffffffff00000000) == 0)
1237                                         val = new UIntLiteral ((uint) ul, Location);
1238                                 else
1239                                         val = new ULongLiteral (ul, Location);
1240                         } else if (is_long){
1241                                 // long if possible, ulong otherwise
1242                                 if ((ul & 0x8000000000000000) != 0)
1243                                         val = new ULongLiteral (ul, Location);
1244                                 else
1245                                         val = new LongLiteral ((long) ul, Location);
1246                         } else {
1247                                 // int, uint, long or ulong in that order
1248                                 if ((ul & 0xffffffff00000000) == 0){
1249                                         uint ui = (uint) ul;
1250
1251                                         if ((ui & 0x80000000) != 0)
1252                                                 val = new UIntLiteral (ui, Location);
1253                                         else
1254                                                 val = new IntLiteral ((int) ui, Location);
1255                                 } else {
1256                                         if ((ul & 0x8000000000000000) != 0)
1257                                                 val = new ULongLiteral (ul, Location);
1258                                         else
1259                                                 val = new LongLiteral ((long) ul, Location);
1260                                 }
1261                         }
1262                         return Token.LITERAL;
1263                 }
1264
1265                 //
1266                 // given `c' as the next char in the input decide whether
1267                 // we need to convert to a special type, and then choose
1268                 // the best representation for the integer
1269                 //
1270                 int adjust_int (int c)
1271                 {
1272                         try {
1273                                 if (number_pos > 9){
1274                                         ulong ul = (uint) (number_builder [0] - '0');
1275
1276                                         for (int i = 1; i < number_pos; i++){
1277                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1278                                         }
1279                                         return integer_type_suffix (ul, c);
1280                                 } else {
1281                                         uint ui = (uint) (number_builder [0] - '0');
1282
1283                                         for (int i = 1; i < number_pos; i++){
1284                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1285                                         }
1286                                         return integer_type_suffix (ui, c);
1287                                 }
1288                         } catch (OverflowException) {
1289                                 error_details = "Integral constant is too large";
1290                                 Report.Error (1021, Location, error_details);
1291                                 val = new IntLiteral (0, Location);
1292                                 return Token.LITERAL;
1293                         }
1294                         catch (FormatException) {
1295                                 Report.Error (1013, Location, "Invalid number");
1296                                 val = new IntLiteral (0, Location);
1297                                 return Token.LITERAL;
1298                         }
1299                 }
1300
1301                 int adjust_real (TypeCode t)
1302                 {
1303                         string s = new String (number_builder, 0, number_pos);
1304                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1305
1306                         switch (t){
1307                         case TypeCode.Decimal:
1308                                 try {
1309                                         val = new DecimalLiteral (decimal.Parse (s, styles, csharp_format_info), Location);
1310                                 } catch (OverflowException) {
1311                                         val = new DecimalLiteral (0, Location);
1312                                         Report.Error (594, Location, error_details, "decimal");
1313                                 }
1314                                 break;
1315                         case TypeCode.Single:
1316                                 try {
1317                                         val = new FloatLiteral (float.Parse (s, styles, csharp_format_info), Location);
1318                                 } catch (OverflowException) {
1319                                         val = new FloatLiteral (0, Location);
1320                                         Report.Error (594, Location, error_details, "float");
1321                                 }
1322                                 break;
1323                         default:
1324                                 try {
1325                                         val = new DoubleLiteral (double.Parse (s, styles, csharp_format_info), Location);
1326                                 } catch (OverflowException) {
1327                                         val = new DoubleLiteral (0, Location);
1328                                         Report.Error (594, Location, error_details, "double");
1329                                 }
1330                                 break;
1331                         }
1332
1333                         return Token.LITERAL;
1334                 }
1335
1336                 int handle_hex ()
1337                 {
1338                         int d;
1339                         ulong ul;
1340
1341                         get_char ();
1342                         while ((d = peek_char ()) != -1){
1343                                 if (is_hex (d)){
1344                                         number_builder [number_pos++] = (char) d;
1345                                         get_char ();
1346                                 } else
1347                                         break;
1348                         }
1349
1350                         string s = new String (number_builder, 0, number_pos);
1351                         try {
1352                                 if (number_pos <= 8)
1353                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1354                                 else
1355                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1356                         } catch (OverflowException){
1357                                 error_details = "Integral constant is too large";
1358                                 Report.Error (1021, Location, error_details);
1359                                 val = new IntLiteral (0, Location);
1360                                 return Token.LITERAL;
1361                         }
1362                         catch (FormatException) {
1363                                 Report.Error (1013, Location, "Invalid number");
1364                                 val = new IntLiteral (0, Location);
1365                                 return Token.LITERAL;
1366                         }
1367
1368                         return integer_type_suffix (ul, peek_char ());
1369                 }
1370
1371                 //
1372                 // Invoked if we know we have .digits or digits
1373                 //
1374                 int is_number (int c)
1375                 {
1376                         bool is_real = false;
1377
1378                         number_pos = 0;
1379
1380                         if (c >= '0' && c <= '9'){
1381                                 if (c == '0'){
1382                                         int peek = peek_char ();
1383
1384                                         if (peek == 'x' || peek == 'X')
1385                                                 return handle_hex ();
1386                                 }
1387                                 decimal_digits (c);
1388                                 c = get_char ();
1389                         }
1390
1391                         //
1392                         // We need to handle the case of
1393                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1394                         //
1395                         if (c == '.'){
1396                                 if (decimal_digits ('.')){
1397                                         is_real = true;
1398                                         c = get_char ();
1399                                 } else {
1400                                         putback ('.');
1401                                         number_pos--;
1402                                         return adjust_int (-1);
1403                                 }
1404                         }
1405
1406                         if (c == 'e' || c == 'E'){
1407                                 is_real = true;
1408                                 if (number_pos == max_number_size)
1409                                         Error_NumericConstantTooLong ();
1410                                 number_builder [number_pos++] = 'e';
1411                                 c = get_char ();
1412
1413                                 if (c == '+'){
1414                                         if (number_pos == max_number_size)
1415                                                 Error_NumericConstantTooLong ();
1416                                         number_builder [number_pos++] = '+';
1417                                         c = -1;
1418                                 } else if (c == '-') {
1419                                         if (number_pos == max_number_size)
1420                                                 Error_NumericConstantTooLong ();
1421                                         number_builder [number_pos++] = '-';
1422                                         c = -1;
1423                                 } else {
1424                                         if (number_pos == max_number_size)
1425                                                 Error_NumericConstantTooLong ();
1426                                         number_builder [number_pos++] = '+';
1427                                 }
1428
1429                                 decimal_digits (c);
1430                                 c = get_char ();
1431                         }
1432
1433                         var type = real_type_suffix (c);
1434                         if (type == TypeCode.Empty && !is_real){
1435                                 putback (c);
1436                                 return adjust_int (c);
1437                         }
1438
1439                         is_real = true;
1440
1441                         if (type == TypeCode.Empty){
1442                                 putback (c);
1443                         }
1444
1445                         if (is_real)
1446                                 return adjust_real (type);
1447
1448                         throw new Exception ("Is Number should never reach this point");
1449                 }
1450
1451                 //
1452                 // Accepts exactly count (4 or 8) hex, no more no less
1453                 //
1454                 int getHex (int count, out int surrogate, out bool error)
1455                 {
1456                         int i;
1457                         int total = 0;
1458                         int c;
1459                         int top = count != -1 ? count : 4;
1460
1461                         get_char ();
1462                         error = false;
1463                         surrogate = 0;
1464                         for (i = 0; i < top; i++){
1465                                 c = get_char ();
1466
1467                                 if (c >= '0' && c <= '9')
1468                                         c = (int) c - (int) '0';
1469                                 else if (c >= 'A' && c <= 'F')
1470                                         c = (int) c - (int) 'A' + 10;
1471                                 else if (c >= 'a' && c <= 'f')
1472                                         c = (int) c - (int) 'a' + 10;
1473                                 else {
1474                                         error = true;
1475                                         return 0;
1476                                 }
1477
1478                                 total = (total * 16) + c;
1479                                 if (count == -1){
1480                                         int p = peek_char ();
1481                                         if (p == -1)
1482                                                 break;
1483                                         if (!is_hex ((char)p))
1484                                                 break;
1485                                 }
1486                         }
1487
1488                         if (top == 8) {
1489                                 if (total > 0x0010FFFF) {
1490                                         error = true;
1491                                         return 0;
1492                                 }
1493
1494                                 if (total >= 0x00010000) {
1495                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1496                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1497                                 }
1498                         }
1499
1500                         return total;
1501                 }
1502
1503                 int escape (int c, out int surrogate)
1504                 {
1505                         bool error;
1506                         int d;
1507                         int v;
1508
1509                         d = peek_char ();
1510                         if (c != '\\') {
1511                                 surrogate = 0;
1512                                 return c;
1513                         }
1514
1515                         switch (d){
1516                         case 'a':
1517                                 v = '\a'; break;
1518                         case 'b':
1519                                 v = '\b'; break;
1520                         case 'n':
1521                                 v = '\n'; break;
1522                         case 't':
1523                                 v = '\t'; break;
1524                         case 'v':
1525                                 v = '\v'; break;
1526                         case 'r':
1527                                 v = '\r'; break;
1528                         case '\\':
1529                                 v = '\\'; break;
1530                         case 'f':
1531                                 v = '\f'; break;
1532                         case '0':
1533                                 v = 0; break;
1534                         case '"':
1535                                 v = '"'; break;
1536                         case '\'':
1537                                 v = '\''; break;
1538                         case 'x':
1539                                 v = getHex (-1, out surrogate, out error);
1540                                 if (error)
1541                                         goto default;
1542                                 return v;
1543                         case 'u':
1544                         case 'U':
1545                                 return EscapeUnicode (d, out surrogate);
1546                         default:
1547                                 surrogate = 0;
1548                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1549                                 return d;
1550                         }
1551
1552                         get_char ();
1553                         surrogate = 0;
1554                         return v;
1555                 }
1556
1557                 int EscapeUnicode (int ch, out int surrogate)
1558                 {
1559                         bool error;
1560                         if (ch == 'U') {
1561                                 ch = getHex (8, out surrogate, out error);
1562                         } else {
1563                                 ch = getHex (4, out surrogate, out error);
1564                         }
1565
1566                         if (error)
1567                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1568
1569                         return ch;
1570                 }
1571
1572                 int get_char ()
1573                 {
1574                         int x;
1575                         if (putback_char != -1) {
1576                                 x = putback_char;
1577                                 putback_char = -1;
1578                         } else
1579                                 x = reader.Read ();
1580                         if (x == '\n') {
1581                                 advance_line ();
1582                         } else {
1583                                 col++;
1584                         }
1585                         return x;
1586                 }
1587
1588                 void advance_line ()
1589                 {
1590                         line++;
1591                         ref_line++;
1592                         previous_col = col;
1593                         col = 0;
1594                 }
1595
1596                 int peek_char ()
1597                 {
1598                         if (putback_char == -1)
1599                                 putback_char = reader.Read ();
1600                         return putback_char;
1601                 }
1602
1603                 int peek_char2 ()
1604                 {
1605                         if (putback_char != -1)
1606                                 return putback_char;
1607                         return reader.Peek ();
1608                 }
1609
1610                 void putback (int c)
1611                 {
1612                         if (putback_char != -1){
1613                                 Console.WriteLine ("Col: " + col);
1614                                 Console.WriteLine ("Row: " + line);
1615                                 Console.WriteLine ("Name: " + ref_name.Name);
1616                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1617                                 throw new Exception ("This should not happen putback on putback");
1618                         }
1619                         if (c == '\n' || col == 0) {
1620                                 // It won't happen though.
1621                                 line--;
1622                                 ref_line--;
1623                                 col = previous_col;
1624                         }
1625                         else
1626                                 col--;
1627                         putback_char = c;
1628                 }
1629
1630                 public bool advance ()
1631                 {
1632                         return peek_char () != -1 || CompleteOnEOF;
1633                 }
1634
1635                 public Object Value {
1636                         get {
1637                                 return val;
1638                         }
1639                 }
1640
1641                 public Object value ()
1642                 {
1643                         return val;
1644                 }
1645
1646                 public int token ()
1647                 {
1648                         current_token = xtoken ();
1649                         return current_token;
1650                 }
1651
1652                 void get_cmd_arg (out string cmd, out string arg)
1653                 {
1654                         int c;
1655
1656                         tokens_seen = false;
1657                         arg = "";
1658
1659                         // skip over white space
1660                         do {
1661                                 c = get_char ();
1662                         } while (c == '\r' || c == ' ' || c == '\t');
1663
1664                         static_cmd_arg.Length = 0;
1665                         while (c != -1 && is_identifier_part_character ((char)c)) {
1666                                 static_cmd_arg.Append ((char)c);
1667                                 c = get_char ();
1668                                 if (c == '\\') {
1669                                         int peek = peek_char ();
1670                                         if (peek == 'U' || peek == 'u') {
1671                                                 int surrogate;
1672                                                 c = EscapeUnicode (c, out surrogate);
1673                                                 if (surrogate != 0) {
1674                                                         if (is_identifier_part_character ((char) c))
1675                                                                 static_cmd_arg.Append ((char) c);
1676                                                         c = surrogate;
1677                                                 }
1678                                         }
1679                                 }
1680                         }
1681
1682                         cmd = static_cmd_arg.ToString ();
1683
1684                         // skip over white space
1685                         while (c == '\r' || c == ' ' || c == '\t')
1686                                 c = get_char ();
1687
1688                         static_cmd_arg.Length = 0;
1689                         int has_identifier_argument = 0;
1690
1691                         while (c != -1 && c != '\n' && c != '\r') {
1692                                 if (c == '\\' && has_identifier_argument >= 0) {
1693                                         if (has_identifier_argument != 0 || (cmd == "define" || cmd == "if" || cmd == "elif" || cmd == "undef")) {
1694                                                 has_identifier_argument = 1;
1695
1696                                                 int peek = peek_char ();
1697                                                 if (peek == 'U' || peek == 'u') {
1698                                                         int surrogate;
1699                                                         c = EscapeUnicode (c, out surrogate);
1700                                                         if (surrogate != 0) {
1701                                                                 if (is_identifier_part_character ((char) c))
1702                                                                         static_cmd_arg.Append ((char) c);
1703                                                                 c = surrogate;
1704                                                         }
1705                                                 }
1706                                         } else {
1707                                                 has_identifier_argument = -1;
1708                                         }
1709                                 }
1710                                 static_cmd_arg.Append ((char) c);
1711                                 c = get_char ();
1712                         }
1713
1714                         if (static_cmd_arg.Length != 0)
1715                                 arg = static_cmd_arg.ToString ();
1716                 }
1717
1718                 //
1719                 // Handles the #line directive
1720                 //
1721                 bool PreProcessLine (string arg)
1722                 {
1723                         if (arg.Length == 0)
1724                                 return false;
1725
1726                         if (arg == "default"){
1727                                 ref_line = line;
1728                                 ref_name = file_name;
1729                                 hidden = false;
1730                                 Location.Push (file_name, ref_name);
1731                                 return true;
1732                         } else if (arg == "hidden"){
1733                                 hidden = true;
1734                                 return true;
1735                         }
1736
1737                         try {
1738                                 int pos;
1739
1740                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1741                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1742                                         pos++;
1743
1744                                         char [] quotes = { '\"' };
1745
1746                                         string name = arg.Substring (pos). Trim (quotes);
1747                                         ref_name = Location.LookupFile (file_name, name);
1748                                         file_name.AddFile (ref_name);
1749                                         hidden = false;
1750                                         Location.Push (file_name, ref_name);
1751                                 } else {
1752                                         ref_line = System.Int32.Parse (arg);
1753                                         hidden = false;
1754                                 }
1755                         } catch {
1756                                 return false;
1757                         }
1758
1759                         return true;
1760                 }
1761
1762                 //
1763                 // Handles #define and #undef
1764                 //
1765                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
1766                 {
1767                         if (ident.Length == 0 || ident == "true" || ident == "false"){
1768                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
1769                                 return;
1770                         }
1771
1772                         if (ident.IndexOfAny (simple_whitespaces) != -1){
1773                                 Error_EndLineExpected ();
1774                                 return;
1775                         }
1776
1777                         if (!is_identifier_start_character (ident [0]))
1778                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
1779
1780                         foreach (char c in ident.Substring (1)){
1781                                 if (!is_identifier_part_character (c)){
1782                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
1783                                         return;
1784                                 }
1785                         }
1786
1787                         if (!caller_is_taking)
1788                                 return;
1789
1790                         if (is_define) {
1791                                 //
1792                                 // #define ident
1793                                 //
1794                                 if (RootContext.IsConditionalDefined (ident))
1795                                         return;
1796
1797                                 file_name.AddDefine (ident);
1798                         } else {
1799                                 //
1800                                 // #undef ident
1801                                 //
1802                                 file_name.AddUndefine (ident);
1803                         }
1804                 }
1805
1806                 static byte read_hex (string arg, int pos, out bool error)
1807                 {
1808                         error = false;
1809
1810                         int total;
1811                         char c = arg [pos];
1812
1813                         if ((c >= '0') && (c <= '9'))
1814                                 total = (int) c - (int) '0';
1815                         else if ((c >= 'A') && (c <= 'F'))
1816                                 total = (int) c - (int) 'A' + 10;
1817                         else if ((c >= 'a') && (c <= 'f'))
1818                                 total = (int) c - (int) 'a' + 10;
1819                         else {
1820                                 error = true;
1821                                 return 0;
1822                         }
1823
1824                         total *= 16;
1825                         c = arg [pos+1];
1826
1827                         if ((c >= '0') && (c <= '9'))
1828                                 total += (int) c - (int) '0';
1829                         else if ((c >= 'A') && (c <= 'F'))
1830                                 total += (int) c - (int) 'A' + 10;
1831                         else if ((c >= 'a') && (c <= 'f'))
1832                                 total += (int) c - (int) 'a' + 10;
1833                         else {
1834                                 error = true;
1835                                 return 0;
1836                         }
1837
1838                         return (byte) total;
1839                 }
1840
1841                 /// <summary>
1842                 /// Handles #pragma checksum
1843                 /// </summary>
1844                 bool PreProcessPragmaChecksum (string arg)
1845                 {
1846                         if ((arg [0] != ' ') && (arg [0] != '\t'))
1847                                 return false;
1848
1849                         arg = arg.Trim (simple_whitespaces);
1850                         if ((arg.Length < 2) || (arg [0] != '"'))
1851                                 return false;
1852
1853                         StringBuilder file_sb = new StringBuilder ();
1854
1855                         int pos = 1;
1856                         char ch;
1857                         while ((ch = arg [pos++]) != '"') {
1858                                 if (pos >= arg.Length)
1859                                         return false;
1860
1861                                 if (ch == '\\') {
1862                                         if (pos+1 >= arg.Length)
1863                                                 return false;
1864                                         ch = arg [pos++];
1865                                 }
1866
1867                                 file_sb.Append (ch);
1868                         }
1869
1870                         if ((pos+2 >= arg.Length) || ((arg [pos] != ' ') && (arg [pos] != '\t')))
1871                                 return false;
1872
1873                         arg = arg.Substring (pos).Trim (simple_whitespaces);
1874                         if ((arg.Length < 42) || (arg [0] != '"') || (arg [1] != '{') ||
1875                             (arg [10] != '-') || (arg [15] != '-') || (arg [20] != '-') ||
1876                             (arg [25] != '-') || (arg [38] != '}') || (arg [39] != '"'))
1877                                 return false;
1878
1879                         bool error;
1880                         byte[] guid_bytes = new byte [16];
1881
1882                         for (int i = 0; i < 4; i++) {
1883                                 guid_bytes [i] = read_hex (arg, 2+2*i, out error);
1884                                 if (error)
1885                                         return false;
1886                         }
1887                         for (int i = 0; i < 2; i++) {
1888                                 guid_bytes [i+4] = read_hex (arg, 11+2*i, out error);
1889                                 if (error)
1890                                         return false;
1891                                 guid_bytes [i+6] = read_hex (arg, 16+2*i, out error);
1892                                 if (error)
1893                                         return false;
1894                                 guid_bytes [i+8] = read_hex (arg, 21+2*i, out error);
1895                                 if (error)
1896                                         return false;
1897                         }
1898
1899                         for (int i = 0; i < 6; i++) {
1900                                 guid_bytes [i+10] = read_hex (arg, 26+2*i, out error);
1901                                 if (error)
1902                                         return false;
1903                         }
1904
1905                         arg = arg.Substring (40).Trim (simple_whitespaces);
1906                         if ((arg.Length < 34) || (arg [0] != '"') || (arg [33] != '"'))
1907                                 return false;
1908
1909                         byte[] checksum_bytes = new byte [16];
1910                         for (int i = 0; i < 16; i++) {
1911                                 checksum_bytes [i] = read_hex (arg, 1+2*i, out error);
1912                                 if (error)
1913                                         return false;
1914                         }
1915
1916                         arg = arg.Substring (34).Trim (simple_whitespaces);
1917                         if (arg.Length > 0)
1918                                 return false;
1919
1920                         SourceFile file = Location.LookupFile (file_name, file_sb.ToString ());
1921                         file.SetChecksum (guid_bytes, checksum_bytes);
1922                         ref_name.AutoGenerated = true;
1923                         return true;
1924                 }
1925
1926                 /// <summary>
1927                 /// Handles #pragma directive
1928                 /// </summary>
1929                 void PreProcessPragma (string arg)
1930                 {
1931                         const string warning = "warning";
1932                         const string w_disable = "warning disable";
1933                         const string w_restore = "warning restore";
1934                         const string checksum = "checksum";
1935
1936                         if (arg == w_disable) {
1937                                 Report.RegisterWarningRegion (Location).WarningDisable (Location.Row);
1938                                 return;
1939                         }
1940
1941                         if (arg == w_restore) {
1942                                 Report.RegisterWarningRegion (Location).WarningEnable (Location.Row);
1943                                 return;
1944                         }
1945
1946                         if (arg.StartsWith (w_disable)) {
1947                                 int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
1948                                 foreach (int code in codes) {
1949                                         if (code != 0)
1950                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code, Report);
1951                                 }
1952                                 return;
1953                         }
1954
1955                         if (arg.StartsWith (w_restore)) {
1956                                 int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
1957                                 var w_table = Report.warning_ignore_table;
1958                                 foreach (int code in codes) {
1959                                         if (w_table != null && w_table.ContainsKey (code))
1960                                                 Report.Warning (1635, 1, Location, "Cannot restore warning `CS{0:0000}' because it was disabled globally", code);
1961                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code, Report);
1962                                 }
1963                                 return;
1964                         }
1965
1966                         if (arg.StartsWith (warning)) {
1967                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
1968                                 return;
1969                         }
1970
1971                         if (arg.StartsWith (checksum)) {
1972                                 if (!PreProcessPragmaChecksum (arg.Substring (checksum.Length)))
1973                                         Warning_InvalidPragmaChecksum ();
1974                                 return;
1975                         }
1976
1977                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
1978                 }
1979
1980                 int[] ParseNumbers (string text)
1981                 {
1982                         string[] string_array = text.Split (',');
1983                         int[] values = new int [string_array.Length];
1984                         int index = 0;
1985                         foreach (string string_code in string_array) {
1986                                 try {
1987                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1988                                 }
1989                                 catch (FormatException) {
1990                                         Report.Warning (1692, 1, Location, "Invalid number");
1991                                 }
1992                         }
1993                         return values;
1994                 }
1995
1996                 bool eval_val (string s)
1997                 {
1998                         if (s == "true")
1999                                 return true;
2000                         if (s == "false")
2001                                 return false;
2002
2003                         return file_name.IsConditionalDefined (s);
2004                 }
2005
2006                 bool pp_primary (ref string s)
2007                 {
2008                         s = s.Trim ();
2009                         int len = s.Length;
2010
2011                         if (len > 0){
2012                                 char c = s [0];
2013
2014                                 if (c == '('){
2015                                         s = s.Substring (1);
2016                                         bool val = pp_expr (ref s, false);
2017                                         if (s.Length > 0 && s [0] == ')'){
2018                                                 s = s.Substring (1);
2019                                                 return val;
2020                                         }
2021                                         Error_InvalidDirective ();
2022                                         return false;
2023                                 }
2024
2025                                 if (is_identifier_start_character (c)){
2026                                         int j = 1;
2027
2028                                         while (j < len){
2029                                                 c = s [j];
2030
2031                                                 if (is_identifier_part_character (c)){
2032                                                         j++;
2033                                                         continue;
2034                                                 }
2035                                                 bool v = eval_val (s.Substring (0, j));
2036                                                 s = s.Substring (j);
2037                                                 return v;
2038                                         }
2039                                         bool vv = eval_val (s);
2040                                         s = "";
2041                                         return vv;
2042                                 }
2043                         }
2044                         Error_InvalidDirective ();
2045                         return false;
2046                 }
2047
2048                 bool pp_unary (ref string s)
2049                 {
2050                         s = s.Trim ();
2051                         int len = s.Length;
2052
2053                         if (len > 0){
2054                                 if (s [0] == '!'){
2055                                         if (len > 1 && s [1] == '='){
2056                                                 Error_InvalidDirective ();
2057                                                 return false;
2058                                         }
2059                                         s = s.Substring (1);
2060                                         return ! pp_primary (ref s);
2061                                 } else
2062                                         return pp_primary (ref s);
2063                         } else {
2064                                 Error_InvalidDirective ();
2065                                 return false;
2066                         }
2067                 }
2068
2069                 bool pp_eq (ref string s)
2070                 {
2071                         bool va = pp_unary (ref s);
2072
2073                         s = s.Trim ();
2074                         int len = s.Length;
2075                         if (len > 0){
2076                                 if (s [0] == '='){
2077                                         if (len > 2 && s [1] == '='){
2078                                                 s = s.Substring (2);
2079                                                 return va == pp_unary (ref s);
2080                                         } else {
2081                                                 Error_InvalidDirective ();
2082                                                 return false;
2083                                         }
2084                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2085                                         s = s.Substring (2);
2086
2087                                         return va != pp_unary (ref s);
2088
2089                                 }
2090                         }
2091
2092                         return va;
2093
2094                 }
2095
2096                 bool pp_and (ref string s)
2097                 {
2098                         bool va = pp_eq (ref s);
2099
2100                         s = s.Trim ();
2101                         int len = s.Length;
2102                         if (len > 0){
2103                                 if (s [0] == '&'){
2104                                         if (len > 2 && s [1] == '&'){
2105                                                 s = s.Substring (2);
2106                                                 return (va & pp_and (ref s));
2107                                         } else {
2108                                                 Error_InvalidDirective ();
2109                                                 return false;
2110                                         }
2111                                 }
2112                         }
2113                         return va;
2114                 }
2115
2116                 //
2117                 // Evaluates an expression for `#if' or `#elif'
2118                 //
2119                 bool pp_expr (ref string s, bool isTerm)
2120                 {
2121                         bool va = pp_and (ref s);
2122                         s = s.Trim ();
2123                         int len = s.Length;
2124                         if (len > 0){
2125                                 char c = s [0];
2126
2127                                 if (c == '|'){
2128                                         if (len > 2 && s [1] == '|'){
2129                                                 s = s.Substring (2);
2130                                                 return va | pp_expr (ref s, isTerm);
2131                                         } else {
2132                                                 Error_InvalidDirective ();
2133                                                 return false;
2134                                         }
2135                                 }
2136                                 if (isTerm) {
2137                                         Error_EndLineExpected ();
2138                                         return false;
2139                                 }
2140                         }
2141
2142                         return va;
2143                 }
2144
2145                 bool eval (string s)
2146                 {
2147                         bool v = pp_expr (ref s, true);
2148                         s = s.Trim ();
2149                         if (s.Length != 0){
2150                                 return false;
2151                         }
2152
2153                         return v;
2154                 }
2155
2156                 void Error_NumericConstantTooLong ()
2157                 {
2158                         Report.Error (1021, Location, "Numeric constant too long");
2159                 }
2160
2161                 void Error_InvalidDirective ()
2162                 {
2163                         Report.Error (1517, Location, "Invalid preprocessor directive");
2164                 }
2165
2166                 void Error_UnexpectedDirective (string extra)
2167                 {
2168                         Report.Error (
2169                                 1028, Location,
2170                                 "Unexpected processor directive ({0})", extra);
2171                 }
2172
2173                 void Error_TokensSeen ()
2174                 {
2175                         Report.Error (1032, Location,
2176                                 "Cannot define or undefine preprocessor symbols after first token in file");
2177                 }
2178
2179                 void Eror_WrongPreprocessorLocation ()
2180                 {
2181                         Report.Error (1040, Location,
2182                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2183                 }
2184
2185                 void Error_EndLineExpected ()
2186                 {
2187                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2188                 }
2189
2190                 void Warning_InvalidPragmaChecksum ()
2191                 {
2192                         Report.Warning (1695, 1, Location,
2193                                         "Invalid #pragma checksum syntax; should be " +
2194                                         "#pragma checksum \"filename\" " +
2195                                         "\"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2196                 }
2197                 //
2198                 // if true, then the code continues processing the code
2199                 // if false, the code stays in a loop until another directive is
2200                 // reached.
2201                 // When caller_is_taking is false we ignore all directives except the ones
2202                 // which can help us to identify where the #if block ends
2203                 bool handle_preprocessing_directive (bool caller_is_taking)
2204                 {
2205                         string cmd, arg;
2206                         bool region_directive = false;
2207
2208                         get_cmd_arg (out cmd, out arg);
2209
2210                         // Eat any trailing whitespaces and single-line comments
2211                         if (arg.IndexOf ("//") != -1)
2212                                 arg = arg.Substring (0, arg.IndexOf ("//"));
2213                         arg = arg.Trim (simple_whitespaces);
2214
2215                         //
2216                         // The first group of pre-processing instructions is always processed
2217                         //
2218                         switch (cmd){
2219                         case "region":
2220                                 region_directive = true;
2221                                 arg = "true";
2222                                 goto case "if";
2223
2224                         case "endregion":
2225                                 if (ifstack == null || ifstack.Count == 0){
2226                                         Error_UnexpectedDirective ("no #region for this #endregion");
2227                                         return true;
2228                                 }
2229                                 int pop = ifstack.Pop ();
2230
2231                                 if ((pop & REGION) == 0)
2232                                         Report.Error (1027, Location, "Expected `#endif' directive");
2233
2234                                 return caller_is_taking;
2235
2236                         case "if":
2237                                 if (ifstack == null)
2238                                         ifstack = new Stack<int> (2);
2239
2240                                 int flags = region_directive ? REGION : 0;
2241                                 if (ifstack.Count == 0){
2242                                         flags |= PARENT_TAKING;
2243                                 } else {
2244                                         int state = ifstack.Peek ();
2245                                         if ((state & TAKING) != 0) {
2246                                                 flags |= PARENT_TAKING;
2247                                         }
2248                                 }
2249
2250                                 if (caller_is_taking && eval (arg)) {
2251                                         ifstack.Push (flags | TAKING);
2252                                         return true;
2253                                 }
2254                                 ifstack.Push (flags);
2255                                 return false;
2256
2257                         case "endif":
2258                                 if (ifstack == null || ifstack.Count == 0){
2259                                         Error_UnexpectedDirective ("no #if for this #endif");
2260                                         return true;
2261                                 } else {
2262                                         pop = ifstack.Pop ();
2263
2264                                         if ((pop & REGION) != 0)
2265                                                 Report.Error (1038, Location, "#endregion directive expected");
2266
2267                                         if (arg.Length != 0) {
2268                                                 Error_EndLineExpected ();
2269                                         }
2270
2271                                         if (ifstack.Count == 0)
2272                                                 return true;
2273
2274                                         int state = ifstack.Peek ();
2275                                         return (state & TAKING) != 0;
2276                                 }
2277
2278                         case "elif":
2279                                 if (ifstack == null || ifstack.Count == 0){
2280                                         Error_UnexpectedDirective ("no #if for this #elif");
2281                                         return true;
2282                                 } else {
2283                                         int state = ifstack.Pop ();
2284
2285                                         if ((state & REGION) != 0) {
2286                                                 Report.Error (1038, Location, "#endregion directive expected");
2287                                                 return true;
2288                                         }
2289
2290                                         if ((state & ELSE_SEEN) != 0){
2291                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2292                                                 return true;
2293                                         }
2294
2295                                         if ((state & TAKING) != 0) {
2296                                                 ifstack.Push (0);
2297                                                 return false;
2298                                         }
2299
2300                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2301                                                 ifstack.Push (state | TAKING);
2302                                                 return true;
2303                                         }
2304
2305                                         ifstack.Push (state);
2306                                         return false;
2307                                 }
2308
2309                         case "else":
2310                                 if (ifstack == null || ifstack.Count == 0){
2311                                         Error_UnexpectedDirective ("no #if for this #else");
2312                                         return true;
2313                                 } else {
2314                                         int state = ifstack.Peek ();
2315
2316                                         if ((state & REGION) != 0) {
2317                                                 Report.Error (1038, Location, "#endregion directive expected");
2318                                                 return true;
2319                                         }
2320
2321                                         if ((state & ELSE_SEEN) != 0){
2322                                                 Error_UnexpectedDirective ("#else within #else");
2323                                                 return true;
2324                                         }
2325
2326                                         ifstack.Pop ();
2327
2328                                         if (arg.Length != 0) {
2329                                                 Error_EndLineExpected ();
2330                                                 return true;
2331                                         }
2332
2333                                         bool ret = false;
2334                                         if ((state & PARENT_TAKING) != 0) {
2335                                                 ret = (state & TAKING) == 0;
2336
2337                                                 if (ret)
2338                                                         state |= TAKING;
2339                                                 else
2340                                                         state &= ~TAKING;
2341                                         }
2342
2343                                         ifstack.Push (state | ELSE_SEEN);
2344
2345                                         return ret;
2346                                 }
2347                         case "define":
2348                                 if (any_token_seen){
2349                                         Error_TokensSeen ();
2350                                         return caller_is_taking;
2351                                 }
2352                                 PreProcessDefinition (true, arg, caller_is_taking);
2353                                 return caller_is_taking;
2354
2355                         case "undef":
2356                                 if (any_token_seen){
2357                                         Error_TokensSeen ();
2358                                         return caller_is_taking;
2359                                 }
2360                                 PreProcessDefinition (false, arg, caller_is_taking);
2361                                 return caller_is_taking;
2362                         }
2363
2364                         //
2365                         // These are only processed if we are in a `taking' block
2366                         //
2367                         if (!caller_is_taking)
2368                                 return false;
2369
2370                         switch (cmd){
2371                         case "error":
2372                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2373                                 return true;
2374
2375                         case "warning":
2376                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2377                                 return true;
2378
2379                         case "pragma":
2380                                 if (RootContext.Version == LanguageVersion.ISO_1) {
2381                                         Report.FeatureIsNotAvailable (Location, "#pragma");
2382                                         return true;
2383                                 }
2384
2385                                 PreProcessPragma (arg);
2386                                 return true;
2387
2388                         case "line":
2389                                 if (!PreProcessLine (arg))
2390                                         Report.Error (
2391                                                 1576, Location,
2392                                                 "The line number specified for #line directive is missing or invalid");
2393                                 return caller_is_taking;
2394                         }
2395
2396                         Report.Error (1024, Location, "Wrong preprocessor directive");
2397                         return true;
2398
2399                 }
2400
2401                 private int consume_string (bool quoted)
2402                 {
2403                         int c;
2404                         string_builder.Length = 0;
2405
2406                         while ((c = get_char ()) != -1){
2407                                 if (c == '"'){
2408                                         if (quoted && peek_char () == '"'){
2409                                                 string_builder.Append ((char) c);
2410                                                 get_char ();
2411                                                 continue;
2412                                         } else {
2413                                                 val = new StringLiteral (string_builder.ToString (), Location);
2414                                                 return Token.LITERAL;
2415                                         }
2416                                 }
2417
2418                                 if (c == '\n'){
2419                                         if (!quoted)
2420                                                 Report.Error (1010, Location, "Newline in constant");
2421                                 }
2422
2423                                 if (!quoted){
2424                                         int surrogate;
2425                                         c = escape (c, out surrogate);
2426                                         if (c == -1)
2427                                                 return Token.ERROR;
2428                                         if (surrogate != 0) {
2429                                                 string_builder.Append ((char) c);
2430                                                 c = surrogate;
2431                                         }
2432                                 }
2433                                 string_builder.Append ((char) c);
2434                         }
2435
2436                         Report.Error (1039, Location, "Unterminated string literal");
2437                         return Token.EOF;
2438                 }
2439
2440                 private int consume_identifier (int s)
2441                 {
2442                         int res = consume_identifier (s, false);
2443
2444                         if (doc_state == XmlCommentState.Allowed)
2445                                 doc_state = XmlCommentState.NotAllowed;
2446
2447                         return res;
2448                 }
2449
2450                 int consume_identifier (int c, bool quoted)
2451                 {
2452                         //
2453                         // This method is very performance sensitive. It accounts
2454                         // for approximately 25% of all parser time
2455                         //
2456
2457                         int pos = 0;
2458                         int column = col;
2459
2460                         if (c == '\\') {
2461                                 int surrogate;
2462                                 c = escape (c, out surrogate);
2463                                 if (surrogate != 0) {
2464                                         id_builder [pos++] = (char) c;
2465                                         c = surrogate;
2466                                 }
2467                         }
2468
2469                         id_builder [pos++] = (char) c;
2470
2471                         try {
2472                                 while (true) {
2473                                         c = reader.Read ();
2474
2475                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
2476                                                 id_builder [pos++] = (char) c;
2477                                                 continue;
2478                                         }
2479
2480                                         if (c < 0x80) {
2481                                                 if (c == '\\') {
2482                                                         int surrogate;
2483                                                         c = escape (c, out surrogate);
2484                                                         if (surrogate != 0) {
2485                                                                 if (is_identifier_part_character ((char) c))
2486                                                                         id_builder[pos++] = (char) c;
2487                                                                 c = surrogate;
2488                                                         }
2489
2490                                                         continue;
2491                                                 }
2492                                         } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
2493                                                 id_builder [pos++] = (char) c;
2494                                                 continue;
2495                                         }
2496
2497                                         putback_char = c;
2498                                         break;
2499                                 }
2500                         } catch (IndexOutOfRangeException) {
2501                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2502                                 col += pos - 1;
2503                                 return Token.ERROR;
2504                         }
2505
2506                         col += pos - 1;
2507
2508                         //
2509                         // Optimization: avoids doing the keyword lookup
2510                         // on uppercase letters
2511                         //
2512                         if (id_builder [0] >= '_' && !quoted) {
2513                                 int keyword = GetKeyword (id_builder, pos);
2514                                 if (keyword != -1) {
2515                                         val = LocatedToken.Create (null, ref_line, column);
2516                                         return keyword;
2517                                 }
2518                         }
2519
2520                         //
2521                         // Keep identifiers in an array of hashtables to avoid needless
2522                         // allocations
2523                         //
2524                         var identifiers_group = identifiers [pos];
2525                         string s;
2526                         if (identifiers_group != null) {
2527                                 if (identifiers_group.TryGetValue (id_builder, out s)) {
2528                                         val = LocatedToken.Create (s, ref_line, column);
2529                                         if (quoted)
2530                                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2531                                         return Token.IDENTIFIER;
2532                                 }
2533                         } else {
2534                                 // TODO: this should be number of files dependant
2535                                 // corlib compilation peaks at 1000 and System.Core at 150
2536                                 int capacity = pos > 20 ? 10 : 100;
2537                                 identifiers_group = new Dictionary<char[],string> (capacity, new IdentifiersComparer (pos));
2538                                 identifiers [pos] = identifiers_group;
2539                         }
2540
2541                         char [] chars = new char [pos];
2542                         Array.Copy (id_builder, chars, pos);
2543
2544                         s = new string (id_builder, 0, pos);
2545                         identifiers_group.Add (chars, s);
2546
2547                         val = LocatedToken.Create (s, ref_line, column);
2548                         if (quoted)
2549                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2550
2551                         return Token.IDENTIFIER;
2552                 }
2553
2554                 public int xtoken ()
2555                 {
2556                         int d, c;
2557
2558                         // Whether we have seen comments on the current line
2559                         bool comments_seen = false;
2560                         while ((c = get_char ()) != -1) {
2561                                 switch (c) {
2562                                 case '\t':
2563                                         col = ((col + 8) / 8) * 8;
2564                                         continue;
2565
2566                                 case ' ':
2567                                 case '\f':
2568                                 case '\v':
2569                                 case 0xa0:
2570                                 case 0:
2571                                 case 0xFEFF:    // Ignore BOM anywhere in the file
2572                                         continue;
2573
2574 /*                              This is required for compatibility with .NET
2575                                 case 0xEF:
2576                                         if (peek_char () == 0xBB) {
2577                                                 PushPosition ();
2578                                                 get_char ();
2579                                                 if (get_char () == 0xBF)
2580                                                         continue;
2581                                                 PopPosition ();
2582                                         }
2583                                         break;
2584 */
2585                                 case '\r':
2586                                         if (peek_char () != '\n')
2587                                                 advance_line ();
2588                                         else
2589                                                 get_char ();
2590
2591                                         any_token_seen |= tokens_seen;
2592                                         tokens_seen = false;
2593                                         comments_seen = false;
2594                                         continue;
2595
2596                                 case '\\':
2597                                         tokens_seen = true;
2598                                         return consume_identifier (c);
2599
2600                                 case '{':
2601                                         val = LocatedToken.Create (ref_line, col);
2602                                         return Token.OPEN_BRACE;
2603                                 case '}':
2604                                         val = LocatedToken.Create (ref_line, col);
2605                                         return Token.CLOSE_BRACE;
2606                                 case '[':
2607                                         // To block doccomment inside attribute declaration.
2608                                         if (doc_state == XmlCommentState.Allowed)
2609                                                 doc_state = XmlCommentState.NotAllowed;
2610                                         return Token.OPEN_BRACKET;
2611                                 case ']':
2612                                         return Token.CLOSE_BRACKET;
2613                                 case '(':
2614                                         val = LocatedToken.Create (ref_line, col);
2615                                         //
2616                                         // An expression versions of parens can appear in block context only
2617                                         //
2618                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
2619
2620                                                 //
2621                                                 // Optmize most common case where we know that parens
2622                                                 // is not special
2623                                                 //
2624                                                 switch (current_token) {
2625                                                 case Token.IDENTIFIER:
2626                                                 case Token.IF:
2627                                                 case Token.FOR:
2628                                                 case Token.FOREACH:
2629                                                 case Token.TYPEOF:
2630                                                 case Token.WHILE:
2631                                                 case Token.USING:
2632                                                 case Token.DEFAULT:
2633                                                 case Token.DELEGATE:
2634                                                 case Token.OP_GENERICS_GT:
2635                                                         return Token.OPEN_PARENS;
2636                                                 }
2637
2638                                                 // Optimize using peek
2639                                                 int xx = peek_char ();
2640                                                 switch (xx) {
2641                                                 case '(':
2642                                                 case '\'':
2643                                                 case '"':
2644                                                 case '0':
2645                                                 case '1':
2646                                                         return Token.OPEN_PARENS;
2647                                                 }
2648
2649                                                 lambda_arguments_parsing = true;
2650                                                 PushPosition ();
2651                                                 d = TokenizeOpenParens ();
2652                                                 PopPosition ();
2653                                                 lambda_arguments_parsing = false;
2654                                                 return d;
2655                                         }
2656
2657                                         return Token.OPEN_PARENS;
2658                                 case ')':
2659                                         return Token.CLOSE_PARENS;
2660                                 case ',':
2661                                         return Token.COMMA;
2662                                 case ';':
2663                                         return Token.SEMICOLON;
2664                                 case '~':
2665                                         val = LocatedToken.Create (ref_line, col);
2666                                         return Token.TILDE;
2667                                 case '?':
2668                                         val = LocatedToken.Create (ref_line, col);
2669                                         return TokenizePossibleNullableType ();
2670                                 case '<':
2671                                         if (parsing_generic_less_than++ > 0)
2672                                                 return Token.OP_GENERICS_LT;
2673
2674                                         return TokenizeLessThan ();
2675
2676                                 case '>':
2677                                         d = peek_char ();
2678
2679                                         if (d == '='){
2680                                                 get_char ();
2681                                                 return Token.OP_GE;
2682                                         }
2683
2684                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
2685                                                 parsing_generic_less_than--;
2686                                                 return Token.OP_GENERICS_GT;
2687                                         }
2688
2689                                         if (d == '>') {
2690                                                 get_char ();
2691                                                 d = peek_char ();
2692
2693                                                 if (d == '=') {
2694                                                         get_char ();
2695                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
2696                                                 }
2697                                                 return Token.OP_SHIFT_RIGHT;
2698                                         }
2699
2700                                         return Token.OP_GT;
2701
2702                                 case '+':
2703                                         val = LocatedToken.Create (ref_line, col);
2704                                         d = peek_char ();
2705                                         if (d == '+') {
2706                                                 d = Token.OP_INC;
2707                                         } else if (d == '=') {
2708                                                 d = Token.OP_ADD_ASSIGN;
2709                                         } else {
2710                                                 return Token.PLUS;
2711                                         }
2712                                         get_char ();
2713                                         return d;
2714
2715                                 case '-':
2716                                         val = LocatedToken.Create (ref_line, col);
2717                                         d = peek_char ();
2718                                         if (d == '-') {
2719                                                 d = Token.OP_DEC;
2720                                         } else if (d == '=')
2721                                                 d = Token.OP_SUB_ASSIGN;
2722                                         else if (d == '>')
2723                                                 d = Token.OP_PTR;
2724                                         else {
2725                                                 return Token.MINUS;
2726                                         }
2727                                         get_char ();
2728                                         return d;
2729
2730                                 case '!':
2731                                         val = LocatedToken.Create (ref_line, col);
2732                                         if (peek_char () == '='){
2733                                                 get_char ();
2734                                                 return Token.OP_NE;
2735                                         }
2736                                         return Token.BANG;
2737
2738                                 case '=':
2739                                         val = LocatedToken.Create (ref_line, col);
2740                                         d = peek_char ();
2741                                         if (d == '='){
2742                                                 get_char ();
2743                                                 return Token.OP_EQ;
2744                                         }
2745                                         if (d == '>'){
2746                                                 get_char ();
2747                                                 return Token.ARROW;
2748                                         }
2749
2750                                         return Token.ASSIGN;
2751
2752                                 case '&':
2753                                         val = LocatedToken.Create (ref_line, col);
2754                                         d = peek_char ();
2755                                         if (d == '&'){
2756                                                 get_char ();
2757                                                 return Token.OP_AND;
2758                                         }
2759                                         if (d == '='){
2760                                                 get_char ();
2761                                                 return Token.OP_AND_ASSIGN;
2762                                         }
2763                                         return Token.BITWISE_AND;
2764
2765                                 case '|':
2766                                         val = LocatedToken.Create (ref_line, col);
2767                                         d = peek_char ();
2768                                         if (d == '|'){
2769                                                 get_char ();
2770                                                 return Token.OP_OR;
2771                                         }
2772                                         if (d == '='){
2773                                                 get_char ();
2774                                                 return Token.OP_OR_ASSIGN;
2775                                         }
2776                                         return Token.BITWISE_OR;
2777
2778                                 case '*':
2779                                         val = LocatedToken.Create (ref_line, col);
2780                                         if (peek_char () == '='){
2781                                                 get_char ();
2782                                                 return Token.OP_MULT_ASSIGN;
2783                                         }
2784                                         return Token.STAR;
2785
2786                                 case '/':
2787                                         d = peek_char ();
2788                                         if (d == '='){
2789                                                 val = LocatedToken.Create (ref_line, col);
2790                                                 get_char ();
2791                                                 return Token.OP_DIV_ASSIGN;
2792                                         }
2793
2794                                         // Handle double-slash comments.
2795                                         if (d == '/'){
2796                                                 get_char ();
2797                                                 if (RootContext.Documentation != null && peek_char () == '/') {
2798                                                         get_char ();
2799                                                         // Don't allow ////.
2800                                                         if ((d = peek_char ()) != '/') {
2801                                                                 update_comment_location ();
2802                                                                 if (doc_state == XmlCommentState.Allowed)
2803                                                                         handle_one_line_xml_comment ();
2804                                                                 else if (doc_state == XmlCommentState.NotAllowed)
2805                                                                         warn_incorrect_doc_comment ();
2806                                                         }
2807                                                 }
2808                                                 while ((d = get_char ()) != -1 && (d != '\n') && d != '\r');
2809
2810                                                 any_token_seen |= tokens_seen;
2811                                                 tokens_seen = false;
2812                                                 comments_seen = false;
2813                                                 continue;
2814                                         } else if (d == '*'){
2815                                                 get_char ();
2816                                                 bool docAppend = false;
2817                                                 if (RootContext.Documentation != null && peek_char () == '*') {
2818                                                         get_char ();
2819                                                         update_comment_location ();
2820                                                         // But when it is /**/, just do nothing.
2821                                                         if (peek_char () == '/') {
2822                                                                 get_char ();
2823                                                                 continue;
2824                                                         }
2825                                                         if (doc_state == XmlCommentState.Allowed)
2826                                                                 docAppend = true;
2827                                                         else if (doc_state == XmlCommentState.NotAllowed)
2828                                                                 warn_incorrect_doc_comment ();
2829                                                 }
2830
2831                                                 int current_comment_start = 0;
2832                                                 if (docAppend) {
2833                                                         current_comment_start = xml_comment_buffer.Length;
2834                                                         xml_comment_buffer.Append (Environment.NewLine);
2835                                                 }
2836
2837                                                 while ((d = get_char ()) != -1){
2838                                                         if (d == '*' && peek_char () == '/'){
2839                                                                 get_char ();
2840                                                                 comments_seen = true;
2841                                                                 break;
2842                                                         }
2843                                                         if (docAppend)
2844                                                                 xml_comment_buffer.Append ((char) d);
2845
2846                                                         if (d == '\n'){
2847                                                                 any_token_seen |= tokens_seen;
2848                                                                 tokens_seen = false;
2849                                                                 //
2850                                                                 // Reset 'comments_seen' just to be consistent.
2851                                                                 // It doesn't matter either way, here.
2852                                                                 //
2853                                                                 comments_seen = false;
2854                                                         }
2855                                                 }
2856                                                 if (!comments_seen)
2857                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
2858
2859                                                 if (docAppend)
2860                                                         update_formatted_doc_comment (current_comment_start);
2861                                                 continue;
2862                                         }
2863                                         return Token.DIV;
2864
2865                                 case '%':
2866                                         val = LocatedToken.Create (ref_line, col);
2867                                         if (peek_char () == '='){
2868                                                 get_char ();
2869                                                 return Token.OP_MOD_ASSIGN;
2870                                         }
2871                                         return Token.PERCENT;
2872
2873                                 case '^':
2874                                         val = LocatedToken.Create (ref_line, col);
2875                                         if (peek_char () == '='){
2876                                                 get_char ();
2877                                                 return Token.OP_XOR_ASSIGN;
2878                                         }
2879                                         return Token.CARRET;
2880
2881                                 case ':':
2882                                         val = LocatedToken.Create (ref_line, col);
2883                                         if (peek_char () == ':') {
2884                                                 get_char ();
2885                                                 return Token.DOUBLE_COLON;
2886                                         }
2887                                         return Token.COLON;
2888
2889                                 case '0': case '1': case '2': case '3': case '4':
2890                                 case '5': case '6': case '7': case '8': case '9':
2891                                         tokens_seen = true;
2892                                         return is_number (c);
2893
2894                                 case '\n': // white space
2895                                         any_token_seen |= tokens_seen;
2896                                         tokens_seen = false;
2897                                         comments_seen = false;
2898                                         continue;
2899
2900                                 case '.':
2901                                         tokens_seen = true;
2902                                         d = peek_char ();
2903                                         if (d >= '0' && d <= '9')
2904                                                 return is_number (c);
2905                                         return Token.DOT;
2906
2907                                 case '#':
2908                                         if (tokens_seen || comments_seen) {
2909                                                 Eror_WrongPreprocessorLocation ();
2910                                                 return Token.ERROR;
2911                                         }
2912
2913                                         if (handle_preprocessing_directive (true))
2914                                                 continue;
2915
2916                                         bool directive_expected = false;
2917                                         while ((c = get_char ()) != -1) {
2918                                                 if (col == 1) {
2919                                                         directive_expected = true;
2920                                                 } else if (!directive_expected) {
2921                                                         // TODO: Implement comment support for disabled code and uncomment this code
2922 //                                                      if (c == '#') {
2923 //                                                              Eror_WrongPreprocessorLocation ();
2924 //                                                              return Token.ERROR;
2925 //                                                      }
2926                                                         continue;
2927                                                 }
2928
2929                                                 if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
2930                                                         continue;
2931
2932                                                 if (c == '#') {
2933                                                         if (handle_preprocessing_directive (false))
2934                                                                 break;
2935                                                 }
2936                                                 directive_expected = false;
2937                                         }
2938
2939                                         if (c != -1) {
2940                                                 tokens_seen = false;
2941                                                 continue;
2942                                         }
2943
2944                                         return Token.EOF;
2945
2946                                 case '"':
2947                                         return consume_string (false);
2948
2949                                 case '\'':
2950                                         return TokenizeBackslash ();
2951
2952                                 case '@':
2953                                         c = get_char ();
2954                                         if (c == '"') {
2955                                                 tokens_seen = true;
2956                                                 return consume_string (true);
2957                                         }
2958
2959                                         if (is_identifier_start_character (c)){
2960                                                 return consume_identifier (c, true);
2961                                         }
2962
2963                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
2964                                         return Token.ERROR;
2965
2966                                 case EvalStatementParserCharacter:
2967                                         return Token.EVAL_STATEMENT_PARSER;
2968                                 case EvalCompilationUnitParserCharacter:
2969                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
2970                                 case EvalUsingDeclarationsParserCharacter:
2971                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
2972                                 }
2973
2974                                 if (is_identifier_start_character (c)) {
2975                                         tokens_seen = true;
2976                                         return consume_identifier (c);
2977                                 }
2978
2979                                 error_details = ((char)c).ToString ();
2980                                 return Token.ERROR;
2981                         }
2982
2983                         if (CompleteOnEOF){
2984                                 if (generated)
2985                                         return Token.COMPLETE_COMPLETION;
2986
2987                                 generated = true;
2988                                 return Token.GENERATE_COMPLETION;
2989                         }
2990
2991
2992                         return Token.EOF;
2993                 }
2994
2995                 int TokenizeBackslash ()
2996                 {
2997                         int c = get_char ();
2998                         tokens_seen = true;
2999                         if (c == '\'') {
3000                                 error_details = "Empty character literal";
3001                                 Report.Error (1011, Location, error_details);
3002                                 return Token.ERROR;
3003                         }
3004                         if (c == '\r' || c == '\n') {
3005                                 Report.Error (1010, Location, "Newline in constant");
3006                                 return Token.ERROR;
3007                         }
3008
3009                         int d;
3010                         c = escape (c, out d);
3011                         if (c == -1)
3012                                 return Token.ERROR;
3013                         if (d != 0)
3014                                 throw new NotImplementedException ();
3015
3016                         val = new CharLiteral ((char) c, Location);
3017                         c = get_char ();
3018
3019                         if (c != '\'') {
3020                                 Report.Error (1012, Location, "Too many characters in character literal");
3021
3022                                 // Try to recover, read until newline or next "'"
3023                                 while ((c = get_char ()) != -1) {
3024                                         if (c == '\n' || c == '\'')
3025                                                 break;
3026                                 }
3027                                 return Token.ERROR;
3028                         }
3029
3030                         return Token.LITERAL;
3031                 }
3032
3033                 int TokenizeLessThan ()
3034                 {
3035                         int d;
3036                         if (handle_typeof) {
3037                                 PushPosition ();
3038                                 if (parse_generic_dimension (out d)) {
3039                                         val = d;
3040                                         DiscardPosition ();
3041                                         return Token.GENERIC_DIMENSION;
3042                                 }
3043                                 PopPosition ();
3044                         }
3045
3046                         // Save current position and parse next token.
3047                         PushPosition ();
3048                         if (parse_less_than ()) {
3049                                 if (parsing_generic_declaration && token () != Token.DOT) {
3050                                         d = Token.OP_GENERICS_LT_DECL;
3051                                 } else {
3052                                         d = Token.OP_GENERICS_LT;
3053                                 }
3054                                 PopPosition ();
3055                                 return d;
3056                         }
3057
3058                         PopPosition ();
3059                         parsing_generic_less_than = 0;
3060
3061                         d = peek_char ();
3062                         if (d == '<') {
3063                                 get_char ();
3064                                 d = peek_char ();
3065
3066                                 if (d == '=') {
3067                                         get_char ();
3068                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3069                                 }
3070                                 return Token.OP_SHIFT_LEFT;
3071                         }
3072
3073                         if (d == '=') {
3074                                 get_char ();
3075                                 return Token.OP_LE;
3076                         }
3077                         return Token.OP_LT;
3078                 }
3079
3080                 //
3081                 // Handles one line xml comment
3082                 //
3083                 private void handle_one_line_xml_comment ()
3084                 {
3085                         int c;
3086                         while ((c = peek_char ()) == ' ')
3087                                 get_char (); // skip heading whitespaces.
3088                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3089                                 xml_comment_buffer.Append ((char) get_char ());
3090                         }
3091                         if (c == '\r' || c == '\n')
3092                                 xml_comment_buffer.Append (Environment.NewLine);
3093                 }
3094
3095                 //
3096                 // Remove heading "*" in Javadoc-like xml documentation.
3097                 //
3098                 private void update_formatted_doc_comment (int current_comment_start)
3099                 {
3100                         int length = xml_comment_buffer.Length - current_comment_start;
3101                         string [] lines = xml_comment_buffer.ToString (
3102                                 current_comment_start,
3103                                 length).Replace ("\r", "").Split ('\n');
3104
3105                         // The first line starts with /**, thus it is not target
3106                         // for the format check.
3107                         for (int i = 1; i < lines.Length; i++) {
3108                                 string s = lines [i];
3109                                 int idx = s.IndexOf ('*');
3110                                 string head = null;
3111                                 if (idx < 0) {
3112                                         if (i < lines.Length - 1)
3113                                                 return;
3114                                         head = s;
3115                                 } else
3116                                         head = s.Substring (0, idx);
3117                                 foreach (char c in head)
3118                                         if (c != ' ')
3119                                                 return;
3120                                 lines [i] = s.Substring (idx + 1);
3121                         }
3122                         xml_comment_buffer.Remove (current_comment_start, length);
3123                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3124                 }
3125
3126                 //
3127                 // Updates current comment location.
3128                 //
3129                 private void update_comment_location ()
3130                 {
3131                         if (current_comment_location.IsNull) {
3132                                 // "-2" is for heading "//" or "/*"
3133                                 current_comment_location =
3134                                         new Location (ref_line, hidden ? -1 : col - 2);
3135                         }
3136                 }
3137
3138                 //
3139                 // Checks if there was incorrect doc comments and raise
3140                 // warnings.
3141                 //
3142                 public void check_incorrect_doc_comment ()
3143                 {
3144                         if (xml_comment_buffer.Length > 0)
3145                                 warn_incorrect_doc_comment ();
3146                 }
3147
3148                 //
3149                 // Raises a warning when tokenizer found incorrect doccomment
3150                 // markup.
3151                 //
3152                 private void warn_incorrect_doc_comment ()
3153                 {
3154                         if (doc_state != XmlCommentState.Error) {
3155                                 doc_state = XmlCommentState.Error;
3156                                 // in csc, it is 'XML comment is not placed on
3157                                 // a valid language element'. But that does not
3158                                 // make sense.
3159                                 Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
3160                         }
3161                 }
3162
3163                 //
3164                 // Consumes the saved xml comment lines (if any)
3165                 // as for current target member or type.
3166                 //
3167                 public string consume_doc_comment ()
3168                 {
3169                         if (xml_comment_buffer.Length > 0) {
3170                                 string ret = xml_comment_buffer.ToString ();
3171                                 reset_doc_comment ();
3172                                 return ret;
3173                         }
3174                         return null;
3175                 }
3176
3177                 Report Report {
3178                         get { return context.Report; }
3179                 }
3180
3181                 void reset_doc_comment ()
3182                 {
3183                         xml_comment_buffer.Length = 0;
3184                         current_comment_location = Location.Null;
3185                 }
3186
3187                 public void cleanup ()
3188                 {
3189                         if (ifstack != null && ifstack.Count >= 1) {
3190                                 int state = ifstack.Pop ();
3191                                 if ((state & REGION) != 0)
3192                                         Report.Error (1038, Location, "#endregion directive expected");
3193                                 else
3194                                         Report.Error (1027, Location, "Expected `#endif' directive");
3195                         }
3196                 }
3197         }
3198
3199         //
3200         // Indicates whether it accepts XML documentation or not.
3201         //
3202         public enum XmlCommentState {
3203                 // comment is allowed in this state.
3204                 Allowed,
3205                 // comment is not allowed in this state.
3206                 NotAllowed,
3207                 // once comments appeared when it is NotAllowed, then the
3208                 // state is changed to it, until the state is changed to
3209                 // .Allowed.
3210                 Error
3211         }
3212 }
3213