mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 //
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.IO;
  19 using System.Globalization;
  20 using System.Reflection;
  21 using System.Diagnostics;
  22
  23 namespace Mono.CSharp
  24 {
  25         /// <summary>
  26         ///    Tokenizer for C# source code.
  27         /// </summary>
  28
  29         public class Tokenizer : yyParser.yyInput
  30         {
  31                 class KeywordEntry
  32                 {
  33                         public readonly int Token;
  34                         public KeywordEntry Next;
  35                         public readonly char[] Value;
  36
  37                         public KeywordEntry (string value, int token)
  38                         {
  39                                 this.Value = value.ToCharArray ();
  40                                 this.Token = token;
  41                         }
  42                 }
  43
  44                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  45                 {
  46                         readonly int length;
  47
  48                         public IdentifiersComparer (int length)
  49                         {
  50                                 this.length = length;
  51                         }
  52
  53                         public bool Equals (char[] x, char[] y)
  54                         {
  55                                 for (int i = 0; i < length; ++i)
  56                                         if (x [i] != y [i])
  57                                                 return false;
  58
  59                                 return true;
  60                         }
  61
  62                         public int GetHashCode (char[] obj)
  63                         {
  64                                 int h = 0;
  65                                 for (int i = 0; i < length; ++i)
  66                                         h = (h << 5) - h + obj [i];
  67
  68                                 return h;
  69                         }
  70                 }
  71
  72                 //
  73                 // This class has to be used in the parser only, it reuses token
  74                 // details after each parse
  75                 //
  76                 public class LocatedToken
  77                 {
  78                         int row, column;
  79                         string value;
  80
  81                         static LocatedToken[] buffer;
  82                         static int pos;
  83
  84                         private LocatedToken ()
  85                         {
  86                         }
  87
  88                         public static LocatedToken Create (int row, int column)
  89                         {
  90                                 return Create (null, row, column);
  91                         }
  92
  93                         public static LocatedToken Create (string value, int row, int column)
  94                         {
  95                                 //
  96                                 // TODO: I am not very happy about the logic but it's the best
  97                                 // what I could come up with for now.
  98                                 // Ideally we should be using just tiny buffer (256 elements) which
  99                                 // is enough to hold all details for currect stack and recycle elements
 100                                 // poped from the stack but there is a trick needed to recycle
 101                                 // them properly.
 102                                 //
 103                                 LocatedToken entry;
 104                                 if (pos >= buffer.Length) {
 105                                         entry = new LocatedToken ();
 106                                 } else {
 107                                         entry = buffer [pos];
 108                                         if (entry == null) {
 109                                                 entry = new LocatedToken ();
 110                                                 buffer [pos] = entry;
 111                                         }
 112
 113                                         ++pos;
 114                                 }
 115                                 entry.value = value;
 116                                 entry.row = row;
 117                                 entry.column = column;
 118                                 return entry;
 119                         }
 120
 121                         //
 122                         // Used for token not required by expression evaluator
 123                         //
 124                         [Conditional ("FULL_AST")]
 125                         public static void CreateOptional (int row, int col, ref object token)
 126                         {
 127                                 token = Create (row, col);
 128                         }
 129
 130                         public static void Initialize ()
 131                         {
 132                                 if (buffer == null)
 133                                         buffer = new LocatedToken [10000];
 134                                 pos = 0;
 135                         }
 136
 137                         public Location Location {
 138                                 get { return new Location (row, column); }
 139                         }
 140
 141                         public string Value {
 142                                 get { return value; }
 143                         }
 144                 }
 145
 146                 SeekableStreamReader reader;
 147                 SourceFile ref_name;
 148                 CompilationUnit file_name;
 149                 CompilerContext context;
 150                 bool hidden = false;
 151                 int ref_line = 1;
 152                 int line = 1;
 153                 int col = 0;
 154                 int previous_col;
 155                 int current_token;
 156                 int tab_size;
 157                 bool handle_get_set = false;
 158                 bool handle_remove_add = false;
 159                 bool handle_where = false;
 160                 bool handle_typeof = false;
 161                 bool lambda_arguments_parsing;
 162                 Location current_comment_location = Location.Null;
 163                 List<Location> escaped_identifiers;
 164                 int parsing_generic_less_than;
 165
 166                 //
 167                 // Used mainly for parser optimizations. Some expressions for instance
 168                 // can appear only in block (including initializer, base initializer)
 169                 // scope only
 170                 //
 171                 public int parsing_block;
 172                 internal bool query_parsing;
 173
 174                 //
 175                 // When parsing type only, useful for ambiguous nullable types
 176                 //
 177                 public int parsing_type;
 178
 179                 //
 180                 // Set when parsing generic declaration (type or method header)
 181                 //
 182                 public bool parsing_generic_declaration;
 183
 184                 //
 185                 // The value indicates that we have not reach any declaration or
 186                 // namespace yet
 187                 //
 188                 public int parsing_declaration;
 189
 190                 //
 191                 // The special character to inject on streams to trigger the EXPRESSION_PARSE
 192                 // token to be returned.   It just happens to be a Unicode character that
 193                 // would never be part of a program (can not be an identifier).
 194                 //
 195                 // This character is only tested just before the tokenizer is about to report
 196                 // an error;   So on the regular operation mode, this addition will have no
 197                 // impact on the tokenizer's performance.
 198                 //
 199
 200                 public const int EvalStatementParserCharacter = 0x2190;   // Unicode Left Arrow
 201                 public const int EvalCompilationUnitParserCharacter = 0x2191;  // Unicode Arrow
 202                 public const int EvalUsingDeclarationsParserCharacter = 0x2192;  // Unicode Arrow
 203
 204                 //
 205                 // XML documentation buffer. The save point is used to divide
 206                 // comments on types and comments on members.
 207                 //
 208                 StringBuilder xml_comment_buffer;
 209
 210                 //
 211                 // See comment on XmlCommentState enumeration.
 212                 //
 213                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 214
 215                 //
 216                 // Whether tokens have been seen on this line
 217                 //
 218                 bool tokens_seen = false;
 219
 220                 //
 221                 // Set to true once the GENERATE_COMPLETION token has bee
 222                 // returned.   This helps produce one GENERATE_COMPLETION,
 223                 // as many COMPLETE_COMPLETION as necessary to complete the
 224                 // AST tree and one final EOF.
 225                 //
 226                 bool generated;
 227
 228                 //
 229                 // Whether a token has been seen on the file
 230                 // This is needed because `define' is not allowed to be used
 231                 // after a token has been seen.
 232                 //
 233                 bool any_token_seen = false;
 234
 235                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 236                 static readonly char[] pragma_value_separator = new char[] { ',' };
 237
 238                 public bool PropertyParsing {
 239                         get { return handle_get_set; }
 240                         set { handle_get_set = value; }
 241                 }
 242
 243                 public bool EventParsing {
 244                         get { return handle_remove_add; }
 245                         set { handle_remove_add = value; }
 246                 }
 247
 248                 public bool ConstraintsParsing {
 249                         get { return handle_where; }
 250                         set { handle_where = value; }
 251                 }
 252
 253                 public bool TypeOfParsing {
 254                         get { return handle_typeof; }
 255                         set { handle_typeof = value; }
 256                 }
 257
 258                 public int TabSize {
 259                         get { return tab_size; }
 260                         set { tab_size = value; }
 261                 }
 262
 263                 public XmlCommentState doc_state {
 264                         get { return xml_doc_state; }
 265                         set {
 266                                 if (value == XmlCommentState.Allowed) {
 267                                         check_incorrect_doc_comment ();
 268                                         reset_doc_comment ();
 269                                 }
 270                                 xml_doc_state = value;
 271                         }
 272                 }
 273
 274                 //
 275                 // This is used to trigger completion generation on the parser
 276                 public bool CompleteOnEOF;
 277
 278                 void AddEscapedIdentifier (Location loc)
 279                 {
 280                         if (escaped_identifiers == null)
 281                                 escaped_identifiers = new List<Location> ();
 282
 283                         escaped_identifiers.Add (loc);
 284                 }
 285
 286                 public bool IsEscapedIdentifier (Location loc)
 287                 {
 288                         if (escaped_identifiers != null) {
 289                                 foreach (Location lt in escaped_identifiers)
 290                                         if (lt.Equals (loc))
 291                                                 return true;
 292                         }
 293
 294                         return false;
 295                 }
 296
 297                 //
 298                 // Class variables
 299                 //
 300                 static KeywordEntry[][] keywords;
 301                 static Dictionary<string, object> keyword_strings;              // TODO: HashSet
 302                 static NumberStyles styles;
 303                 static NumberFormatInfo csharp_format_info;
 304
 305                 //
 306                 // Values for the associated token returned
 307                 //
 308                 internal int putback_char;      // Used by repl only
 309                 object val;
 310
 311                 //
 312                 // Pre-processor
 313                 //
 314                 const int TAKING        = 1;
 315                 const int ELSE_SEEN     = 4;
 316                 const int PARENT_TAKING = 8;
 317                 const int REGION        = 16;
 318
 319                 //
 320                 // pre-processor if stack state:
 321                 //
 322                 Stack<int> ifstack;
 323
 324                 static System.Text.StringBuilder string_builder;
 325
 326                 const int max_id_size = 512;
 327                 static char [] id_builder = new char [max_id_size];
 328
 329                 public static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 330
 331                 const int max_number_size = 512;
 332                 static char [] number_builder = new char [max_number_size];
 333                 static int number_pos;
 334
 335                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
 336
 337                 //
 338                 // Details about the error encoutered by the tokenizer
 339                 //
 340                 string error_details;
 341
 342                 public string error {
 343                         get {
 344                                 return error_details;
 345                         }
 346                 }
 347
 348                 public int Line {
 349                         get {
 350                                 return ref_line;
 351                         }
 352                 }
 353
 354                 //
 355                 // This is used when the tokenizer needs to save
 356                 // the current position as it needs to do some parsing
 357                 // on its own to deamiguate a token in behalf of the
 358                 // parser.
 359                 //
 360                 Stack<Position> position_stack = new Stack<Position> (2);
 361
 362                 class Position {
 363                         public int position;
 364                         public int line;
 365                         public int ref_line;
 366                         public int col;
 367                         public bool hidden;
 368                         public int putback_char;
 369                         public int previous_col;
 370                         public Stack<int> ifstack;
 371                         public int parsing_generic_less_than;
 372                         public int current_token;
 373                         public object val;
 374
 375                         public Position (Tokenizer t)
 376                         {
 377                                 position = t.reader.Position;
 378                                 line = t.line;
 379                                 ref_line = t.ref_line;
 380                                 col = t.col;
 381                                 hidden = t.hidden;
 382                                 putback_char = t.putback_char;
 383                                 previous_col = t.previous_col;
 384                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 385                                         // There is no simple way to clone Stack<T> all
 386                                         // methods reverse the order
 387                                         var clone = t.ifstack.ToArray ();
 388                                         Array.Reverse (clone);
 389                                         ifstack = new Stack<int> (clone);
 390                                 }
 391                                 parsing_generic_less_than = t.parsing_generic_less_than;
 392                                 current_token = t.current_token;
 393                                 val = t.val;
 394                         }
 395                 }
 396
 397                 public void PushPosition ()
 398                 {
 399                         position_stack.Push (new Position (this));
 400                 }
 401
 402                 public void PopPosition ()
 403                 {
 404                         Position p = position_stack.Pop ();
 405
 406                         reader.Position = p.position;
 407                         ref_line = p.ref_line;
 408                         line = p.line;
 409                         col = p.col;
 410                         hidden = p.hidden;
 411                         putback_char = p.putback_char;
 412                         previous_col = p.previous_col;
 413                         ifstack = p.ifstack;
 414                         parsing_generic_less_than = p.parsing_generic_less_than;
 415                         current_token = p.current_token;
 416                         val = p.val;
 417                 }
 418
 419                 // Do not reset the position, ignore it.
 420                 public void DiscardPosition ()
 421                 {
 422                         position_stack.Pop ();
 423                 }
 424
 425                 static void AddKeyword (string kw, int token)
 426                 {
 427                         keyword_strings.Add (kw, null);
 428
 429                         int length = kw.Length;
 430                         if (keywords [length] == null) {
 431                                 keywords [length] = new KeywordEntry ['z' - '_' + 1];
 432                         }
 433
 434                         int char_index = kw [0] - '_';
 435                         KeywordEntry kwe = keywords [length] [char_index];
 436                         if (kwe == null) {
 437                                 keywords [length] [char_index] = new KeywordEntry (kw, token);
 438                                 return;
 439                         }
 440
 441                         while (kwe.Next != null) {
 442                                 kwe = kwe.Next;
 443                         }
 444
 445                         kwe.Next = new KeywordEntry (kw, token);
 446                 }
 447
 448                 static void InitTokens ()
 449                 {
 450                         keyword_strings = new Dictionary<string, object> ();
 451
 452                         // 11 is the length of the longest keyword for now
 453                         keywords = new KeywordEntry [11] [];
 454
 455                         AddKeyword ("__arglist", Token.ARGLIST);
 456                         AddKeyword ("abstract", Token.ABSTRACT);
 457                         AddKeyword ("as", Token.AS);
 458                         AddKeyword ("add", Token.ADD);
 459                         AddKeyword ("base", Token.BASE);
 460                         AddKeyword ("bool", Token.BOOL);
 461                         AddKeyword ("break", Token.BREAK);
 462                         AddKeyword ("byte", Token.BYTE);
 463                         AddKeyword ("case", Token.CASE);
 464                         AddKeyword ("catch", Token.CATCH);
 465                         AddKeyword ("char", Token.CHAR);
 466                         AddKeyword ("checked", Token.CHECKED);
 467                         AddKeyword ("class", Token.CLASS);
 468                         AddKeyword ("const", Token.CONST);
 469                         AddKeyword ("continue", Token.CONTINUE);
 470                         AddKeyword ("decimal", Token.DECIMAL);
 471                         AddKeyword ("default", Token.DEFAULT);
 472                         AddKeyword ("delegate", Token.DELEGATE);
 473                         AddKeyword ("do", Token.DO);
 474                         AddKeyword ("double", Token.DOUBLE);
 475                         AddKeyword ("else", Token.ELSE);
 476                         AddKeyword ("enum", Token.ENUM);
 477                         AddKeyword ("event", Token.EVENT);
 478                         AddKeyword ("explicit", Token.EXPLICIT);
 479                         AddKeyword ("extern", Token.EXTERN);
 480                         AddKeyword ("false", Token.FALSE);
 481                         AddKeyword ("finally", Token.FINALLY);
 482                         AddKeyword ("fixed", Token.FIXED);
 483                         AddKeyword ("float", Token.FLOAT);
 484                         AddKeyword ("for", Token.FOR);
 485                         AddKeyword ("foreach", Token.FOREACH);
 486                         AddKeyword ("goto", Token.GOTO);
 487                         AddKeyword ("get", Token.GET);
 488                         AddKeyword ("if", Token.IF);
 489                         AddKeyword ("implicit", Token.IMPLICIT);
 490                         AddKeyword ("in", Token.IN);
 491                         AddKeyword ("int", Token.INT);
 492                         AddKeyword ("interface", Token.INTERFACE);
 493                         AddKeyword ("internal", Token.INTERNAL);
 494                         AddKeyword ("is", Token.IS);
 495                         AddKeyword ("lock", Token.LOCK);
 496                         AddKeyword ("long", Token.LONG);
 497                         AddKeyword ("namespace", Token.NAMESPACE);
 498                         AddKeyword ("new", Token.NEW);
 499                         AddKeyword ("null", Token.NULL);
 500                         AddKeyword ("object", Token.OBJECT);
 501                         AddKeyword ("operator", Token.OPERATOR);
 502                         AddKeyword ("out", Token.OUT);
 503                         AddKeyword ("override", Token.OVERRIDE);
 504                         AddKeyword ("params", Token.PARAMS);
 505                         AddKeyword ("private", Token.PRIVATE);
 506                         AddKeyword ("protected", Token.PROTECTED);
 507                         AddKeyword ("public", Token.PUBLIC);
 508                         AddKeyword ("readonly", Token.READONLY);
 509                         AddKeyword ("ref", Token.REF);
 510                         AddKeyword ("remove", Token.REMOVE);
 511                         AddKeyword ("return", Token.RETURN);
 512                         AddKeyword ("sbyte", Token.SBYTE);
 513                         AddKeyword ("sealed", Token.SEALED);
 514                         AddKeyword ("set", Token.SET);
 515                         AddKeyword ("short", Token.SHORT);
 516                         AddKeyword ("sizeof", Token.SIZEOF);
 517                         AddKeyword ("stackalloc", Token.STACKALLOC);
 518                         AddKeyword ("static", Token.STATIC);
 519                         AddKeyword ("string", Token.STRING);
 520                         AddKeyword ("struct", Token.STRUCT);
 521                         AddKeyword ("switch", Token.SWITCH);
 522                         AddKeyword ("this", Token.THIS);
 523                         AddKeyword ("throw", Token.THROW);
 524                         AddKeyword ("true", Token.TRUE);
 525                         AddKeyword ("try", Token.TRY);
 526                         AddKeyword ("typeof", Token.TYPEOF);
 527                         AddKeyword ("uint", Token.UINT);
 528                         AddKeyword ("ulong", Token.ULONG);
 529                         AddKeyword ("unchecked", Token.UNCHECKED);
 530                         AddKeyword ("unsafe", Token.UNSAFE);
 531                         AddKeyword ("ushort", Token.USHORT);
 532                         AddKeyword ("using", Token.USING);
 533                         AddKeyword ("virtual", Token.VIRTUAL);
 534                         AddKeyword ("void", Token.VOID);
 535                         AddKeyword ("volatile", Token.VOLATILE);
 536                         AddKeyword ("while", Token.WHILE);
 537                         AddKeyword ("partial", Token.PARTIAL);
 538                         AddKeyword ("where", Token.WHERE);
 539
 540                         // LINQ keywords
 541                         AddKeyword ("from", Token.FROM);
 542                         AddKeyword ("join", Token.JOIN);
 543                         AddKeyword ("on", Token.ON);
 544                         AddKeyword ("equals", Token.EQUALS);
 545                         AddKeyword ("select", Token.SELECT);
 546                         AddKeyword ("group", Token.GROUP);
 547                         AddKeyword ("by", Token.BY);
 548                         AddKeyword ("let", Token.LET);
 549                         AddKeyword ("orderby", Token.ORDERBY);
 550                         AddKeyword ("ascending", Token.ASCENDING);
 551                         AddKeyword ("descending", Token.DESCENDING);
 552                         AddKeyword ("into", Token.INTO);
 553                 }
 554
 555                 //
 556                 // Class initializer
 557                 //
 558                 static Tokenizer ()
 559                 {
 560                         InitTokens ();
 561                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 562                         styles = NumberStyles.Float;
 563
 564                         string_builder = new System.Text.StringBuilder ();
 565                 }
 566
 567                 int GetKeyword (char[] id, int id_len)
 568                 {
 569                         //
 570                         // Keywords are stored in an array of arrays grouped by their
 571                         // length and then by the first character
 572                         //
 573                         if (id_len >= keywords.Length || keywords [id_len] == null)
 574                                 return -1;
 575
 576                         int first_index = id [0] - '_';
 577                         if (first_index > 'z' - '_')
 578                                 return -1;
 579
 580                         KeywordEntry kwe = keywords [id_len] [first_index];
 581                         if (kwe == null)
 582                                 return -1;
 583
 584                         int res;
 585                         do {
 586                                 res = kwe.Token;
 587                                 for (int i = 1; i < id_len; ++i) {
 588                                         if (id [i] != kwe.Value [i]) {
 589                                                 res = 0;
 590                                                 kwe = kwe.Next;
 591                                                 break;
 592                                         }
 593                                 }
 594                         } while (res == 0 && kwe != null);
 595
 596                         if (res == 0)
 597                                 return -1;
 598
 599                         int next_token;
 600                         switch (res) {
 601                         case Token.GET:
 602                         case Token.SET:
 603                                 if (!handle_get_set)
 604                                         res = -1;
 605                                 break;
 606                         case Token.REMOVE:
 607                         case Token.ADD:
 608                                 if (!handle_remove_add)
 609                                         res = -1;
 610                                 break;
 611                         case Token.EXTERN:
 612                                 if (parsing_declaration == 0)
 613                                         res = Token.EXTERN_ALIAS;
 614                                 break;
 615                         case Token.DEFAULT:
 616                                 if (peek_token () == Token.COLON) {
 617                                         token ();
 618                                         res = Token.DEFAULT_COLON;
 619                                 }
 620                                 break;
 621                         case Token.WHERE:
 622                                 if (!handle_where && !query_parsing)
 623                                         res = -1;
 624                                 break;
 625                         case Token.FROM:
 626                                 //
 627                                 // A query expression is any expression that starts with `from identifier'
 628                                 // followed by any token except ; , =
 629                                 //
 630                                 if (!query_parsing) {
 631                                         if (lambda_arguments_parsing) {
 632                                                 res = -1;
 633                                                 break;
 634                                         }
 635
 636                                         PushPosition ();
 637                                         // HACK: to disable generics micro-parser, because PushPosition does not
 638                                         // store identifiers array
 639                                         parsing_generic_less_than = 1;
 640                                         switch (xtoken ()) {
 641                                         case Token.IDENTIFIER:
 642                                         case Token.INT:
 643                                         case Token.BOOL:
 644                                         case Token.BYTE:
 645                                         case Token.CHAR:
 646                                         case Token.DECIMAL:
 647                                         case Token.FLOAT:
 648                                         case Token.LONG:
 649                                         case Token.OBJECT:
 650                                         case Token.STRING:
 651                                         case Token.UINT:
 652                                         case Token.ULONG:
 653                                                 next_token = xtoken ();
 654                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 655                                                         goto default;
 656
 657                                                 res = Token.FROM_FIRST;
 658                                                 query_parsing = true;
 659                                                 if (RootContext.Version <= LanguageVersion.ISO_2)
 660                                                         Report.FeatureIsNotAvailable (Location, "query expressions");
 661                                                 break;
 662                                         case Token.VOID:
 663                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 664                                                 break;
 665                                         default:
 666                                                 PopPosition ();
 667                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 668                                                 // which has been overwritten before we grabbed the identifier
 669                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 670                                                 return -1;
 671                                         }
 672                                         PopPosition ();
 673                                 }
 674                                 break;
 675                         case Token.JOIN:
 676                         case Token.ON:
 677                         case Token.EQUALS:
 678                         case Token.SELECT:
 679                         case Token.GROUP:
 680                         case Token.BY:
 681                         case Token.LET:
 682                         case Token.ORDERBY:
 683                         case Token.ASCENDING:
 684                         case Token.DESCENDING:
 685                         case Token.INTO:
 686                                 if (!query_parsing)
 687                                         res = -1;
 688                                 break;
 689
 690                         case Token.USING:
 691                         case Token.NAMESPACE:
 692                                 // TODO: some explanation needed
 693                                 check_incorrect_doc_comment ();
 694                                 break;
 695
 696                         case Token.PARTIAL:
 697                                 if (parsing_block > 0) {
 698                                         res = -1;
 699                                         break;
 700                                 }
 701
 702                                 // Save current position and parse next token.
 703                                 PushPosition ();
 704
 705                                 next_token = token ();
 706                                 bool ok = (next_token == Token.CLASS) ||
 707                                         (next_token == Token.STRUCT) ||
 708                                         (next_token == Token.INTERFACE) ||
 709                                         (next_token == Token.VOID);
 710
 711                                 PopPosition ();
 712
 713                                 if (ok) {
 714                                         if (next_token == Token.VOID) {
 715                                                 if (RootContext.Version == LanguageVersion.ISO_1 ||
 716                                                     RootContext.Version == LanguageVersion.ISO_2)
 717                                                         Report.FeatureIsNotAvailable (Location, "partial methods");
 718                                         } else if (RootContext.Version == LanguageVersion.ISO_1)
 719                                                 Report.FeatureIsNotAvailable (Location, "partial types");
 720
 721                                         return res;
 722                                 }
 723
 724                                 if (next_token < Token.LAST_KEYWORD) {
 725                                         Report.Error (267, Location,
 726                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 727                                         return token ();
 728                                 }
 729
 730                                 res = -1;
 731                                 break;
 732                         }
 733
 734                         return res;
 735                 }
 736
 737                 public Location Location {
 738                         get {
 739                                 return new Location (ref_line, hidden ? -1 : col);
 740                         }
 741                 }
 742
 743                 public Tokenizer (SeekableStreamReader input, CompilationUnit file, CompilerContext ctx)
 744                 {
 745                         this.ref_name = file;
 746                         this.file_name = file;
 747                         this.context = ctx;
 748                         reader = input;
 749
 750                         putback_char = -1;
 751
 752                         xml_comment_buffer = new StringBuilder ();
 753
 754                         if (Environment.OSVersion.Platform == PlatformID.Win32NT)
 755                                 tab_size = 4;
 756                         else
 757                                 tab_size = 8;
 758
 759                         //
 760                         // FIXME: This could be `Location.Push' but we have to
 761                         // find out why the MS compiler allows this
 762                         //
 763                         Mono.CSharp.Location.Push (file, file);
 764                 }
 765
 766                 static bool is_identifier_start_character (int c)
 767                 {
 768                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 769                 }
 770
 771                 static bool is_identifier_part_character (char c)
 772                 {
 773                         if (c >= 'a' && c <= 'z')
 774                                 return true;
 775
 776                         if (c >= 'A' && c <= 'Z')
 777                                 return true;
 778
 779                         if (c == '_' || (c >= '0' && c <= '9'))
 780                                 return true;
 781
 782                         if (c < 0x80)
 783                                 return false;
 784
 785                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 786                 }
 787
 788                 public static bool IsKeyword (string s)
 789                 {
 790                         return keyword_strings.ContainsKey (s);
 791                 }
 792
 793                 //
 794                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 795                 //
 796                 int TokenizeOpenParens ()
 797                 {
 798                         int ptoken;
 799                         current_token = -1;
 800
 801                         int bracket_level = 0;
 802                         bool is_type = false;
 803                         bool can_be_type = false;
 804
 805                         while (true) {
 806                                 ptoken = current_token;
 807                                 token ();
 808
 809                                 switch (current_token) {
 810                                 case Token.CLOSE_PARENS:
 811                                         token ();
 812
 813                                         //
 814                                         // Expression inside parens is lambda, (int i) =>
 815                                         //
 816                                         if (current_token == Token.ARROW)
 817                                                 return Token.OPEN_PARENS_LAMBDA;
 818
 819                                         //
 820                                         // Expression inside parens is single type, (int[])
 821                                         //
 822                                         if (is_type)
 823                                                 return Token.OPEN_PARENS_CAST;
 824
 825                                         //
 826                                         // Expression is possible cast, look at next token, (T)null
 827                                         //
 828                                         if (can_be_type) {
 829                                                 switch (current_token) {
 830                                                 case Token.OPEN_PARENS:
 831                                                 case Token.BANG:
 832                                                 case Token.TILDE:
 833                                                 case Token.IDENTIFIER:
 834                                                 case Token.LITERAL:
 835                                                 case Token.BASE:
 836                                                 case Token.CHECKED:
 837                                                 case Token.DELEGATE:
 838                                                 case Token.FALSE:
 839                                                 case Token.FIXED:
 840                                                 case Token.NEW:
 841                                                 case Token.NULL:
 842                                                 case Token.SIZEOF:
 843                                                 case Token.THIS:
 844                                                 case Token.THROW:
 845                                                 case Token.TRUE:
 846                                                 case Token.TYPEOF:
 847                                                 case Token.UNCHECKED:
 848                                                 case Token.UNSAFE:
 849                                                 case Token.DEFAULT:
 850
 851                                                 //
 852                                                 // These can be part of a member access
 853                                                 //
 854                                                 case Token.INT:
 855                                                 case Token.UINT:
 856                                                 case Token.SHORT:
 857                                                 case Token.USHORT:
 858                                                 case Token.LONG:
 859                                                 case Token.ULONG:
 860                                                 case Token.DOUBLE:
 861                                                 case Token.FLOAT:
 862                                                 case Token.CHAR:
 863                                                 case Token.BYTE:
 864                                                 case Token.DECIMAL:
 865                                                 case Token.BOOL:
 866                                                         return Token.OPEN_PARENS_CAST;
 867                                                 }
 868                                         }
 869                                         return Token.OPEN_PARENS;
 870
 871                                 case Token.DOT:
 872                                 case Token.DOUBLE_COLON:
 873                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
 874                                                 goto default;
 875
 876                                         continue;
 877
 878                                 case Token.IDENTIFIER:
 879                                         switch (ptoken) {
 880                                         case Token.DOT:
 881                                         case Token.OP_GENERICS_LT:
 882                                         case Token.COMMA:
 883                                         case Token.DOUBLE_COLON:
 884                                         case -1:
 885                                                 if (bracket_level == 0)
 886                                                         can_be_type = true;
 887                                                 continue;
 888                                         default:
 889                                                 can_be_type = is_type = false;
 890                                                 continue;
 891                                         }
 892
 893                                 case Token.OBJECT:
 894                                 case Token.STRING:
 895                                 case Token.BOOL:
 896                                 case Token.DECIMAL:
 897                                 case Token.FLOAT:
 898                                 case Token.DOUBLE:
 899                                 case Token.SBYTE:
 900                                 case Token.BYTE:
 901                                 case Token.SHORT:
 902                                 case Token.USHORT:
 903                                 case Token.INT:
 904                                 case Token.UINT:
 905                                 case Token.LONG:
 906                                 case Token.ULONG:
 907                                 case Token.CHAR:
 908                                 case Token.VOID:
 909                                         if (bracket_level == 0)
 910                                                 is_type = true;
 911                                         continue;
 912
 913                                 case Token.COMMA:
 914                                         if (bracket_level == 0) {
 915                                                 bracket_level = 100;
 916                                                 can_be_type = is_type = false;
 917                                         }
 918                                         continue;
 919
 920                                 case Token.OP_GENERICS_LT:
 921                                 case Token.OPEN_BRACKET:
 922                                         if (bracket_level++ == 0)
 923                                                 is_type = true;
 924                                         continue;
 925
 926                                 case Token.OP_GENERICS_GT:
 927                                 case Token.CLOSE_BRACKET:
 928                                         --bracket_level;
 929                                         continue;
 930
 931                                 case Token.INTERR_NULLABLE:
 932                                 case Token.STAR:
 933                                         if (bracket_level == 0)
 934                                                 is_type = true;
 935                                         continue;
 936
 937                                 case Token.REF:
 938                                 case Token.OUT:
 939                                         can_be_type = is_type = false;
 940                                         continue;
 941
 942                                 default:
 943                                         return Token.OPEN_PARENS;
 944                                 }
 945                         }
 946                 }
 947
 948                 public static bool IsValidIdentifier (string s)
 949                 {
 950                         if (s == null || s.Length == 0)
 951                                 return false;
 952
 953                         if (!is_identifier_start_character (s [0]))
 954                                 return false;
 955
 956                         for (int i = 1; i < s.Length; i ++)
 957                                 if (! is_identifier_part_character (s [i]))
 958                                         return false;
 959
 960                         return true;
 961                 }
 962
 963                 bool parse_less_than ()
 964                 {
 965                 start:
 966                         int the_token = token ();
 967                         if (the_token == Token.OPEN_BRACKET) {
 968                                 do {
 969                                         the_token = token ();
 970                                 } while (the_token != Token.CLOSE_BRACKET);
 971                                 the_token = token ();
 972                         } else if (the_token == Token.IN || the_token == Token.OUT) {
 973                                 the_token = token ();
 974                         }
 975                         switch (the_token) {
 976                         case Token.IDENTIFIER:
 977                         case Token.OBJECT:
 978                         case Token.STRING:
 979                         case Token.BOOL:
 980                         case Token.DECIMAL:
 981                         case Token.FLOAT:
 982                         case Token.DOUBLE:
 983                         case Token.SBYTE:
 984                         case Token.BYTE:
 985                         case Token.SHORT:
 986                         case Token.USHORT:
 987                         case Token.INT:
 988                         case Token.UINT:
 989                         case Token.LONG:
 990                         case Token.ULONG:
 991                         case Token.CHAR:
 992                         case Token.VOID:
 993                                 break;
 994                         case Token.OP_GENERICS_GT:
 995                                 return true;
 996
 997                         default:
 998                                 return false;
 999                         }
1000                 again:
1001                         the_token = token ();
1002
1003                         if (the_token == Token.OP_GENERICS_GT)
1004                                 return true;
1005                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1006                                 goto start;
1007                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1008                                 goto again;
1009                         else if (the_token == Token.OP_GENERICS_LT) {
1010                                 if (!parse_less_than ())
1011                                         return false;
1012                                 goto again;
1013                         } else if (the_token == Token.OPEN_BRACKET) {
1014                         rank_specifiers:
1015                                 the_token = token ();
1016                                 if (the_token == Token.CLOSE_BRACKET)
1017                                         goto again;
1018                                 else if (the_token == Token.COMMA)
1019                                         goto rank_specifiers;
1020                                 return false;
1021                         }
1022
1023                         return false;
1024                 }
1025
1026                 bool parse_generic_dimension (out int dimension)
1027                 {
1028                         dimension = 1;
1029
1030                 again:
1031                         int the_token = token ();
1032                         if (the_token == Token.OP_GENERICS_GT)
1033                                 return true;
1034                         else if (the_token == Token.COMMA) {
1035                                 dimension++;
1036                                 goto again;
1037                         }
1038
1039                         return false;
1040                 }
1041
1042                 public int peek_token ()
1043                 {
1044                         int the_token;
1045
1046                         PushPosition ();
1047                         the_token = token ();
1048                         PopPosition ();
1049
1050                         return the_token;
1051                 }
1052
1053                 //
1054                 // Tonizes `?' using custom disambiguous rules to return one
1055                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1056                 //
1057                 // Tricky expression look like:
1058                 //
1059                 // Foo ? a = x ? b : c;
1060                 //
1061                 int TokenizePossibleNullableType ()
1062                 {
1063                         if (parsing_block == 0 || parsing_type > 0)
1064                                 return Token.INTERR_NULLABLE;
1065
1066                         int d = peek_char ();
1067                         if (d == '?') {
1068                                 get_char ();
1069                                 return Token.OP_COALESCING;
1070                         }
1071
1072                         switch (current_token) {
1073                         case Token.CLOSE_PARENS:
1074                         case Token.TRUE:
1075                         case Token.FALSE:
1076                         case Token.NULL:
1077                         case Token.LITERAL:
1078                                 return Token.INTERR;
1079                         }
1080
1081                         if (d != ' ') {
1082                                 if (d == ',' || d == ';' || d == '>')
1083                                         return Token.INTERR_NULLABLE;
1084                                 if (d == '*' || (d >= '0' && d <= '9'))
1085                                         return Token.INTERR;
1086                         }
1087
1088                         PushPosition ();
1089                         current_token = Token.NONE;
1090                         int next_token;
1091                         switch (xtoken ()) {
1092                         case Token.LITERAL:
1093                         case Token.TRUE:
1094                         case Token.FALSE:
1095                         case Token.NULL:
1096                         case Token.THIS:
1097                         case Token.NEW:
1098                                 next_token = Token.INTERR;
1099                                 break;
1100
1101                         case Token.SEMICOLON:
1102                         case Token.COMMA:
1103                         case Token.CLOSE_PARENS:
1104                         case Token.OPEN_BRACKET:
1105                         case Token.OP_GENERICS_GT:
1106                                 next_token = Token.INTERR_NULLABLE;
1107                                 break;
1108
1109                         default:
1110                                 next_token = -1;
1111                                 break;
1112                         }
1113
1114                         if (next_token == -1) {
1115                                 switch (xtoken ()) {
1116                                 case Token.COMMA:
1117                                 case Token.SEMICOLON:
1118                                 case Token.OPEN_BRACE:
1119                                 case Token.CLOSE_PARENS:
1120                                 case Token.IN:
1121                                         next_token = Token.INTERR_NULLABLE;
1122                                         break;
1123
1124                                 case Token.COLON:
1125                                         next_token = Token.INTERR;
1126                                         break;
1127
1128                                 default:
1129                                         int ntoken;
1130                                         int interrs = 1;
1131                                         int colons = 0;
1132                                         //
1133                                         // All shorcuts failed, do it hard way
1134                                         //
1135                                         while ((ntoken = xtoken ()) != Token.EOF) {
1136                                                 if (ntoken == Token.SEMICOLON)
1137                                                         break;
1138
1139                                                 if (ntoken == Token.COLON) {
1140                                                         if (++colons == interrs)
1141                                                                 break;
1142                                                         continue;
1143                                                 }
1144
1145                                                 if (ntoken == Token.INTERR) {
1146                                                         ++interrs;
1147                                                         continue;
1148                                                 }
1149                                         }
1150
1151                                         next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1152                                         break;
1153                                 }
1154                         }
1155
1156                         PopPosition ();
1157                         return next_token;
1158                 }
1159
1160                 bool decimal_digits (int c)
1161                 {
1162                         int d;
1163                         bool seen_digits = false;
1164
1165                         if (c != -1){
1166                                 if (number_pos == max_number_size)
1167                                         Error_NumericConstantTooLong ();
1168                                 number_builder [number_pos++] = (char) c;
1169                         }
1170
1171                         //
1172                         // We use peek_char2, because decimal_digits needs to do a
1173                         // 2-character look-ahead (5.ToString for example).
1174                         //
1175                         while ((d = peek_char2 ()) != -1){
1176                                 if (d >= '0' && d <= '9'){
1177                                         if (number_pos == max_number_size)
1178                                                 Error_NumericConstantTooLong ();
1179                                         number_builder [number_pos++] = (char) d;
1180                                         get_char ();
1181                                         seen_digits = true;
1182                                 } else
1183                                         break;
1184                         }
1185
1186                         return seen_digits;
1187                 }
1188
1189                 static bool is_hex (int e)
1190                 {
1191                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1192                 }
1193
1194                 static TypeCode real_type_suffix (int c)
1195                 {
1196                         switch (c){
1197                         case 'F': case 'f':
1198                                 return TypeCode.Single;
1199                         case 'D': case 'd':
1200                                 return TypeCode.Double;
1201                         case 'M': case 'm':
1202                                 return TypeCode.Decimal;
1203                         default:
1204                                 return TypeCode.Empty;
1205                         }
1206                 }
1207
1208                 int integer_type_suffix (ulong ul, int c)
1209                 {
1210                         bool is_unsigned = false;
1211                         bool is_long = false;
1212
1213                         if (c != -1){
1214                                 bool scanning = true;
1215                                 do {
1216                                         switch (c){
1217                                         case 'U': case 'u':
1218                                                 if (is_unsigned)
1219                                                         scanning = false;
1220                                                 is_unsigned = true;
1221                                                 get_char ();
1222                                                 break;
1223
1224                                         case 'l':
1225                                                 if (!is_unsigned){
1226                                                         //
1227                                                         // if we have not seen anything in between
1228                                                         // report this error
1229                                                         //
1230                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1231                                                 }
1232
1233                                                 goto case 'L';
1234
1235                                         case 'L':
1236                                                 if (is_long)
1237                                                         scanning = false;
1238                                                 is_long = true;
1239                                                 get_char ();
1240                                                 break;
1241
1242                                         default:
1243                                                 scanning = false;
1244                                                 break;
1245                                         }
1246                                         c = peek_char ();
1247                                 } while (scanning);
1248                         }
1249
1250                         if (is_long && is_unsigned){
1251                                 val = new ULongLiteral (ul, Location);
1252                                 return Token.LITERAL;
1253                         }
1254
1255                         if (is_unsigned){
1256                                 // uint if possible, or ulong else.
1257
1258                                 if ((ul & 0xffffffff00000000) == 0)
1259                                         val = new UIntLiteral ((uint) ul, Location);
1260                                 else
1261                                         val = new ULongLiteral (ul, Location);
1262                         } else if (is_long){
1263                                 // long if possible, ulong otherwise
1264                                 if ((ul & 0x8000000000000000) != 0)
1265                                         val = new ULongLiteral (ul, Location);
1266                                 else
1267                                         val = new LongLiteral ((long) ul, Location);
1268                         } else {
1269                                 // int, uint, long or ulong in that order
1270                                 if ((ul & 0xffffffff00000000) == 0){
1271                                         uint ui = (uint) ul;
1272
1273                                         if ((ui & 0x80000000) != 0)
1274                                                 val = new UIntLiteral (ui, Location);
1275                                         else
1276                                                 val = new IntLiteral ((int) ui, Location);
1277                                 } else {
1278                                         if ((ul & 0x8000000000000000) != 0)
1279                                                 val = new ULongLiteral (ul, Location);
1280                                         else
1281                                                 val = new LongLiteral ((long) ul, Location);
1282                                 }
1283                         }
1284                         return Token.LITERAL;
1285                 }
1286
1287                 //
1288                 // given `c' as the next char in the input decide whether
1289                 // we need to convert to a special type, and then choose
1290                 // the best representation for the integer
1291                 //
1292                 int adjust_int (int c)
1293                 {
1294                         try {
1295                                 if (number_pos > 9){
1296                                         ulong ul = (uint) (number_builder [0] - '0');
1297
1298                                         for (int i = 1; i < number_pos; i++){
1299                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1300                                         }
1301                                         return integer_type_suffix (ul, c);
1302                                 } else {
1303                                         uint ui = (uint) (number_builder [0] - '0');
1304
1305                                         for (int i = 1; i < number_pos; i++){
1306                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1307                                         }
1308                                         return integer_type_suffix (ui, c);
1309                                 }
1310                         } catch (OverflowException) {
1311                                 error_details = "Integral constant is too large";
1312                                 Report.Error (1021, Location, error_details);
1313                                 val = new IntLiteral (0, Location);
1314                                 return Token.LITERAL;
1315                         }
1316                         catch (FormatException) {
1317                                 Report.Error (1013, Location, "Invalid number");
1318                                 val = new IntLiteral (0, Location);
1319                                 return Token.LITERAL;
1320                         }
1321                 }
1322
1323                 int adjust_real (TypeCode t)
1324                 {
1325                         string s = new String (number_builder, 0, number_pos);
1326                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1327
1328                         switch (t){
1329                         case TypeCode.Decimal:
1330                                 try {
1331                                         val = new DecimalLiteral (decimal.Parse (s, styles, csharp_format_info), Location);
1332                                 } catch (OverflowException) {
1333                                         val = new DecimalLiteral (0, Location);
1334                                         Report.Error (594, Location, error_details, "decimal");
1335                                 }
1336                                 break;
1337                         case TypeCode.Single:
1338                                 try {
1339                                         val = new FloatLiteral (float.Parse (s, styles, csharp_format_info), Location);
1340                                 } catch (OverflowException) {
1341                                         val = new FloatLiteral (0, Location);
1342                                         Report.Error (594, Location, error_details, "float");
1343                                 }
1344                                 break;
1345                         default:
1346                                 try {
1347                                         val = new DoubleLiteral (double.Parse (s, styles, csharp_format_info), Location);
1348                                 } catch (OverflowException) {
1349                                         val = new DoubleLiteral (0, Location);
1350                                         Report.Error (594, Location, error_details, "double");
1351                                 }
1352                                 break;
1353                         }
1354
1355                         return Token.LITERAL;
1356                 }
1357
1358                 int handle_hex ()
1359                 {
1360                         int d;
1361                         ulong ul;
1362
1363                         get_char ();
1364                         while ((d = peek_char ()) != -1){
1365                                 if (is_hex (d)){
1366                                         number_builder [number_pos++] = (char) d;
1367                                         get_char ();
1368                                 } else
1369                                         break;
1370                         }
1371
1372                         string s = new String (number_builder, 0, number_pos);
1373                         try {
1374                                 if (number_pos <= 8)
1375                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1376                                 else
1377                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1378                         } catch (OverflowException){
1379                                 error_details = "Integral constant is too large";
1380                                 Report.Error (1021, Location, error_details);
1381                                 val = new IntLiteral (0, Location);
1382                                 return Token.LITERAL;
1383                         }
1384                         catch (FormatException) {
1385                                 Report.Error (1013, Location, "Invalid number");
1386                                 val = new IntLiteral (0, Location);
1387                                 return Token.LITERAL;
1388                         }
1389
1390                         return integer_type_suffix (ul, peek_char ());
1391                 }
1392
1393                 //
1394                 // Invoked if we know we have .digits or digits
1395                 //
1396                 int is_number (int c)
1397                 {
1398                         bool is_real = false;
1399
1400                         number_pos = 0;
1401
1402                         if (c >= '0' && c <= '9'){
1403                                 if (c == '0'){
1404                                         int peek = peek_char ();
1405
1406                                         if (peek == 'x' || peek == 'X')
1407                                                 return handle_hex ();
1408                                 }
1409                                 decimal_digits (c);
1410                                 c = get_char ();
1411                         }
1412
1413                         //
1414                         // We need to handle the case of
1415                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1416                         //
1417                         if (c == '.'){
1418                                 if (decimal_digits ('.')){
1419                                         is_real = true;
1420                                         c = get_char ();
1421                                 } else {
1422                                         putback ('.');
1423                                         number_pos--;
1424                                         return adjust_int (-1);
1425                                 }
1426                         }
1427
1428                         if (c == 'e' || c == 'E'){
1429                                 is_real = true;
1430                                 if (number_pos == max_number_size)
1431                                         Error_NumericConstantTooLong ();
1432                                 number_builder [number_pos++] = 'e';
1433                                 c = get_char ();
1434
1435                                 if (c == '+'){
1436                                         if (number_pos == max_number_size)
1437                                                 Error_NumericConstantTooLong ();
1438                                         number_builder [number_pos++] = '+';
1439                                         c = -1;
1440                                 } else if (c == '-') {
1441                                         if (number_pos == max_number_size)
1442                                                 Error_NumericConstantTooLong ();
1443                                         number_builder [number_pos++] = '-';
1444                                         c = -1;
1445                                 } else {
1446                                         if (number_pos == max_number_size)
1447                                                 Error_NumericConstantTooLong ();
1448                                         number_builder [number_pos++] = '+';
1449                                 }
1450
1451                                 decimal_digits (c);
1452                                 c = get_char ();
1453                         }
1454
1455                         var type = real_type_suffix (c);
1456                         if (type == TypeCode.Empty && !is_real){
1457                                 putback (c);
1458                                 return adjust_int (c);
1459                         }
1460
1461                         is_real = true;
1462
1463                         if (type == TypeCode.Empty){
1464                                 putback (c);
1465                         }
1466
1467                         if (is_real)
1468                                 return adjust_real (type);
1469
1470                         throw new Exception ("Is Number should never reach this point");
1471                 }
1472
1473                 //
1474                 // Accepts exactly count (4 or 8) hex, no more no less
1475                 //
1476                 int getHex (int count, out int surrogate, out bool error)
1477                 {
1478                         int i;
1479                         int total = 0;
1480                         int c;
1481                         int top = count != -1 ? count : 4;
1482
1483                         get_char ();
1484                         error = false;
1485                         surrogate = 0;
1486                         for (i = 0; i < top; i++){
1487                                 c = get_char ();
1488
1489                                 if (c >= '0' && c <= '9')
1490                                         c = (int) c - (int) '0';
1491                                 else if (c >= 'A' && c <= 'F')
1492                                         c = (int) c - (int) 'A' + 10;
1493                                 else if (c >= 'a' && c <= 'f')
1494                                         c = (int) c - (int) 'a' + 10;
1495                                 else {
1496                                         error = true;
1497                                         return 0;
1498                                 }
1499
1500                                 total = (total * 16) + c;
1501                                 if (count == -1){
1502                                         int p = peek_char ();
1503                                         if (p == -1)
1504                                                 break;
1505                                         if (!is_hex ((char)p))
1506                                                 break;
1507                                 }
1508                         }
1509
1510                         if (top == 8) {
1511                                 if (total > 0x0010FFFF) {
1512                                         error = true;
1513                                         return 0;
1514                                 }
1515
1516                                 if (total >= 0x00010000) {
1517                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1518                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1519                                 }
1520                         }
1521
1522                         return total;
1523                 }
1524
1525                 int escape (int c, out int surrogate)
1526                 {
1527                         bool error;
1528                         int d;
1529                         int v;
1530
1531                         d = peek_char ();
1532                         if (c != '\\') {
1533                                 surrogate = 0;
1534                                 return c;
1535                         }
1536
1537                         switch (d){
1538                         case 'a':
1539                                 v = '\a'; break;
1540                         case 'b':
1541                                 v = '\b'; break;
1542                         case 'n':
1543                                 v = '\n'; break;
1544                         case 't':
1545                                 v = '\t'; break;
1546                         case 'v':
1547                                 v = '\v'; break;
1548                         case 'r':
1549                                 v = '\r'; break;
1550                         case '\\':
1551                                 v = '\\'; break;
1552                         case 'f':
1553                                 v = '\f'; break;
1554                         case '0':
1555                                 v = 0; break;
1556                         case '"':
1557                                 v = '"'; break;
1558                         case '\'':
1559                                 v = '\''; break;
1560                         case 'x':
1561                                 v = getHex (-1, out surrogate, out error);
1562                                 if (error)
1563                                         goto default;
1564                                 return v;
1565                         case 'u':
1566                         case 'U':
1567                                 return EscapeUnicode (d, out surrogate);
1568                         default:
1569                                 surrogate = 0;
1570                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1571                                 return d;
1572                         }
1573
1574                         get_char ();
1575                         surrogate = 0;
1576                         return v;
1577                 }
1578
1579                 int EscapeUnicode (int ch, out int surrogate)
1580                 {
1581                         bool error;
1582                         if (ch == 'U') {
1583                                 ch = getHex (8, out surrogate, out error);
1584                         } else {
1585                                 ch = getHex (4, out surrogate, out error);
1586                         }
1587
1588                         if (error)
1589                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1590
1591                         return ch;
1592                 }
1593
1594                 int get_char ()
1595                 {
1596                         int x;
1597                         if (putback_char != -1) {
1598                                 x = putback_char;
1599                                 putback_char = -1;
1600                         } else
1601                                 x = reader.Read ();
1602                         if (x == '\n') {
1603                                 advance_line ();
1604                         } else {
1605                                 col++;
1606                         }
1607                         return x;
1608                 }
1609
1610                 void advance_line ()
1611                 {
1612                         line++;
1613                         ref_line++;
1614                         previous_col = col;
1615                         col = 0;
1616                 }
1617
1618                 int peek_char ()
1619                 {
1620                         if (putback_char == -1)
1621                                 putback_char = reader.Read ();
1622                         return putback_char;
1623                 }
1624
1625                 int peek_char2 ()
1626                 {
1627                         if (putback_char != -1)
1628                                 return putback_char;
1629                         return reader.Peek ();
1630                 }
1631
1632                 void putback (int c)
1633                 {
1634                         if (putback_char != -1){
1635                                 Console.WriteLine ("Col: " + col);
1636                                 Console.WriteLine ("Row: " + line);
1637                                 Console.WriteLine ("Name: " + ref_name.Name);
1638                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1639                                 throw new Exception ("This should not happen putback on putback");
1640                         }
1641                         if (c == '\n' || col == 0) {
1642                                 // It won't happen though.
1643                                 line--;
1644                                 ref_line--;
1645                                 col = previous_col;
1646                         }
1647                         else
1648                                 col--;
1649                         putback_char = c;
1650                 }
1651
1652                 public bool advance ()
1653                 {
1654                         return peek_char () != -1 || CompleteOnEOF;
1655                 }
1656
1657                 public Object Value {
1658                         get {
1659                                 return val;
1660                         }
1661                 }
1662
1663                 public Object value ()
1664                 {
1665                         return val;
1666                 }
1667
1668                 public int token ()
1669                 {
1670                         current_token = xtoken ();
1671                         return current_token;
1672                 }
1673
1674                 void get_cmd_arg (out string cmd, out string arg)
1675                 {
1676                         int c;
1677
1678                         tokens_seen = false;
1679                         arg = "";
1680
1681                         // skip over white space
1682                         do {
1683                                 c = get_char ();
1684                         } while (c == '\r' || c == ' ' || c == '\t');
1685
1686                         static_cmd_arg.Length = 0;
1687                         while (c != -1 && is_identifier_part_character ((char)c)) {
1688                                 static_cmd_arg.Append ((char)c);
1689                                 c = get_char ();
1690                                 if (c == '\\') {
1691                                         int peek = peek_char ();
1692                                         if (peek == 'U' || peek == 'u') {
1693                                                 int surrogate;
1694                                                 c = EscapeUnicode (c, out surrogate);
1695                                                 if (surrogate != 0) {
1696                                                         if (is_identifier_part_character ((char) c))
1697                                                                 static_cmd_arg.Append ((char) c);
1698                                                         c = surrogate;
1699                                                 }
1700                                         }
1701                                 }
1702                         }
1703
1704                         cmd = static_cmd_arg.ToString ();
1705
1706                         // skip over white space
1707                         while (c == '\r' || c == ' ' || c == '\t')
1708                                 c = get_char ();
1709
1710                         static_cmd_arg.Length = 0;
1711                         int has_identifier_argument = 0;
1712
1713                         while (c != -1 && c != '\n' && c != '\r') {
1714                                 if (c == '\\' && has_identifier_argument >= 0) {
1715                                         if (has_identifier_argument != 0 || (cmd == "define" || cmd == "if" || cmd == "elif" || cmd == "undef")) {
1716                                                 has_identifier_argument = 1;
1717
1718                                                 int peek = peek_char ();
1719                                                 if (peek == 'U' || peek == 'u') {
1720                                                         int surrogate;
1721                                                         c = EscapeUnicode (c, out surrogate);
1722                                                         if (surrogate != 0) {
1723                                                                 if (is_identifier_part_character ((char) c))
1724                                                                         static_cmd_arg.Append ((char) c);
1725                                                                 c = surrogate;
1726                                                         }
1727                                                 }
1728                                         } else {
1729                                                 has_identifier_argument = -1;
1730                                         }
1731                                 }
1732                                 static_cmd_arg.Append ((char) c);
1733                                 c = get_char ();
1734                         }
1735
1736                         if (static_cmd_arg.Length != 0)
1737                                 arg = static_cmd_arg.ToString ();
1738                 }
1739
1740                 //
1741                 // Handles the #line directive
1742                 //
1743                 bool PreProcessLine (string arg)
1744                 {
1745                         if (arg.Length == 0)
1746                                 return false;
1747
1748                         if (arg == "default"){
1749                                 ref_line = line;
1750                                 ref_name = file_name;
1751                                 hidden = false;
1752                                 Location.Push (file_name, ref_name);
1753                                 return true;
1754                         } else if (arg == "hidden"){
1755                                 hidden = true;
1756                                 return true;
1757                         }
1758
1759                         try {
1760                                 int pos;
1761
1762                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1763                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1764                                         pos++;
1765
1766                                         char [] quotes = { '\"' };
1767
1768                                         string name = arg.Substring (pos). Trim (quotes);
1769                                         ref_name = Location.LookupFile (file_name, name);
1770                                         file_name.AddFile (ref_name);
1771                                         hidden = false;
1772                                         Location.Push (file_name, ref_name);
1773                                 } else {
1774                                         ref_line = System.Int32.Parse (arg);
1775                                         hidden = false;
1776                                 }
1777                         } catch {
1778                                 return false;
1779                         }
1780
1781                         return true;
1782                 }
1783
1784                 //
1785                 // Handles #define and #undef
1786                 //
1787                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
1788                 {
1789                         if (ident.Length == 0 || ident == "true" || ident == "false"){
1790                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
1791                                 return;
1792                         }
1793
1794                         if (ident.IndexOfAny (simple_whitespaces) != -1){
1795                                 Error_EndLineExpected ();
1796                                 return;
1797                         }
1798
1799                         if (!is_identifier_start_character (ident [0]))
1800                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
1801
1802                         foreach (char c in ident.Substring (1)){
1803                                 if (!is_identifier_part_character (c)){
1804                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
1805                                         return;
1806                                 }
1807                         }
1808
1809                         if (!caller_is_taking)
1810                                 return;
1811
1812                         if (is_define) {
1813                                 //
1814                                 // #define ident
1815                                 //
1816                                 if (RootContext.IsConditionalDefined (ident))
1817                                         return;
1818
1819                                 file_name.AddDefine (ident);
1820                         } else {
1821                                 //
1822                                 // #undef ident
1823                                 //
1824                                 file_name.AddUndefine (ident);
1825                         }
1826                 }
1827
1828                 static byte read_hex (string arg, int pos, out bool error)
1829                 {
1830                         error = false;
1831
1832                         int total;
1833                         char c = arg [pos];
1834
1835                         if ((c >= '0') && (c <= '9'))
1836                                 total = (int) c - (int) '0';
1837                         else if ((c >= 'A') && (c <= 'F'))
1838                                 total = (int) c - (int) 'A' + 10;
1839                         else if ((c >= 'a') && (c <= 'f'))
1840                                 total = (int) c - (int) 'a' + 10;
1841                         else {
1842                                 error = true;
1843                                 return 0;
1844                         }
1845
1846                         total *= 16;
1847                         c = arg [pos+1];
1848
1849                         if ((c >= '0') && (c <= '9'))
1850                                 total += (int) c - (int) '0';
1851                         else if ((c >= 'A') && (c <= 'F'))
1852                                 total += (int) c - (int) 'A' + 10;
1853                         else if ((c >= 'a') && (c <= 'f'))
1854                                 total += (int) c - (int) 'a' + 10;
1855                         else {
1856                                 error = true;
1857                                 return 0;
1858                         }
1859
1860                         return (byte) total;
1861                 }
1862
1863                 /// <summary>
1864                 /// Handles #pragma checksum
1865                 /// </summary>
1866                 bool PreProcessPragmaChecksum (string arg)
1867                 {
1868                         if ((arg [0] != ' ') && (arg [0] != '\t'))
1869                                 return false;
1870
1871                         arg = arg.Trim (simple_whitespaces);
1872                         if ((arg.Length < 2) || (arg [0] != '"'))
1873                                 return false;
1874
1875                         StringBuilder file_sb = new StringBuilder ();
1876
1877                         int pos = 1;
1878                         char ch;
1879                         while ((ch = arg [pos++]) != '"') {
1880                                 if (pos >= arg.Length)
1881                                         return false;
1882
1883                                 if (ch == '\\') {
1884                                         if (pos+1 >= arg.Length)
1885                                                 return false;
1886                                         ch = arg [pos++];
1887                                 }
1888
1889                                 file_sb.Append (ch);
1890                         }
1891
1892                         if ((pos+2 >= arg.Length) || ((arg [pos] != ' ') && (arg [pos] != '\t')))
1893                                 return false;
1894
1895                         arg = arg.Substring (pos).Trim (simple_whitespaces);
1896                         if ((arg.Length < 42) || (arg [0] != '"') || (arg [1] != '{') ||
1897                             (arg [10] != '-') || (arg [15] != '-') || (arg [20] != '-') ||
1898                             (arg [25] != '-') || (arg [38] != '}') || (arg [39] != '"'))
1899                                 return false;
1900
1901                         bool error;
1902                         byte[] guid_bytes = new byte [16];
1903
1904                         for (int i = 0; i < 4; i++) {
1905                                 guid_bytes [i] = read_hex (arg, 2+2*i, out error);
1906                                 if (error)
1907                                         return false;
1908                         }
1909                         for (int i = 0; i < 2; i++) {
1910                                 guid_bytes [i+4] = read_hex (arg, 11+2*i, out error);
1911                                 if (error)
1912                                         return false;
1913                                 guid_bytes [i+6] = read_hex (arg, 16+2*i, out error);
1914                                 if (error)
1915                                         return false;
1916                                 guid_bytes [i+8] = read_hex (arg, 21+2*i, out error);
1917                                 if (error)
1918                                         return false;
1919                         }
1920
1921                         for (int i = 0; i < 6; i++) {
1922                                 guid_bytes [i+10] = read_hex (arg, 26+2*i, out error);
1923                                 if (error)
1924                                         return false;
1925                         }
1926
1927                         arg = arg.Substring (40).Trim (simple_whitespaces);
1928                         if ((arg.Length < 34) || (arg [0] != '"') || (arg [33] != '"'))
1929                                 return false;
1930
1931                         byte[] checksum_bytes = new byte [16];
1932                         for (int i = 0; i < 16; i++) {
1933                                 checksum_bytes [i] = read_hex (arg, 1+2*i, out error);
1934                                 if (error)
1935                                         return false;
1936                         }
1937
1938                         arg = arg.Substring (34).Trim (simple_whitespaces);
1939                         if (arg.Length > 0)
1940                                 return false;
1941
1942                         SourceFile file = Location.LookupFile (file_name, file_sb.ToString ());
1943                         file.SetChecksum (guid_bytes, checksum_bytes);
1944                         ref_name.AutoGenerated = true;
1945                         return true;
1946                 }
1947
1948                 /// <summary>
1949                 /// Handles #pragma directive
1950                 /// </summary>
1951                 void PreProcessPragma (string arg)
1952                 {
1953                         const string warning = "warning";
1954                         const string w_disable = "warning disable";
1955                         const string w_restore = "warning restore";
1956                         const string checksum = "checksum";
1957
1958                         if (arg == w_disable) {
1959                                 Report.RegisterWarningRegion (Location).WarningDisable (Location.Row);
1960                                 return;
1961                         }
1962
1963                         if (arg == w_restore) {
1964                                 Report.RegisterWarningRegion (Location).WarningEnable (Location.Row);
1965                                 return;
1966                         }
1967
1968                         if (arg.StartsWith (w_disable, StringComparison.Ordinal)) {
1969                                 int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
1970                                 var loc = Location;
1971                                 foreach (int code in codes) {
1972                                         if (code != 0)
1973                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, Report);
1974                                 }
1975                                 return;
1976                         }
1977
1978                         if (arg.StartsWith (w_restore, StringComparison.Ordinal)) {
1979                                 int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
1980                                 var loc = Location;
1981                                 foreach (int code in codes) {
1982                                         if (code != 0)
1983                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, Report);
1984                                 }
1985                                 return;
1986                         }
1987
1988                         if (arg.StartsWith (warning, StringComparison.Ordinal)) {
1989                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
1990                                 return;
1991                         }
1992
1993                         if (arg.StartsWith (checksum, StringComparison.Ordinal)) {
1994                                 if (!PreProcessPragmaChecksum (arg.Substring (checksum.Length)))
1995                                         Warning_InvalidPragmaChecksum ();
1996                                 return;
1997                         }
1998
1999                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2000                 }
2001
2002                 int[] ParseNumbers (string text)
2003                 {
2004                         string[] string_array = text.Split (pragma_value_separator);
2005                         int[] values = new int [string_array.Length];
2006                         int index = 0;
2007                         foreach (string string_code in string_array) {
2008                                 try {
2009                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
2010                                 }
2011                                 catch (FormatException) {
2012                                         Report.Warning (1692, 1, Location, "Invalid number");
2013                                 }
2014                         }
2015                         return values;
2016                 }
2017
2018                 bool eval_val (string s)
2019                 {
2020                         if (s == "true")
2021                                 return true;
2022                         if (s == "false")
2023                                 return false;
2024
2025                         return file_name.IsConditionalDefined (s);
2026                 }
2027
2028                 bool pp_primary (ref string s)
2029                 {
2030                         s = s.Trim ();
2031                         int len = s.Length;
2032
2033                         if (len > 0){
2034                                 char c = s [0];
2035
2036                                 if (c == '('){
2037                                         s = s.Substring (1);
2038                                         bool val = pp_expr (ref s, false);
2039                                         if (s.Length > 0 && s [0] == ')'){
2040                                                 s = s.Substring (1);
2041                                                 return val;
2042                                         }
2043                                         Error_InvalidDirective ();
2044                                         return false;
2045                                 }
2046
2047                                 if (is_identifier_start_character (c)){
2048                                         int j = 1;
2049
2050                                         while (j < len){
2051                                                 c = s [j];
2052
2053                                                 if (is_identifier_part_character (c)){
2054                                                         j++;
2055                                                         continue;
2056                                                 }
2057                                                 bool v = eval_val (s.Substring (0, j));
2058                                                 s = s.Substring (j);
2059                                                 return v;
2060                                         }
2061                                         bool vv = eval_val (s);
2062                                         s = "";
2063                                         return vv;
2064                                 }
2065                         }
2066                         Error_InvalidDirective ();
2067                         return false;
2068                 }
2069
2070                 bool pp_unary (ref string s)
2071                 {
2072                         s = s.Trim ();
2073                         int len = s.Length;
2074
2075                         if (len > 0){
2076                                 if (s [0] == '!'){
2077                                         if (len > 1 && s [1] == '='){
2078                                                 Error_InvalidDirective ();
2079                                                 return false;
2080                                         }
2081                                         s = s.Substring (1);
2082                                         return ! pp_primary (ref s);
2083                                 } else
2084                                         return pp_primary (ref s);
2085                         } else {
2086                                 Error_InvalidDirective ();
2087                                 return false;
2088                         }
2089                 }
2090
2091                 bool pp_eq (ref string s)
2092                 {
2093                         bool va = pp_unary (ref s);
2094
2095                         s = s.Trim ();
2096                         int len = s.Length;
2097                         if (len > 0){
2098                                 if (s [0] == '='){
2099                                         if (len > 2 && s [1] == '='){
2100                                                 s = s.Substring (2);
2101                                                 return va == pp_unary (ref s);
2102                                         } else {
2103                                                 Error_InvalidDirective ();
2104                                                 return false;
2105                                         }
2106                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2107                                         s = s.Substring (2);
2108
2109                                         return va != pp_unary (ref s);
2110
2111                                 }
2112                         }
2113
2114                         return va;
2115
2116                 }
2117
2118                 bool pp_and (ref string s)
2119                 {
2120                         bool va = pp_eq (ref s);
2121
2122                         s = s.Trim ();
2123                         int len = s.Length;
2124                         if (len > 0){
2125                                 if (s [0] == '&'){
2126                                         if (len > 2 && s [1] == '&'){
2127                                                 s = s.Substring (2);
2128                                                 return (va & pp_and (ref s));
2129                                         } else {
2130                                                 Error_InvalidDirective ();
2131                                                 return false;
2132                                         }
2133                                 }
2134                         }
2135                         return va;
2136                 }
2137
2138                 //
2139                 // Evaluates an expression for `#if' or `#elif'
2140                 //
2141                 bool pp_expr (ref string s, bool isTerm)
2142                 {
2143                         bool va = pp_and (ref s);
2144                         s = s.Trim ();
2145                         int len = s.Length;
2146                         if (len > 0){
2147                                 char c = s [0];
2148
2149                                 if (c == '|'){
2150                                         if (len > 2 && s [1] == '|'){
2151                                                 s = s.Substring (2);
2152                                                 return va | pp_expr (ref s, isTerm);
2153                                         } else {
2154                                                 Error_InvalidDirective ();
2155                                                 return false;
2156                                         }
2157                                 }
2158                                 if (isTerm) {
2159                                         Error_EndLineExpected ();
2160                                         return false;
2161                                 }
2162                         }
2163
2164                         return va;
2165                 }
2166
2167                 bool eval (string s)
2168                 {
2169                         bool v = pp_expr (ref s, true);
2170                         s = s.Trim ();
2171                         if (s.Length != 0){
2172                                 return false;
2173                         }
2174
2175                         return v;
2176                 }
2177
2178                 void Error_NumericConstantTooLong ()
2179                 {
2180                         Report.Error (1021, Location, "Numeric constant too long");
2181                 }
2182
2183                 void Error_InvalidDirective ()
2184                 {
2185                         Report.Error (1517, Location, "Invalid preprocessor directive");
2186                 }
2187
2188                 void Error_UnexpectedDirective (string extra)
2189                 {
2190                         Report.Error (
2191                                 1028, Location,
2192                                 "Unexpected processor directive ({0})", extra);
2193                 }
2194
2195                 void Error_TokensSeen ()
2196                 {
2197                         Report.Error (1032, Location,
2198                                 "Cannot define or undefine preprocessor symbols after first token in file");
2199                 }
2200
2201                 void Eror_WrongPreprocessorLocation ()
2202                 {
2203                         Report.Error (1040, Location,
2204                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2205                 }
2206
2207                 void Error_EndLineExpected ()
2208                 {
2209                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2210                 }
2211
2212                 void Warning_InvalidPragmaChecksum ()
2213                 {
2214                         Report.Warning (1695, 1, Location,
2215                                         "Invalid #pragma checksum syntax; should be " +
2216                                         "#pragma checksum \"filename\" " +
2217                                         "\"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2218                 }
2219                 //
2220                 // if true, then the code continues processing the code
2221                 // if false, the code stays in a loop until another directive is
2222                 // reached.
2223                 // When caller_is_taking is false we ignore all directives except the ones
2224                 // which can help us to identify where the #if block ends
2225                 bool handle_preprocessing_directive (bool caller_is_taking)
2226                 {
2227                         string cmd, arg;
2228                         bool region_directive = false;
2229
2230                         get_cmd_arg (out cmd, out arg);
2231
2232                         // Eat any trailing whitespaces and single-line comments
2233                         if (arg.IndexOf ("//") != -1)
2234                                 arg = arg.Substring (0, arg.IndexOf ("//"));
2235                         arg = arg.Trim (simple_whitespaces);
2236
2237                         //
2238                         // The first group of pre-processing instructions is always processed
2239                         //
2240                         switch (cmd){
2241                         case "region":
2242                                 region_directive = true;
2243                                 arg = "true";
2244                                 goto case "if";
2245
2246                         case "endregion":
2247                                 if (ifstack == null || ifstack.Count == 0){
2248                                         Error_UnexpectedDirective ("no #region for this #endregion");
2249                                         return true;
2250                                 }
2251                                 int pop = ifstack.Pop ();
2252
2253                                 if ((pop & REGION) == 0)
2254                                         Report.Error (1027, Location, "Expected `#endif' directive");
2255
2256                                 return caller_is_taking;
2257
2258                         case "if":
2259                                 if (ifstack == null)
2260                                         ifstack = new Stack<int> (2);
2261
2262                                 int flags = region_directive ? REGION : 0;
2263                                 if (ifstack.Count == 0){
2264                                         flags |= PARENT_TAKING;
2265                                 } else {
2266                                         int state = ifstack.Peek ();
2267                                         if ((state & TAKING) != 0) {
2268                                                 flags |= PARENT_TAKING;
2269                                         }
2270                                 }
2271
2272                                 if (caller_is_taking && eval (arg)) {
2273                                         ifstack.Push (flags | TAKING);
2274                                         return true;
2275                                 }
2276                                 ifstack.Push (flags);
2277                                 return false;
2278
2279                         case "endif":
2280                                 if (ifstack == null || ifstack.Count == 0){
2281                                         Error_UnexpectedDirective ("no #if for this #endif");
2282                                         return true;
2283                                 } else {
2284                                         pop = ifstack.Pop ();
2285
2286                                         if ((pop & REGION) != 0)
2287                                                 Report.Error (1038, Location, "#endregion directive expected");
2288
2289                                         if (arg.Length != 0) {
2290                                                 Error_EndLineExpected ();
2291                                         }
2292
2293                                         if (ifstack.Count == 0)
2294                                                 return true;
2295
2296                                         int state = ifstack.Peek ();
2297                                         return (state & TAKING) != 0;
2298                                 }
2299
2300                         case "elif":
2301                                 if (ifstack == null || ifstack.Count == 0){
2302                                         Error_UnexpectedDirective ("no #if for this #elif");
2303                                         return true;
2304                                 } else {
2305                                         int state = ifstack.Pop ();
2306
2307                                         if ((state & REGION) != 0) {
2308                                                 Report.Error (1038, Location, "#endregion directive expected");
2309                                                 return true;
2310                                         }
2311
2312                                         if ((state & ELSE_SEEN) != 0){
2313                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2314                                                 return true;
2315                                         }
2316
2317                                         if ((state & TAKING) != 0) {
2318                                                 ifstack.Push (0);
2319                                                 return false;
2320                                         }
2321
2322                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2323                                                 ifstack.Push (state | TAKING);
2324                                                 return true;
2325                                         }
2326
2327                                         ifstack.Push (state);
2328                                         return false;
2329                                 }
2330
2331                         case "else":
2332                                 if (ifstack == null || ifstack.Count == 0){
2333                                         Error_UnexpectedDirective ("no #if for this #else");
2334                                         return true;
2335                                 } else {
2336                                         int state = ifstack.Peek ();
2337
2338                                         if ((state & REGION) != 0) {
2339                                                 Report.Error (1038, Location, "#endregion directive expected");
2340                                                 return true;
2341                                         }
2342
2343                                         if ((state & ELSE_SEEN) != 0){
2344                                                 Error_UnexpectedDirective ("#else within #else");
2345                                                 return true;
2346                                         }
2347
2348                                         ifstack.Pop ();
2349
2350                                         if (arg.Length != 0) {
2351                                                 Error_EndLineExpected ();
2352                                                 return true;
2353                                         }
2354
2355                                         bool ret = false;
2356                                         if ((state & PARENT_TAKING) != 0) {
2357                                                 ret = (state & TAKING) == 0;
2358
2359                                                 if (ret)
2360                                                         state |= TAKING;
2361                                                 else
2362                                                         state &= ~TAKING;
2363                                         }
2364
2365                                         ifstack.Push (state | ELSE_SEEN);
2366
2367                                         return ret;
2368                                 }
2369                         case "define":
2370                                 if (any_token_seen){
2371                                         Error_TokensSeen ();
2372                                         return caller_is_taking;
2373                                 }
2374                                 PreProcessDefinition (true, arg, caller_is_taking);
2375                                 return caller_is_taking;
2376
2377                         case "undef":
2378                                 if (any_token_seen){
2379                                         Error_TokensSeen ();
2380                                         return caller_is_taking;
2381                                 }
2382                                 PreProcessDefinition (false, arg, caller_is_taking);
2383                                 return caller_is_taking;
2384                         }
2385
2386                         //
2387                         // These are only processed if we are in a `taking' block
2388                         //
2389                         if (!caller_is_taking)
2390                                 return false;
2391
2392                         switch (cmd){
2393                         case "error":
2394                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2395                                 return true;
2396
2397                         case "warning":
2398                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2399                                 return true;
2400
2401                         case "pragma":
2402                                 if (RootContext.Version == LanguageVersion.ISO_1) {
2403                                         Report.FeatureIsNotAvailable (Location, "#pragma");
2404                                         return true;
2405                                 }
2406
2407                                 PreProcessPragma (arg);
2408                                 return true;
2409
2410                         case "line":
2411                                 if (!PreProcessLine (arg))
2412                                         Report.Error (
2413                                                 1576, Location,
2414                                                 "The line number specified for #line directive is missing or invalid");
2415                                 return caller_is_taking;
2416                         }
2417
2418                         Report.Error (1024, Location, "Wrong preprocessor directive");
2419                         return true;
2420
2421                 }
2422
2423                 private int consume_string (bool quoted)
2424                 {
2425                         int c;
2426                         string_builder.Length = 0;
2427
2428                         while ((c = get_char ()) != -1){
2429                                 if (c == '"'){
2430                                         if (quoted && peek_char () == '"'){
2431                                                 string_builder.Append ((char) c);
2432                                                 get_char ();
2433                                                 continue;
2434                                         } else {
2435                                                 val = new StringLiteral (string_builder.ToString (), Location);
2436                                                 return Token.LITERAL;
2437                                         }
2438                                 }
2439
2440                                 if (c == '\n'){
2441                                         if (!quoted)
2442                                                 Report.Error (1010, Location, "Newline in constant");
2443                                 }
2444
2445                                 if (!quoted){
2446                                         int surrogate;
2447                                         c = escape (c, out surrogate);
2448                                         if (c == -1)
2449                                                 return Token.ERROR;
2450                                         if (surrogate != 0) {
2451                                                 string_builder.Append ((char) c);
2452                                                 c = surrogate;
2453                                         }
2454                                 }
2455                                 string_builder.Append ((char) c);
2456                         }
2457
2458                         Report.Error (1039, Location, "Unterminated string literal");
2459                         return Token.EOF;
2460                 }
2461
2462                 private int consume_identifier (int s)
2463                 {
2464                         int res = consume_identifier (s, false);
2465
2466                         if (doc_state == XmlCommentState.Allowed)
2467                                 doc_state = XmlCommentState.NotAllowed;
2468
2469                         return res;
2470                 }
2471
2472                 int consume_identifier (int c, bool quoted)
2473                 {
2474                         //
2475                         // This method is very performance sensitive. It accounts
2476                         // for approximately 25% of all parser time
2477                         //
2478
2479                         int pos = 0;
2480                         int column = col;
2481
2482                         if (c == '\\') {
2483                                 int surrogate;
2484                                 c = escape (c, out surrogate);
2485                                 if (surrogate != 0) {
2486                                         id_builder [pos++] = (char) c;
2487                                         c = surrogate;
2488                                 }
2489                         }
2490
2491                         id_builder [pos++] = (char) c;
2492
2493                         try {
2494                                 while (true) {
2495                                         c = reader.Read ();
2496
2497                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
2498                                                 id_builder [pos++] = (char) c;
2499                                                 continue;
2500                                         }
2501
2502                                         if (c < 0x80) {
2503                                                 if (c == '\\') {
2504                                                         int surrogate;
2505                                                         c = escape (c, out surrogate);
2506                                                         if (surrogate != 0) {
2507                                                                 if (is_identifier_part_character ((char) c))
2508                                                                         id_builder[pos++] = (char) c;
2509                                                                 c = surrogate;
2510                                                         }
2511
2512                                                         continue;
2513                                                 }
2514                                         } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
2515                                                 id_builder [pos++] = (char) c;
2516                                                 continue;
2517                                         }
2518
2519                                         putback_char = c;
2520                                         break;
2521                                 }
2522                         } catch (IndexOutOfRangeException) {
2523                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2524                                 col += pos - 1;
2525                                 return Token.ERROR;
2526                         }
2527
2528                         col += pos - 1;
2529
2530                         //
2531                         // Optimization: avoids doing the keyword lookup
2532                         // on uppercase letters
2533                         //
2534                         if (id_builder [0] >= '_' && !quoted) {
2535                                 int keyword = GetKeyword (id_builder, pos);
2536                                 if (keyword != -1) {
2537                                         val = LocatedToken.Create (null, ref_line, column);
2538                                         return keyword;
2539                                 }
2540                         }
2541
2542                         //
2543                         // Keep identifiers in an array of hashtables to avoid needless
2544                         // allocations
2545                         //
2546                         var identifiers_group = identifiers [pos];
2547                         string s;
2548                         if (identifiers_group != null) {
2549                                 if (identifiers_group.TryGetValue (id_builder, out s)) {
2550                                         val = LocatedToken.Create (s, ref_line, column);
2551                                         if (quoted)
2552                                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2553                                         return Token.IDENTIFIER;
2554                                 }
2555                         } else {
2556                                 // TODO: this should be number of files dependant
2557                                 // corlib compilation peaks at 1000 and System.Core at 150
2558                                 int capacity = pos > 20 ? 10 : 100;
2559                                 identifiers_group = new Dictionary<char[],string> (capacity, new IdentifiersComparer (pos));
2560                                 identifiers [pos] = identifiers_group;
2561                         }
2562
2563                         char [] chars = new char [pos];
2564                         Array.Copy (id_builder, chars, pos);
2565
2566                         s = new string (id_builder, 0, pos);
2567                         identifiers_group.Add (chars, s);
2568
2569                         val = LocatedToken.Create (s, ref_line, column);
2570                         if (quoted)
2571                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2572
2573                         return Token.IDENTIFIER;
2574                 }
2575
2576                 public int xtoken ()
2577                 {
2578                         int d, c;
2579
2580                         // Whether we have seen comments on the current line
2581                         bool comments_seen = false;
2582                         while ((c = get_char ()) != -1) {
2583                                 switch (c) {
2584                                 case '\t':
2585                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
2586                                         continue;
2587
2588                                 case ' ':
2589                                 case '\f':
2590                                 case '\v':
2591                                 case 0xa0:
2592                                 case 0:
2593                                 case 0xFEFF:    // Ignore BOM anywhere in the file
2594                                         continue;
2595
2596 /*                              This is required for compatibility with .NET
2597                                 case 0xEF:
2598                                         if (peek_char () == 0xBB) {
2599                                                 PushPosition ();
2600                                                 get_char ();
2601                                                 if (get_char () == 0xBF)
2602                                                         continue;
2603                                                 PopPosition ();
2604                                         }
2605                                         break;
2606 */
2607                                 case '\r':
2608                                         if (peek_char () != '\n')
2609                                                 advance_line ();
2610                                         else
2611                                                 get_char ();
2612
2613                                         any_token_seen |= tokens_seen;
2614                                         tokens_seen = false;
2615                                         comments_seen = false;
2616                                         continue;
2617
2618                                 case '\\':
2619                                         tokens_seen = true;
2620                                         return consume_identifier (c);
2621
2622                                 case '{':
2623                                         val = LocatedToken.Create (ref_line, col);
2624                                         return Token.OPEN_BRACE;
2625                                 case '}':
2626                                         val = LocatedToken.Create (ref_line, col);
2627                                         return Token.CLOSE_BRACE;
2628                                 case '[':
2629                                         // To block doccomment inside attribute declaration.
2630                                         if (doc_state == XmlCommentState.Allowed)
2631                                                 doc_state = XmlCommentState.NotAllowed;
2632
2633                                         val = LocatedToken.Create (ref_line, col);
2634
2635                                         if (parsing_block == 0 || lambda_arguments_parsing)
2636                                                 return Token.OPEN_BRACKET;
2637
2638                                         int next = peek_char ();
2639                                         switch (next) {
2640                                         case ']':
2641                                         case ',':
2642                                                 return Token.OPEN_BRACKET;
2643
2644                                         case ' ':
2645                                         case '\f':
2646                                         case '\v':
2647                                         case '\r':
2648                                         case '\n':
2649                                         case '/':
2650                                                 next = peek_token ();
2651                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
2652                                                         return Token.OPEN_BRACKET;
2653
2654                                                 return Token.OPEN_BRACKET_EXPR;
2655                                         default:
2656                                                 return Token.OPEN_BRACKET_EXPR;
2657                                         }
2658                                 case ']':
2659                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2660                                         return Token.CLOSE_BRACKET;
2661                                 case '(':
2662                                         val = LocatedToken.Create (ref_line, col);
2663                                         //
2664                                         // An expression versions of parens can appear in block context only
2665                                         //
2666                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
2667
2668                                                 //
2669                                                 // Optmize most common case where we know that parens
2670                                                 // is not special
2671                                                 //
2672                                                 switch (current_token) {
2673                                                 case Token.IDENTIFIER:
2674                                                 case Token.IF:
2675                                                 case Token.FOR:
2676                                                 case Token.FOREACH:
2677                                                 case Token.TYPEOF:
2678                                                 case Token.WHILE:
2679                                                 case Token.USING:
2680                                                 case Token.DEFAULT:
2681                                                 case Token.DELEGATE:
2682                                                 case Token.OP_GENERICS_GT:
2683                                                         return Token.OPEN_PARENS;
2684                                                 }
2685
2686                                                 // Optimize using peek
2687                                                 int xx = peek_char ();
2688                                                 switch (xx) {
2689                                                 case '(':
2690                                                 case '\'':
2691                                                 case '"':
2692                                                 case '0':
2693                                                 case '1':
2694                                                         return Token.OPEN_PARENS;
2695                                                 }
2696
2697                                                 lambda_arguments_parsing = true;
2698                                                 PushPosition ();
2699                                                 d = TokenizeOpenParens ();
2700                                                 PopPosition ();
2701                                                 lambda_arguments_parsing = false;
2702                                                 return d;
2703                                         }
2704
2705                                         return Token.OPEN_PARENS;
2706                                 case ')':
2707                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2708                                         return Token.CLOSE_PARENS;
2709                                 case ',':
2710                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2711                                         return Token.COMMA;
2712                                 case ';':
2713                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2714                                         return Token.SEMICOLON;
2715                                 case '~':
2716                                         val = LocatedToken.Create (ref_line, col);
2717                                         return Token.TILDE;
2718                                 case '?':
2719                                         val = LocatedToken.Create (ref_line, col);
2720                                         return TokenizePossibleNullableType ();
2721                                 case '<':
2722                                         val = LocatedToken.Create (ref_line, col);
2723                                         if (parsing_generic_less_than++ > 0)
2724                                                 return Token.OP_GENERICS_LT;
2725
2726                                         return TokenizeLessThan ();
2727
2728                                 case '>':
2729                                         val = LocatedToken.Create (ref_line, col);
2730                                         d = peek_char ();
2731
2732                                         if (d == '='){
2733                                                 get_char ();
2734                                                 return Token.OP_GE;
2735                                         }
2736
2737                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
2738                                                 parsing_generic_less_than--;
2739                                                 return Token.OP_GENERICS_GT;
2740                                         }
2741
2742                                         if (d == '>') {
2743                                                 get_char ();
2744                                                 d = peek_char ();
2745
2746                                                 if (d == '=') {
2747                                                         get_char ();
2748                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
2749                                                 }
2750                                                 return Token.OP_SHIFT_RIGHT;
2751                                         }
2752
2753                                         return Token.OP_GT;
2754
2755                                 case '+':
2756                                         val = LocatedToken.Create (ref_line, col);
2757                                         d = peek_char ();
2758                                         if (d == '+') {
2759                                                 d = Token.OP_INC;
2760                                         } else if (d == '=') {
2761                                                 d = Token.OP_ADD_ASSIGN;
2762                                         } else {
2763                                                 return Token.PLUS;
2764                                         }
2765                                         get_char ();
2766                                         return d;
2767
2768                                 case '-':
2769                                         val = LocatedToken.Create (ref_line, col);
2770                                         d = peek_char ();
2771                                         if (d == '-') {
2772                                                 d = Token.OP_DEC;
2773                                         } else if (d == '=')
2774                                                 d = Token.OP_SUB_ASSIGN;
2775                                         else if (d == '>')
2776                                                 d = Token.OP_PTR;
2777                                         else {
2778                                                 return Token.MINUS;
2779                                         }
2780                                         get_char ();
2781                                         return d;
2782
2783                                 case '!':
2784                                         val = LocatedToken.Create (ref_line, col);
2785                                         if (peek_char () == '='){
2786                                                 get_char ();
2787                                                 return Token.OP_NE;
2788                                         }
2789                                         return Token.BANG;
2790
2791                                 case '=':
2792                                         val = LocatedToken.Create (ref_line, col);
2793                                         d = peek_char ();
2794                                         if (d == '='){
2795                                                 get_char ();
2796                                                 return Token.OP_EQ;
2797                                         }
2798                                         if (d == '>'){
2799                                                 get_char ();
2800                                                 return Token.ARROW;
2801                                         }
2802
2803                                         return Token.ASSIGN;
2804
2805                                 case '&':
2806                                         val = LocatedToken.Create (ref_line, col);
2807                                         d = peek_char ();
2808                                         if (d == '&'){
2809                                                 get_char ();
2810                                                 return Token.OP_AND;
2811                                         }
2812                                         if (d == '='){
2813                                                 get_char ();
2814                                                 return Token.OP_AND_ASSIGN;
2815                                         }
2816                                         return Token.BITWISE_AND;
2817
2818                                 case '|':
2819                                         val = LocatedToken.Create (ref_line, col);
2820                                         d = peek_char ();
2821                                         if (d == '|'){
2822                                                 get_char ();
2823                                                 return Token.OP_OR;
2824                                         }
2825                                         if (d == '='){
2826                                                 get_char ();
2827                                                 return Token.OP_OR_ASSIGN;
2828                                         }
2829                                         return Token.BITWISE_OR;
2830
2831                                 case '*':
2832                                         val = LocatedToken.Create (ref_line, col);
2833                                         if (peek_char () == '='){
2834                                                 get_char ();
2835                                                 return Token.OP_MULT_ASSIGN;
2836                                         }
2837                                         return Token.STAR;
2838
2839                                 case '/':
2840                                         d = peek_char ();
2841                                         if (d == '='){
2842                                                 val = LocatedToken.Create (ref_line, col);
2843                                                 get_char ();
2844                                                 return Token.OP_DIV_ASSIGN;
2845                                         }
2846
2847                                         // Handle double-slash comments.
2848                                         if (d == '/'){
2849                                                 get_char ();
2850                                                 if (RootContext.Documentation != null && peek_char () == '/') {
2851                                                         get_char ();
2852                                                         // Don't allow ////.
2853                                                         if ((d = peek_char ()) != '/') {
2854                                                                 update_comment_location ();
2855                                                                 if (doc_state == XmlCommentState.Allowed)
2856                                                                         handle_one_line_xml_comment ();
2857                                                                 else if (doc_state == XmlCommentState.NotAllowed)
2858                                                                         warn_incorrect_doc_comment ();
2859                                                         }
2860                                                 }
2861                                                 while ((d = get_char ()) != -1 && (d != '\n') && d != '\r');
2862
2863                                                 any_token_seen |= tokens_seen;
2864                                                 tokens_seen = false;
2865                                                 comments_seen = false;
2866                                                 continue;
2867                                         } else if (d == '*'){
2868                                                 get_char ();
2869                                                 bool docAppend = false;
2870                                                 if (RootContext.Documentation != null && peek_char () == '*') {
2871                                                         get_char ();
2872                                                         update_comment_location ();
2873                                                         // But when it is /**/, just do nothing.
2874                                                         if (peek_char () == '/') {
2875                                                                 get_char ();
2876                                                                 continue;
2877                                                         }
2878                                                         if (doc_state == XmlCommentState.Allowed)
2879                                                                 docAppend = true;
2880                                                         else if (doc_state == XmlCommentState.NotAllowed)
2881                                                                 warn_incorrect_doc_comment ();
2882                                                 }
2883
2884                                                 int current_comment_start = 0;
2885                                                 if (docAppend) {
2886                                                         current_comment_start = xml_comment_buffer.Length;
2887                                                         xml_comment_buffer.Append (Environment.NewLine);
2888                                                 }
2889
2890                                                 while ((d = get_char ()) != -1){
2891                                                         if (d == '*' && peek_char () == '/'){
2892                                                                 get_char ();
2893                                                                 comments_seen = true;
2894                                                                 break;
2895                                                         }
2896                                                         if (docAppend)
2897                                                                 xml_comment_buffer.Append ((char) d);
2898
2899                                                         if (d == '\n'){
2900                                                                 any_token_seen |= tokens_seen;
2901                                                                 tokens_seen = false;
2902                                                                 //
2903                                                                 // Reset 'comments_seen' just to be consistent.
2904                                                                 // It doesn't matter either way, here.
2905                                                                 //
2906                                                                 comments_seen = false;
2907                                                         }
2908                                                 }
2909                                                 if (!comments_seen)
2910                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
2911
2912                                                 if (docAppend)
2913                                                         update_formatted_doc_comment (current_comment_start);
2914                                                 continue;
2915                                         }
2916                                         val = LocatedToken.Create (ref_line, col);
2917                                         return Token.DIV;
2918
2919                                 case '%':
2920                                         val = LocatedToken.Create (ref_line, col);
2921                                         if (peek_char () == '='){
2922                                                 get_char ();
2923                                                 return Token.OP_MOD_ASSIGN;
2924                                         }
2925                                         return Token.PERCENT;
2926
2927                                 case '^':
2928                                         val = LocatedToken.Create (ref_line, col);
2929                                         if (peek_char () == '='){
2930                                                 get_char ();
2931                                                 return Token.OP_XOR_ASSIGN;
2932                                         }
2933                                         return Token.CARRET;
2934
2935                                 case ':':
2936                                         val = LocatedToken.Create (ref_line, col);
2937                                         if (peek_char () == ':') {
2938                                                 get_char ();
2939                                                 return Token.DOUBLE_COLON;
2940                                         }
2941                                         return Token.COLON;
2942
2943                                 case '0': case '1': case '2': case '3': case '4':
2944                                 case '5': case '6': case '7': case '8': case '9':
2945                                         tokens_seen = true;
2946                                         return is_number (c);
2947
2948                                 case '\n': // white space
2949                                         any_token_seen |= tokens_seen;
2950                                         tokens_seen = false;
2951                                         comments_seen = false;
2952                                         continue;
2953
2954                                 case '.':
2955                                         tokens_seen = true;
2956                                         d = peek_char ();
2957                                         if (d >= '0' && d <= '9')
2958                                                 return is_number (c);
2959
2960                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2961                                         return Token.DOT;
2962
2963                                 case '#':
2964                                         if (tokens_seen || comments_seen) {
2965                                                 Eror_WrongPreprocessorLocation ();
2966                                                 return Token.ERROR;
2967                                         }
2968
2969                                         if (handle_preprocessing_directive (true))
2970                                                 continue;
2971
2972                                         bool directive_expected = false;
2973                                         while ((c = get_char ()) != -1) {
2974                                                 if (col == 1) {
2975                                                         directive_expected = true;
2976                                                 } else if (!directive_expected) {
2977                                                         // TODO: Implement comment support for disabled code and uncomment this code
2978 //                                                      if (c == '#') {
2979 //                                                              Eror_WrongPreprocessorLocation ();
2980 //                                                              return Token.ERROR;
2981 //                                                      }
2982                                                         continue;
2983                                                 }
2984
2985                                                 if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
2986                                                         continue;
2987
2988                                                 if (c == '#') {
2989                                                         if (handle_preprocessing_directive (false))
2990                                                                 break;
2991                                                 }
2992                                                 directive_expected = false;
2993                                         }
2994
2995                                         if (c != -1) {
2996                                                 tokens_seen = false;
2997                                                 continue;
2998                                         }
2999
3000                                         return Token.EOF;
3001
3002                                 case '"':
3003                                         return consume_string (false);
3004
3005                                 case '\'':
3006                                         return TokenizeBackslash ();
3007
3008                                 case '@':
3009                                         c = get_char ();
3010                                         if (c == '"') {
3011                                                 tokens_seen = true;
3012                                                 return consume_string (true);
3013                                         }
3014
3015                                         if (is_identifier_start_character (c)){
3016                                                 return consume_identifier (c, true);
3017                                         }
3018
3019                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3020                                         return Token.ERROR;
3021
3022                                 case EvalStatementParserCharacter:
3023                                         return Token.EVAL_STATEMENT_PARSER;
3024                                 case EvalCompilationUnitParserCharacter:
3025                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3026                                 case EvalUsingDeclarationsParserCharacter:
3027                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3028                                 }
3029
3030                                 if (is_identifier_start_character (c)) {
3031                                         tokens_seen = true;
3032                                         return consume_identifier (c);
3033                                 }
3034
3035                                 error_details = ((char)c).ToString ();
3036                                 return Token.ERROR;
3037                         }
3038
3039                         if (CompleteOnEOF){
3040                                 if (generated)
3041                                         return Token.COMPLETE_COMPLETION;
3042
3043                                 generated = true;
3044                                 return Token.GENERATE_COMPLETION;
3045                         }
3046
3047
3048                         return Token.EOF;
3049                 }
3050
3051                 int TokenizeBackslash ()
3052                 {
3053                         int c = get_char ();
3054                         tokens_seen = true;
3055                         if (c == '\'') {
3056                                 error_details = "Empty character literal";
3057                                 Report.Error (1011, Location, error_details);
3058                                 return Token.ERROR;
3059                         }
3060                         if (c == '\r' || c == '\n') {
3061                                 Report.Error (1010, Location, "Newline in constant");
3062                                 return Token.ERROR;
3063                         }
3064
3065                         int d;
3066                         c = escape (c, out d);
3067                         if (c == -1)
3068                                 return Token.ERROR;
3069                         if (d != 0)
3070                                 throw new NotImplementedException ();
3071
3072                         val = new CharLiteral ((char) c, Location);
3073                         c = get_char ();
3074
3075                         if (c != '\'') {
3076                                 Report.Error (1012, Location, "Too many characters in character literal");
3077
3078                                 // Try to recover, read until newline or next "'"
3079                                 while ((c = get_char ()) != -1) {
3080                                         if (c == '\n' || c == '\'')
3081                                                 break;
3082                                 }
3083                                 return Token.ERROR;
3084                         }
3085
3086                         return Token.LITERAL;
3087                 }
3088
3089                 int TokenizeLessThan ()
3090                 {
3091                         int d;
3092                         if (handle_typeof) {
3093                                 PushPosition ();
3094                                 if (parse_generic_dimension (out d)) {
3095                                         val = d;
3096                                         DiscardPosition ();
3097                                         return Token.GENERIC_DIMENSION;
3098                                 }
3099                                 PopPosition ();
3100                         }
3101
3102                         // Save current position and parse next token.
3103                         PushPosition ();
3104                         if (parse_less_than ()) {
3105                                 if (parsing_generic_declaration && token () != Token.DOT) {
3106                                         d = Token.OP_GENERICS_LT_DECL;
3107                                 } else {
3108                                         d = Token.OP_GENERICS_LT;
3109                                 }
3110                                 PopPosition ();
3111                                 return d;
3112                         }
3113
3114                         PopPosition ();
3115                         parsing_generic_less_than = 0;
3116
3117                         d = peek_char ();
3118                         if (d == '<') {
3119                                 get_char ();
3120                                 d = peek_char ();
3121
3122                                 if (d == '=') {
3123                                         get_char ();
3124                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3125                                 }
3126                                 return Token.OP_SHIFT_LEFT;
3127                         }
3128
3129                         if (d == '=') {
3130                                 get_char ();
3131                                 return Token.OP_LE;
3132                         }
3133                         return Token.OP_LT;
3134                 }
3135
3136                 //
3137                 // Handles one line xml comment
3138                 //
3139                 private void handle_one_line_xml_comment ()
3140                 {
3141                         int c;
3142                         while ((c = peek_char ()) == ' ')
3143                                 get_char (); // skip heading whitespaces.
3144                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3145                                 xml_comment_buffer.Append ((char) get_char ());
3146                         }
3147                         if (c == '\r' || c == '\n')
3148                                 xml_comment_buffer.Append (Environment.NewLine);
3149                 }
3150
3151                 //
3152                 // Remove heading "*" in Javadoc-like xml documentation.
3153                 //
3154                 private void update_formatted_doc_comment (int current_comment_start)
3155                 {
3156                         int length = xml_comment_buffer.Length - current_comment_start;
3157                         string [] lines = xml_comment_buffer.ToString (
3158                                 current_comment_start,
3159                                 length).Replace ("\r", "").Split ('\n');
3160
3161                         // The first line starts with /**, thus it is not target
3162                         // for the format check.
3163                         for (int i = 1; i < lines.Length; i++) {
3164                                 string s = lines [i];
3165                                 int idx = s.IndexOf ('*');
3166                                 string head = null;
3167                                 if (idx < 0) {
3168                                         if (i < lines.Length - 1)
3169                                                 return;
3170                                         head = s;
3171                                 } else
3172                                         head = s.Substring (0, idx);
3173                                 foreach (char c in head)
3174                                         if (c != ' ')
3175                                                 return;
3176                                 lines [i] = s.Substring (idx + 1);
3177                         }
3178                         xml_comment_buffer.Remove (current_comment_start, length);
3179                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3180                 }
3181
3182                 //
3183                 // Updates current comment location.
3184                 //
3185                 private void update_comment_location ()
3186                 {
3187                         if (current_comment_location.IsNull) {
3188                                 // "-2" is for heading "//" or "/*"
3189                                 current_comment_location =
3190                                         new Location (ref_line, hidden ? -1 : col - 2);
3191                         }
3192                 }
3193
3194                 //
3195                 // Checks if there was incorrect doc comments and raise
3196                 // warnings.
3197                 //
3198                 public void check_incorrect_doc_comment ()
3199                 {
3200                         if (xml_comment_buffer.Length > 0)
3201                                 warn_incorrect_doc_comment ();
3202                 }
3203
3204                 //
3205                 // Raises a warning when tokenizer found incorrect doccomment
3206                 // markup.
3207                 //
3208                 private void warn_incorrect_doc_comment ()
3209                 {
3210                         if (doc_state != XmlCommentState.Error) {
3211                                 doc_state = XmlCommentState.Error;
3212                                 // in csc, it is 'XML comment is not placed on
3213                                 // a valid language element'. But that does not
3214                                 // make sense.
3215                                 Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
3216                         }
3217                 }
3218
3219                 //
3220                 // Consumes the saved xml comment lines (if any)
3221                 // as for current target member or type.
3222                 //
3223                 public string consume_doc_comment ()
3224                 {
3225                         if (xml_comment_buffer.Length > 0) {
3226                                 string ret = xml_comment_buffer.ToString ();
3227                                 reset_doc_comment ();
3228                                 return ret;
3229                         }
3230                         return null;
3231                 }
3232
3233                 Report Report {
3234                         get { return context.Report; }
3235                 }
3236
3237                 void reset_doc_comment ()
3238                 {
3239                         xml_comment_buffer.Length = 0;
3240                         current_comment_location = Location.Null;
3241                 }
3242
3243                 public void cleanup ()
3244                 {
3245                         if (ifstack != null && ifstack.Count >= 1) {
3246                                 int state = ifstack.Pop ();
3247                                 if ((state & REGION) != 0)
3248                                         Report.Error (1038, Location, "#endregion directive expected");
3249                                 else
3250                                         Report.Error (1027, Location, "Expected `#endif' directive");
3251                         }
3252                 }
3253         }
3254
3255         //
3256         // Indicates whether it accepts XML documentation or not.
3257         //
3258         public enum XmlCommentState {
3259                 // comment is allowed in this state.
3260                 Allowed,
3261                 // comment is not allowed in this state.
3262                 NotAllowed,
3263                 // once comments appeared when it is NotAllowed, then the
3264                 // state is changed to it, until the state is changed to
3265                 // .Allowed.
3266                 Error
3267         }
3268 }
3269