mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 //
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.IO;
  19 using System.Globalization;
  20 using System.Diagnostics;
  21
  22 namespace Mono.CSharp
  23 {
  24         /// <summary>
  25         ///    Tokenizer for C# source code.
  26         /// </summary>
  27
  28         public class Tokenizer : yyParser.yyInput
  29         {
  30                 class KeywordEntry<T>
  31                 {
  32                         public readonly T Token;
  33                         public KeywordEntry<T> Next;
  34                         public readonly char[] Value;
  35
  36                         public KeywordEntry (string value, T token)
  37                         {
  38                                 this.Value = value.ToCharArray ();
  39                                 this.Token = token;
  40                         }
  41                 }
  42
  43                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  44                 {
  45                         readonly int length;
  46
  47                         public IdentifiersComparer (int length)
  48                         {
  49                                 this.length = length;
  50                         }
  51
  52                         public bool Equals (char[] x, char[] y)
  53                         {
  54                                 for (int i = 0; i < length; ++i)
  55                                         if (x [i] != y [i])
  56                                                 return false;
  57
  58                                 return true;
  59                         }
  60
  61                         public int GetHashCode (char[] obj)
  62                         {
  63                                 int h = 0;
  64                                 for (int i = 0; i < length; ++i)
  65                                         h = (h << 5) - h + obj [i];
  66
  67                                 return h;
  68                         }
  69                 }
  70
  71                 //
  72                 // This class has to be used in the parser only, it reuses token
  73                 // details after each parse
  74                 //
  75                 public class LocatedToken
  76                 {
  77                         int row, column;
  78                         string value;
  79
  80                         static LocatedToken[] buffer;
  81                         static int pos;
  82
  83                         private LocatedToken ()
  84                         {
  85                         }
  86
  87                         public static LocatedToken Create (int row, int column)
  88                         {
  89                                 return Create (null, row, column);
  90                         }
  91
  92                         public static LocatedToken Create (string value, int row, int column)
  93                         {
  94                                 //
  95                                 // TODO: I am not very happy about the logic but it's the best
  96                                 // what I could come up with for now.
  97                                 // Ideally we should be using just tiny buffer (256 elements) which
  98                                 // is enough to hold all details for currect stack and recycle elements
  99                                 // poped from the stack but there is a trick needed to recycle
 100                                 // them properly.
 101                                 //
 102                                 LocatedToken entry;
 103                                 if (pos >= buffer.Length) {
 104                                         entry = new LocatedToken ();
 105                                 } else {
 106                                         entry = buffer [pos];
 107                                         if (entry == null) {
 108                                                 entry = new LocatedToken ();
 109                                                 buffer [pos] = entry;
 110                                         }
 111
 112                                         ++pos;
 113                                 }
 114                                 entry.value = value;
 115                                 entry.row = row;
 116                                 entry.column = column;
 117                                 return entry;
 118                         }
 119
 120                         //
 121                         // Used for token not required by expression evaluator
 122                         //
 123                         [Conditional ("FULL_AST")]
 124                         public static void CreateOptional (int row, int col, ref object token)
 125                         {
 126                                 token = Create (row, col);
 127                         }
 128
 129                         public static void Initialize ()
 130                         {
 131                                 if (buffer == null)
 132                                         buffer = new LocatedToken [10000];
 133                                 pos = 0;
 134                         }
 135
 136                         public Location Location {
 137                                 get { return new Location (row, column); }
 138                         }
 139
 140                         public string Value {
 141                                 get { return value; }
 142                         }
 143                 }
 144
 145                 enum PreprocessorDirective
 146                 {
 147                         Invalid = 0,
 148
 149                         Region = 1,
 150                         Endregion = 2,
 151                         If = 3 | RequiresArgument,
 152                         Endif = 4,
 153                         Elif = 5 | RequiresArgument,
 154                         Else = 6,
 155                         Define = 7 | RequiresArgument,
 156                         Undef = 8 | RequiresArgument,
 157                         Error = 9,
 158                         Warning = 10,
 159                         Pragma = 11 | CustomArgumentsParsing,
 160                         Line = 12,
 161
 162                         CustomArgumentsParsing = 1 << 10,
 163                         RequiresArgument = 1 << 11
 164                 }
 165
 166                 SeekableStreamReader reader;
 167                 SourceFile ref_name;
 168                 CompilationUnit file_name;
 169                 CompilerContext context;
 170                 bool hidden = false;
 171                 int ref_line = 1;
 172                 int line = 1;
 173                 int col = 0;
 174                 int previous_col;
 175                 int current_token;
 176                 int tab_size;
 177                 bool handle_get_set = false;
 178                 bool handle_remove_add = false;
 179                 bool handle_where = false;
 180                 bool handle_typeof = false;
 181                 bool lambda_arguments_parsing;
 182                 Location current_comment_location = Location.Null;
 183                 List<Location> escaped_identifiers;
 184                 int parsing_generic_less_than;
 185
 186                 //
 187                 // Used mainly for parser optimizations. Some expressions for instance
 188                 // can appear only in block (including initializer, base initializer)
 189                 // scope only
 190                 //
 191                 public int parsing_block;
 192                 internal bool query_parsing;
 193
 194                 //
 195                 // When parsing type only, useful for ambiguous nullable types
 196                 //
 197                 public int parsing_type;
 198
 199                 //
 200                 // Set when parsing generic declaration (type or method header)
 201                 //
 202                 public bool parsing_generic_declaration;
 203
 204                 //
 205                 // The value indicates that we have not reach any declaration or
 206                 // namespace yet
 207                 //
 208                 public int parsing_declaration;
 209
 210                 //
 211                 // The special character to inject on streams to trigger the EXPRESSION_PARSE
 212                 // token to be returned.   It just happens to be a Unicode character that
 213                 // would never be part of a program (can not be an identifier).
 214                 //
 215                 // This character is only tested just before the tokenizer is about to report
 216                 // an error;   So on the regular operation mode, this addition will have no
 217                 // impact on the tokenizer's performance.
 218                 //
 219
 220                 public const int EvalStatementParserCharacter = 0x2190;   // Unicode Left Arrow
 221                 public const int EvalCompilationUnitParserCharacter = 0x2191;  // Unicode Arrow
 222                 public const int EvalUsingDeclarationsParserCharacter = 0x2192;  // Unicode Arrow
 223
 224                 //
 225                 // XML documentation buffer. The save point is used to divide
 226                 // comments on types and comments on members.
 227                 //
 228                 StringBuilder xml_comment_buffer;
 229
 230                 //
 231                 // See comment on XmlCommentState enumeration.
 232                 //
 233                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 234
 235                 //
 236                 // Whether tokens have been seen on this line
 237                 //
 238                 bool tokens_seen = false;
 239
 240                 //
 241                 // Set to true once the GENERATE_COMPLETION token has bee
 242                 // returned.   This helps produce one GENERATE_COMPLETION,
 243                 // as many COMPLETE_COMPLETION as necessary to complete the
 244                 // AST tree and one final EOF.
 245                 //
 246                 bool generated;
 247
 248                 //
 249                 // Whether a token has been seen on the file
 250                 // This is needed because `define' is not allowed to be used
 251                 // after a token has been seen.
 252                 //
 253                 bool any_token_seen = false;
 254
 255                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 256
 257                 public bool PropertyParsing {
 258                         get { return handle_get_set; }
 259                         set { handle_get_set = value; }
 260                 }
 261
 262                 public bool EventParsing {
 263                         get { return handle_remove_add; }
 264                         set { handle_remove_add = value; }
 265                 }
 266
 267                 public bool ConstraintsParsing {
 268                         get { return handle_where; }
 269                         set { handle_where = value; }
 270                 }
 271
 272                 public bool TypeOfParsing {
 273                         get { return handle_typeof; }
 274                         set { handle_typeof = value; }
 275                 }
 276
 277                 public int TabSize {
 278                         get { return tab_size; }
 279                         set { tab_size = value; }
 280                 }
 281
 282                 public XmlCommentState doc_state {
 283                         get { return xml_doc_state; }
 284                         set {
 285                                 if (value == XmlCommentState.Allowed) {
 286                                         check_incorrect_doc_comment ();
 287                                         reset_doc_comment ();
 288                                 }
 289                                 xml_doc_state = value;
 290                         }
 291                 }
 292
 293                 //
 294                 // This is used to trigger completion generation on the parser
 295                 public bool CompleteOnEOF;
 296
 297                 void AddEscapedIdentifier (Location loc)
 298                 {
 299                         if (escaped_identifiers == null)
 300                                 escaped_identifiers = new List<Location> ();
 301
 302                         escaped_identifiers.Add (loc);
 303                 }
 304
 305                 public bool IsEscapedIdentifier (MemberName name)
 306                 {
 307                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 308                 }
 309
 310                 //
 311                 // Class variables
 312                 //
 313                 static KeywordEntry<int>[][] keywords;
 314                 static KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 315                 static Dictionary<string, object> keyword_strings;              // TODO: HashSet
 316                 static NumberStyles styles;
 317                 static NumberFormatInfo csharp_format_info;
 318
 319                 // Pragma arguments
 320                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 321                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 322                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 323                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 324
 325                 //
 326                 // Values for the associated token returned
 327                 //
 328                 internal int putback_char;      // Used by repl only
 329                 object val;
 330
 331                 //
 332                 // Pre-processor
 333                 //
 334                 const int TAKING        = 1;
 335                 const int ELSE_SEEN     = 4;
 336                 const int PARENT_TAKING = 8;
 337                 const int REGION        = 16;
 338
 339                 //
 340                 // pre-processor if stack state:
 341                 //
 342                 Stack<int> ifstack;
 343
 344                 static System.Text.StringBuilder string_builder;
 345
 346                 const int max_id_size = 512;
 347                 static char [] id_builder = new char [max_id_size];
 348
 349                 public static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 350
 351                 const int max_number_size = 512;
 352                 static char [] number_builder = new char [max_number_size];
 353                 static int number_pos;
 354
 355                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
 356
 357                 public int Line {
 358                         get {
 359                                 return ref_line;
 360                         }
 361                 }
 362
 363                 //
 364                 // This is used when the tokenizer needs to save
 365                 // the current position as it needs to do some parsing
 366                 // on its own to deamiguate a token in behalf of the
 367                 // parser.
 368                 //
 369                 Stack<Position> position_stack = new Stack<Position> (2);
 370
 371                 class Position {
 372                         public int position;
 373                         public int line;
 374                         public int ref_line;
 375                         public int col;
 376                         public bool hidden;
 377                         public int putback_char;
 378                         public int previous_col;
 379                         public Stack<int> ifstack;
 380                         public int parsing_generic_less_than;
 381                         public int current_token;
 382                         public object val;
 383
 384                         public Position (Tokenizer t)
 385                         {
 386                                 position = t.reader.Position;
 387                                 line = t.line;
 388                                 ref_line = t.ref_line;
 389                                 col = t.col;
 390                                 hidden = t.hidden;
 391                                 putback_char = t.putback_char;
 392                                 previous_col = t.previous_col;
 393                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 394                                         // There is no simple way to clone Stack<T> all
 395                                         // methods reverse the order
 396                                         var clone = t.ifstack.ToArray ();
 397                                         Array.Reverse (clone);
 398                                         ifstack = new Stack<int> (clone);
 399                                 }
 400                                 parsing_generic_less_than = t.parsing_generic_less_than;
 401                                 current_token = t.current_token;
 402                                 val = t.val;
 403                         }
 404                 }
 405
 406                 public void PushPosition ()
 407                 {
 408                         position_stack.Push (new Position (this));
 409                 }
 410
 411                 public void PopPosition ()
 412                 {
 413                         Position p = position_stack.Pop ();
 414
 415                         reader.Position = p.position;
 416                         ref_line = p.ref_line;
 417                         line = p.line;
 418                         col = p.col;
 419                         hidden = p.hidden;
 420                         putback_char = p.putback_char;
 421                         previous_col = p.previous_col;
 422                         ifstack = p.ifstack;
 423                         parsing_generic_less_than = p.parsing_generic_less_than;
 424                         current_token = p.current_token;
 425                         val = p.val;
 426                 }
 427
 428                 // Do not reset the position, ignore it.
 429                 public void DiscardPosition ()
 430                 {
 431                         position_stack.Pop ();
 432                 }
 433
 434                 static void AddKeyword (string kw, int token)
 435                 {
 436                         keyword_strings.Add (kw, null);
 437
 438                         AddKeyword (keywords, kw, token);
 439                 }
 440
 441                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 442                 {
 443                         AddKeyword (keywords_preprocessor, kw, directive);
 444                 }
 445
 446                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 447                 {
 448                         int length = kw.Length;
 449                         if (keywords[length] == null) {
 450                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 451                         }
 452
 453                         int char_index = kw[0] - '_';
 454                         var kwe = keywords[length][char_index];
 455                         if (kwe == null) {
 456                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 457                                 return;
 458                         }
 459
 460                         while (kwe.Next != null) {
 461                                 kwe = kwe.Next;
 462                         }
 463
 464                         kwe.Next = new KeywordEntry<T> (kw, token);
 465                 }
 466
 467                 static void InitTokens ()
 468                 {
 469                         keyword_strings = new Dictionary<string, object> ();
 470
 471                         // 11 is the length of the longest keyword for now
 472                         keywords = new KeywordEntry<int> [11] [];
 473
 474                         AddKeyword ("__arglist", Token.ARGLIST);
 475                         AddKeyword ("abstract", Token.ABSTRACT);
 476                         AddKeyword ("as", Token.AS);
 477                         AddKeyword ("add", Token.ADD);
 478                         AddKeyword ("base", Token.BASE);
 479                         AddKeyword ("bool", Token.BOOL);
 480                         AddKeyword ("break", Token.BREAK);
 481                         AddKeyword ("byte", Token.BYTE);
 482                         AddKeyword ("case", Token.CASE);
 483                         AddKeyword ("catch", Token.CATCH);
 484                         AddKeyword ("char", Token.CHAR);
 485                         AddKeyword ("checked", Token.CHECKED);
 486                         AddKeyword ("class", Token.CLASS);
 487                         AddKeyword ("const", Token.CONST);
 488                         AddKeyword ("continue", Token.CONTINUE);
 489                         AddKeyword ("decimal", Token.DECIMAL);
 490                         AddKeyword ("default", Token.DEFAULT);
 491                         AddKeyword ("delegate", Token.DELEGATE);
 492                         AddKeyword ("do", Token.DO);
 493                         AddKeyword ("double", Token.DOUBLE);
 494                         AddKeyword ("else", Token.ELSE);
 495                         AddKeyword ("enum", Token.ENUM);
 496                         AddKeyword ("event", Token.EVENT);
 497                         AddKeyword ("explicit", Token.EXPLICIT);
 498                         AddKeyword ("extern", Token.EXTERN);
 499                         AddKeyword ("false", Token.FALSE);
 500                         AddKeyword ("finally", Token.FINALLY);
 501                         AddKeyword ("fixed", Token.FIXED);
 502                         AddKeyword ("float", Token.FLOAT);
 503                         AddKeyword ("for", Token.FOR);
 504                         AddKeyword ("foreach", Token.FOREACH);
 505                         AddKeyword ("goto", Token.GOTO);
 506                         AddKeyword ("get", Token.GET);
 507                         AddKeyword ("if", Token.IF);
 508                         AddKeyword ("implicit", Token.IMPLICIT);
 509                         AddKeyword ("in", Token.IN);
 510                         AddKeyword ("int", Token.INT);
 511                         AddKeyword ("interface", Token.INTERFACE);
 512                         AddKeyword ("internal", Token.INTERNAL);
 513                         AddKeyword ("is", Token.IS);
 514                         AddKeyword ("lock", Token.LOCK);
 515                         AddKeyword ("long", Token.LONG);
 516                         AddKeyword ("namespace", Token.NAMESPACE);
 517                         AddKeyword ("new", Token.NEW);
 518                         AddKeyword ("null", Token.NULL);
 519                         AddKeyword ("object", Token.OBJECT);
 520                         AddKeyword ("operator", Token.OPERATOR);
 521                         AddKeyword ("out", Token.OUT);
 522                         AddKeyword ("override", Token.OVERRIDE);
 523                         AddKeyword ("params", Token.PARAMS);
 524                         AddKeyword ("private", Token.PRIVATE);
 525                         AddKeyword ("protected", Token.PROTECTED);
 526                         AddKeyword ("public", Token.PUBLIC);
 527                         AddKeyword ("readonly", Token.READONLY);
 528                         AddKeyword ("ref", Token.REF);
 529                         AddKeyword ("remove", Token.REMOVE);
 530                         AddKeyword ("return", Token.RETURN);
 531                         AddKeyword ("sbyte", Token.SBYTE);
 532                         AddKeyword ("sealed", Token.SEALED);
 533                         AddKeyword ("set", Token.SET);
 534                         AddKeyword ("short", Token.SHORT);
 535                         AddKeyword ("sizeof", Token.SIZEOF);
 536                         AddKeyword ("stackalloc", Token.STACKALLOC);
 537                         AddKeyword ("static", Token.STATIC);
 538                         AddKeyword ("string", Token.STRING);
 539                         AddKeyword ("struct", Token.STRUCT);
 540                         AddKeyword ("switch", Token.SWITCH);
 541                         AddKeyword ("this", Token.THIS);
 542                         AddKeyword ("throw", Token.THROW);
 543                         AddKeyword ("true", Token.TRUE);
 544                         AddKeyword ("try", Token.TRY);
 545                         AddKeyword ("typeof", Token.TYPEOF);
 546                         AddKeyword ("uint", Token.UINT);
 547                         AddKeyword ("ulong", Token.ULONG);
 548                         AddKeyword ("unchecked", Token.UNCHECKED);
 549                         AddKeyword ("unsafe", Token.UNSAFE);
 550                         AddKeyword ("ushort", Token.USHORT);
 551                         AddKeyword ("using", Token.USING);
 552                         AddKeyword ("virtual", Token.VIRTUAL);
 553                         AddKeyword ("void", Token.VOID);
 554                         AddKeyword ("volatile", Token.VOLATILE);
 555                         AddKeyword ("while", Token.WHILE);
 556                         AddKeyword ("partial", Token.PARTIAL);
 557                         AddKeyword ("where", Token.WHERE);
 558                         AddKeyword ("async", Token.ASYNC);
 559
 560                         // LINQ keywords
 561                         AddKeyword ("from", Token.FROM);
 562                         AddKeyword ("join", Token.JOIN);
 563                         AddKeyword ("on", Token.ON);
 564                         AddKeyword ("equals", Token.EQUALS);
 565                         AddKeyword ("select", Token.SELECT);
 566                         AddKeyword ("group", Token.GROUP);
 567                         AddKeyword ("by", Token.BY);
 568                         AddKeyword ("let", Token.LET);
 569                         AddKeyword ("orderby", Token.ORDERBY);
 570                         AddKeyword ("ascending", Token.ASCENDING);
 571                         AddKeyword ("descending", Token.DESCENDING);
 572                         AddKeyword ("into", Token.INTO);
 573
 574                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 575
 576                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 577                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 578                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 579                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 580                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 581                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 582                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 583                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 584                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 585                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 586                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 587                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 588                 }
 589
 590                 //
 591                 // Class initializer
 592                 //
 593                 static Tokenizer ()
 594                 {
 595                         InitTokens ();
 596                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 597                         styles = NumberStyles.Float;
 598
 599                         string_builder = new System.Text.StringBuilder ();
 600                 }
 601
 602                 int GetKeyword (char[] id, int id_len)
 603                 {
 604                         //
 605                         // Keywords are stored in an array of arrays grouped by their
 606                         // length and then by the first character
 607                         //
 608                         if (id_len >= keywords.Length || keywords [id_len] == null)
 609                                 return -1;
 610
 611                         int first_index = id [0] - '_';
 612                         if (first_index > 'z' - '_')
 613                                 return -1;
 614
 615                         var kwe = keywords [id_len] [first_index];
 616                         if (kwe == null)
 617                                 return -1;
 618
 619                         int res;
 620                         do {
 621                                 res = kwe.Token;
 622                                 for (int i = 1; i < id_len; ++i) {
 623                                         if (id [i] != kwe.Value [i]) {
 624                                                 res = 0;
 625                                                 kwe = kwe.Next;
 626                                                 break;
 627                                         }
 628                                 }
 629                         } while (res == 0 && kwe != null);
 630
 631                         if (res == 0)
 632                                 return -1;
 633
 634                         int next_token;
 635                         switch (res) {
 636                         case Token.GET:
 637                         case Token.SET:
 638                                 if (!handle_get_set)
 639                                         res = -1;
 640                                 break;
 641                         case Token.REMOVE:
 642                         case Token.ADD:
 643                                 if (!handle_remove_add)
 644                                         res = -1;
 645                                 break;
 646                         case Token.EXTERN:
 647                                 if (parsing_declaration == 0)
 648                                         res = Token.EXTERN_ALIAS;
 649                                 break;
 650                         case Token.DEFAULT:
 651                                 if (peek_token () == Token.COLON) {
 652                                         token ();
 653                                         res = Token.DEFAULT_COLON;
 654                                 }
 655                                 break;
 656                         case Token.WHERE:
 657                                 if (!handle_where && !query_parsing)
 658                                         res = -1;
 659                                 break;
 660                         case Token.FROM:
 661                                 //
 662                                 // A query expression is any expression that starts with `from identifier'
 663                                 // followed by any token except ; , =
 664                                 //
 665                                 if (!query_parsing) {
 666                                         if (lambda_arguments_parsing) {
 667                                                 res = -1;
 668                                                 break;
 669                                         }
 670
 671                                         PushPosition ();
 672                                         // HACK: to disable generics micro-parser, because PushPosition does not
 673                                         // store identifiers array
 674                                         parsing_generic_less_than = 1;
 675                                         switch (xtoken ()) {
 676                                         case Token.IDENTIFIER:
 677                                         case Token.INT:
 678                                         case Token.BOOL:
 679                                         case Token.BYTE:
 680                                         case Token.CHAR:
 681                                         case Token.DECIMAL:
 682                                         case Token.FLOAT:
 683                                         case Token.LONG:
 684                                         case Token.OBJECT:
 685                                         case Token.STRING:
 686                                         case Token.UINT:
 687                                         case Token.ULONG:
 688                                                 next_token = xtoken ();
 689                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 690                                                         goto default;
 691
 692                                                 res = Token.FROM_FIRST;
 693                                                 query_parsing = true;
 694                                                 if (RootContext.Version <= LanguageVersion.ISO_2)
 695                                                         Report.FeatureIsNotAvailable (Location, "query expressions");
 696                                                 break;
 697                                         case Token.VOID:
 698                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 699                                                 break;
 700                                         default:
 701                                                 PopPosition ();
 702                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 703                                                 // which has been overwritten before we grabbed the identifier
 704                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 705                                                 return -1;
 706                                         }
 707                                         PopPosition ();
 708                                 }
 709                                 break;
 710                         case Token.JOIN:
 711                         case Token.ON:
 712                         case Token.EQUALS:
 713                         case Token.SELECT:
 714                         case Token.GROUP:
 715                         case Token.BY:
 716                         case Token.LET:
 717                         case Token.ORDERBY:
 718                         case Token.ASCENDING:
 719                         case Token.DESCENDING:
 720                         case Token.INTO:
 721                                 if (!query_parsing)
 722                                         res = -1;
 723                                 break;
 724
 725                         case Token.USING:
 726                         case Token.NAMESPACE:
 727                                 // TODO: some explanation needed
 728                                 check_incorrect_doc_comment ();
 729                                 break;
 730
 731                         case Token.PARTIAL:
 732                                 if (parsing_block > 0) {
 733                                         res = -1;
 734                                         break;
 735                                 }
 736
 737                                 // Save current position and parse next token.
 738                                 PushPosition ();
 739
 740                                 next_token = token ();
 741                                 bool ok = (next_token == Token.CLASS) ||
 742                                         (next_token == Token.STRUCT) ||
 743                                         (next_token == Token.INTERFACE) ||
 744                                         (next_token == Token.VOID);
 745
 746                                 PopPosition ();
 747
 748                                 if (ok) {
 749                                         if (next_token == Token.VOID) {
 750                                                 if (RootContext.Version == LanguageVersion.ISO_1 ||
 751                                                     RootContext.Version == LanguageVersion.ISO_2)
 752                                                         Report.FeatureIsNotAvailable (Location, "partial methods");
 753                                         } else if (RootContext.Version == LanguageVersion.ISO_1)
 754                                                 Report.FeatureIsNotAvailable (Location, "partial types");
 755
 756                                         return res;
 757                                 }
 758
 759                                 if (next_token < Token.LAST_KEYWORD) {
 760                                         Report.Error (267, Location,
 761                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 762                                         return token ();
 763                                 }
 764
 765                                 res = -1;
 766                                 break;
 767
 768                         case Token.ASYNC:
 769                                 if (parsing_block > 0 || RootContext.Version != LanguageVersion.Future) {
 770                                         res = -1;
 771                                         break;
 772                                 }
 773                                 break;
 774                         }
 775
 776                         return res;
 777                 }
 778
 779                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 780                 {
 781                         //
 782                         // Keywords are stored in an array of arrays grouped by their
 783                         // length and then by the first character
 784                         //
 785                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 786                                 return PreprocessorDirective.Invalid;
 787
 788                         int first_index = id[0] - '_';
 789                         if (first_index > 'z' - '_')
 790                                 return PreprocessorDirective.Invalid;
 791
 792                         var kwe = keywords_preprocessor[id_len][first_index];
 793                         if (kwe == null)
 794                                 return PreprocessorDirective.Invalid;
 795
 796                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 797                         do {
 798                                 res = kwe.Token;
 799                                 for (int i = 1; i < id_len; ++i) {
 800                                         if (id[i] != kwe.Value[i]) {
 801                                                 res = 0;
 802                                                 kwe = kwe.Next;
 803                                                 break;
 804                                         }
 805                                 }
 806                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 807
 808                         return res;
 809                 }
 810
 811                 public Location Location {
 812                         get {
 813                                 return new Location (ref_line, hidden ? -1 : col);
 814                         }
 815                 }
 816
 817                 public Tokenizer (SeekableStreamReader input, CompilationUnit file, CompilerContext ctx)
 818                 {
 819                         this.ref_name = file;
 820                         this.file_name = file;
 821                         this.context = ctx;
 822                         reader = input;
 823
 824                         putback_char = -1;
 825
 826                         xml_comment_buffer = new StringBuilder ();
 827
 828                         if (Environment.OSVersion.Platform == PlatformID.Win32NT)
 829                                 tab_size = 4;
 830                         else
 831                                 tab_size = 8;
 832
 833                         //
 834                         // FIXME: This could be `Location.Push' but we have to
 835                         // find out why the MS compiler allows this
 836                         //
 837                         Mono.CSharp.Location.Push (file, file);
 838                 }
 839
 840                 static bool is_identifier_start_character (int c)
 841                 {
 842                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 843                 }
 844
 845                 static bool is_identifier_part_character (char c)
 846                 {
 847                         if (c >= 'a' && c <= 'z')
 848                                 return true;
 849
 850                         if (c >= 'A' && c <= 'Z')
 851                                 return true;
 852
 853                         if (c == '_' || (c >= '0' && c <= '9'))
 854                                 return true;
 855
 856                         if (c < 0x80)
 857                                 return false;
 858
 859                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 860                 }
 861
 862                 public static bool IsKeyword (string s)
 863                 {
 864                         return keyword_strings.ContainsKey (s);
 865                 }
 866
 867                 //
 868                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 869                 //
 870                 int TokenizeOpenParens ()
 871                 {
 872                         int ptoken;
 873                         current_token = -1;
 874
 875                         int bracket_level = 0;
 876                         bool is_type = false;
 877                         bool can_be_type = false;
 878
 879                         while (true) {
 880                                 ptoken = current_token;
 881                                 token ();
 882
 883                                 switch (current_token) {
 884                                 case Token.CLOSE_PARENS:
 885                                         token ();
 886
 887                                         //
 888                                         // Expression inside parens is lambda, (int i) =>
 889                                         //
 890                                         if (current_token == Token.ARROW)
 891                                                 return Token.OPEN_PARENS_LAMBDA;
 892
 893                                         //
 894                                         // Expression inside parens is single type, (int[])
 895                                         //
 896                                         if (is_type)
 897                                                 return Token.OPEN_PARENS_CAST;
 898
 899                                         //
 900                                         // Expression is possible cast, look at next token, (T)null
 901                                         //
 902                                         if (can_be_type) {
 903                                                 switch (current_token) {
 904                                                 case Token.OPEN_PARENS:
 905                                                 case Token.BANG:
 906                                                 case Token.TILDE:
 907                                                 case Token.IDENTIFIER:
 908                                                 case Token.LITERAL:
 909                                                 case Token.BASE:
 910                                                 case Token.CHECKED:
 911                                                 case Token.DELEGATE:
 912                                                 case Token.FALSE:
 913                                                 case Token.FIXED:
 914                                                 case Token.NEW:
 915                                                 case Token.NULL:
 916                                                 case Token.SIZEOF:
 917                                                 case Token.THIS:
 918                                                 case Token.THROW:
 919                                                 case Token.TRUE:
 920                                                 case Token.TYPEOF:
 921                                                 case Token.UNCHECKED:
 922                                                 case Token.UNSAFE:
 923                                                 case Token.DEFAULT:
 924
 925                                                 //
 926                                                 // These can be part of a member access
 927                                                 //
 928                                                 case Token.INT:
 929                                                 case Token.UINT:
 930                                                 case Token.SHORT:
 931                                                 case Token.USHORT:
 932                                                 case Token.LONG:
 933                                                 case Token.ULONG:
 934                                                 case Token.DOUBLE:
 935                                                 case Token.FLOAT:
 936                                                 case Token.CHAR:
 937                                                 case Token.BYTE:
 938                                                 case Token.DECIMAL:
 939                                                 case Token.BOOL:
 940                                                         return Token.OPEN_PARENS_CAST;
 941                                                 }
 942                                         }
 943                                         return Token.OPEN_PARENS;
 944
 945                                 case Token.DOT:
 946                                 case Token.DOUBLE_COLON:
 947                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
 948                                                 goto default;
 949
 950                                         continue;
 951
 952                                 case Token.IDENTIFIER:
 953                                         switch (ptoken) {
 954                                         case Token.DOT:
 955                                                 if (bracket_level == 0) {
 956                                                         is_type = false;
 957                                                         can_be_type = true;
 958                                                 }
 959
 960                                                 continue;
 961                                         case Token.OP_GENERICS_LT:
 962                                         case Token.COMMA:
 963                                         case Token.DOUBLE_COLON:
 964                                         case -1:
 965                                                 if (bracket_level == 0)
 966                                                         can_be_type = true;
 967                                                 continue;
 968                                         default:
 969                                                 can_be_type = is_type = false;
 970                                                 continue;
 971                                         }
 972
 973                                 case Token.OBJECT:
 974                                 case Token.STRING:
 975                                 case Token.BOOL:
 976                                 case Token.DECIMAL:
 977                                 case Token.FLOAT:
 978                                 case Token.DOUBLE:
 979                                 case Token.SBYTE:
 980                                 case Token.BYTE:
 981                                 case Token.SHORT:
 982                                 case Token.USHORT:
 983                                 case Token.INT:
 984                                 case Token.UINT:
 985                                 case Token.LONG:
 986                                 case Token.ULONG:
 987                                 case Token.CHAR:
 988                                 case Token.VOID:
 989                                         if (bracket_level == 0)
 990                                                 is_type = true;
 991                                         continue;
 992
 993                                 case Token.COMMA:
 994                                         if (bracket_level == 0) {
 995                                                 bracket_level = 100;
 996                                                 can_be_type = is_type = false;
 997                                         }
 998                                         continue;
 999
1000                                 case Token.OP_GENERICS_LT:
1001                                 case Token.OPEN_BRACKET:
1002                                         if (bracket_level++ == 0)
1003                                                 is_type = true;
1004                                         continue;
1005
1006                                 case Token.OP_GENERICS_GT:
1007                                 case Token.CLOSE_BRACKET:
1008                                         --bracket_level;
1009                                         continue;
1010
1011                                 case Token.INTERR_NULLABLE:
1012                                 case Token.STAR:
1013                                         if (bracket_level == 0)
1014                                                 is_type = true;
1015                                         continue;
1016
1017                                 case Token.REF:
1018                                 case Token.OUT:
1019                                         can_be_type = is_type = false;
1020                                         continue;
1021
1022                                 default:
1023                                         return Token.OPEN_PARENS;
1024                                 }
1025                         }
1026                 }
1027
1028                 public static bool IsValidIdentifier (string s)
1029                 {
1030                         if (s == null || s.Length == 0)
1031                                 return false;
1032
1033                         if (!is_identifier_start_character (s [0]))
1034                                 return false;
1035
1036                         for (int i = 1; i < s.Length; i ++)
1037                                 if (! is_identifier_part_character (s [i]))
1038                                         return false;
1039
1040                         return true;
1041                 }
1042
1043                 bool parse_less_than ()
1044                 {
1045                 start:
1046                         int the_token = token ();
1047                         if (the_token == Token.OPEN_BRACKET) {
1048                                 do {
1049                                         the_token = token ();
1050                                 } while (the_token != Token.CLOSE_BRACKET);
1051                                 the_token = token ();
1052                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1053                                 the_token = token ();
1054                         }
1055                         switch (the_token) {
1056                         case Token.IDENTIFIER:
1057                         case Token.OBJECT:
1058                         case Token.STRING:
1059                         case Token.BOOL:
1060                         case Token.DECIMAL:
1061                         case Token.FLOAT:
1062                         case Token.DOUBLE:
1063                         case Token.SBYTE:
1064                         case Token.BYTE:
1065                         case Token.SHORT:
1066                         case Token.USHORT:
1067                         case Token.INT:
1068                         case Token.UINT:
1069                         case Token.LONG:
1070                         case Token.ULONG:
1071                         case Token.CHAR:
1072                         case Token.VOID:
1073                                 break;
1074                         case Token.OP_GENERICS_GT:
1075                                 return true;
1076
1077                         default:
1078                                 return false;
1079                         }
1080                 again:
1081                         the_token = token ();
1082
1083                         if (the_token == Token.OP_GENERICS_GT)
1084                                 return true;
1085                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1086                                 goto start;
1087                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1088                                 goto again;
1089                         else if (the_token == Token.OP_GENERICS_LT) {
1090                                 if (!parse_less_than ())
1091                                         return false;
1092                                 goto again;
1093                         } else if (the_token == Token.OPEN_BRACKET) {
1094                         rank_specifiers:
1095                                 the_token = token ();
1096                                 if (the_token == Token.CLOSE_BRACKET)
1097                                         goto again;
1098                                 else if (the_token == Token.COMMA)
1099                                         goto rank_specifiers;
1100                                 return false;
1101                         }
1102
1103                         return false;
1104                 }
1105
1106                 bool parse_generic_dimension (out int dimension)
1107                 {
1108                         dimension = 1;
1109
1110                 again:
1111                         int the_token = token ();
1112                         if (the_token == Token.OP_GENERICS_GT)
1113                                 return true;
1114                         else if (the_token == Token.COMMA) {
1115                                 dimension++;
1116                                 goto again;
1117                         }
1118
1119                         return false;
1120                 }
1121
1122                 public int peek_token ()
1123                 {
1124                         int the_token;
1125
1126                         PushPosition ();
1127                         the_token = token ();
1128                         PopPosition ();
1129
1130                         return the_token;
1131                 }
1132
1133                 //
1134                 // Tonizes `?' using custom disambiguous rules to return one
1135                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1136                 //
1137                 // Tricky expression look like:
1138                 //
1139                 // Foo ? a = x ? b : c;
1140                 //
1141                 int TokenizePossibleNullableType ()
1142                 {
1143                         if (parsing_block == 0 || parsing_type > 0)
1144                                 return Token.INTERR_NULLABLE;
1145
1146                         int d = peek_char ();
1147                         if (d == '?') {
1148                                 get_char ();
1149                                 return Token.OP_COALESCING;
1150                         }
1151
1152                         switch (current_token) {
1153                         case Token.CLOSE_PARENS:
1154                         case Token.TRUE:
1155                         case Token.FALSE:
1156                         case Token.NULL:
1157                         case Token.LITERAL:
1158                                 return Token.INTERR;
1159                         }
1160
1161                         if (d != ' ') {
1162                                 if (d == ',' || d == ';' || d == '>')
1163                                         return Token.INTERR_NULLABLE;
1164                                 if (d == '*' || (d >= '0' && d <= '9'))
1165                                         return Token.INTERR;
1166                         }
1167
1168                         PushPosition ();
1169                         current_token = Token.NONE;
1170                         int next_token;
1171                         switch (xtoken ()) {
1172                         case Token.LITERAL:
1173                         case Token.TRUE:
1174                         case Token.FALSE:
1175                         case Token.NULL:
1176                         case Token.THIS:
1177                         case Token.NEW:
1178                                 next_token = Token.INTERR;
1179                                 break;
1180
1181                         case Token.SEMICOLON:
1182                         case Token.COMMA:
1183                         case Token.CLOSE_PARENS:
1184                         case Token.OPEN_BRACKET:
1185                         case Token.OP_GENERICS_GT:
1186                                 next_token = Token.INTERR_NULLABLE;
1187                                 break;
1188
1189                         default:
1190                                 next_token = -1;
1191                                 break;
1192                         }
1193
1194                         if (next_token == -1) {
1195                                 switch (xtoken ()) {
1196                                 case Token.COMMA:
1197                                 case Token.SEMICOLON:
1198                                 case Token.OPEN_BRACE:
1199                                 case Token.CLOSE_PARENS:
1200                                 case Token.IN:
1201                                         next_token = Token.INTERR_NULLABLE;
1202                                         break;
1203
1204                                 case Token.COLON:
1205                                         next_token = Token.INTERR;
1206                                         break;
1207
1208                                 default:
1209                                         int ntoken;
1210                                         int interrs = 1;
1211                                         int colons = 0;
1212                                         //
1213                                         // All shorcuts failed, do it hard way
1214                                         //
1215                                         while ((ntoken = xtoken ()) != Token.EOF) {
1216                                                 if (ntoken == Token.SEMICOLON)
1217                                                         break;
1218
1219                                                 if (ntoken == Token.COLON) {
1220                                                         if (++colons == interrs)
1221                                                                 break;
1222                                                         continue;
1223                                                 }
1224
1225                                                 if (ntoken == Token.INTERR) {
1226                                                         ++interrs;
1227                                                         continue;
1228                                                 }
1229                                         }
1230
1231                                         next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1232                                         break;
1233                                 }
1234                         }
1235
1236                         PopPosition ();
1237                         return next_token;
1238                 }
1239
1240                 bool decimal_digits (int c)
1241                 {
1242                         int d;
1243                         bool seen_digits = false;
1244
1245                         if (c != -1){
1246                                 if (number_pos == max_number_size)
1247                                         Error_NumericConstantTooLong ();
1248                                 number_builder [number_pos++] = (char) c;
1249                         }
1250
1251                         //
1252                         // We use peek_char2, because decimal_digits needs to do a
1253                         // 2-character look-ahead (5.ToString for example).
1254                         //
1255                         while ((d = peek_char2 ()) != -1){
1256                                 if (d >= '0' && d <= '9'){
1257                                         if (number_pos == max_number_size)
1258                                                 Error_NumericConstantTooLong ();
1259                                         number_builder [number_pos++] = (char) d;
1260                                         get_char ();
1261                                         seen_digits = true;
1262                                 } else
1263                                         break;
1264                         }
1265
1266                         return seen_digits;
1267                 }
1268
1269                 static bool is_hex (int e)
1270                 {
1271                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1272                 }
1273
1274                 static TypeCode real_type_suffix (int c)
1275                 {
1276                         switch (c){
1277                         case 'F': case 'f':
1278                                 return TypeCode.Single;
1279                         case 'D': case 'd':
1280                                 return TypeCode.Double;
1281                         case 'M': case 'm':
1282                                 return TypeCode.Decimal;
1283                         default:
1284                                 return TypeCode.Empty;
1285                         }
1286                 }
1287
1288                 int integer_type_suffix (ulong ul, int c)
1289                 {
1290                         bool is_unsigned = false;
1291                         bool is_long = false;
1292
1293                         if (c != -1){
1294                                 bool scanning = true;
1295                                 do {
1296                                         switch (c){
1297                                         case 'U': case 'u':
1298                                                 if (is_unsigned)
1299                                                         scanning = false;
1300                                                 is_unsigned = true;
1301                                                 get_char ();
1302                                                 break;
1303
1304                                         case 'l':
1305                                                 if (!is_unsigned){
1306                                                         //
1307                                                         // if we have not seen anything in between
1308                                                         // report this error
1309                                                         //
1310                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1311                                                 }
1312
1313                                                 goto case 'L';
1314
1315                                         case 'L':
1316                                                 if (is_long)
1317                                                         scanning = false;
1318                                                 is_long = true;
1319                                                 get_char ();
1320                                                 break;
1321
1322                                         default:
1323                                                 scanning = false;
1324                                                 break;
1325                                         }
1326                                         c = peek_char ();
1327                                 } while (scanning);
1328                         }
1329
1330                         if (is_long && is_unsigned){
1331                                 val = new ULongLiteral (ul, Location);
1332                                 return Token.LITERAL;
1333                         }
1334
1335                         if (is_unsigned){
1336                                 // uint if possible, or ulong else.
1337
1338                                 if ((ul & 0xffffffff00000000) == 0)
1339                                         val = new UIntLiteral ((uint) ul, Location);
1340                                 else
1341                                         val = new ULongLiteral (ul, Location);
1342                         } else if (is_long){
1343                                 // long if possible, ulong otherwise
1344                                 if ((ul & 0x8000000000000000) != 0)
1345                                         val = new ULongLiteral (ul, Location);
1346                                 else
1347                                         val = new LongLiteral ((long) ul, Location);
1348                         } else {
1349                                 // int, uint, long or ulong in that order
1350                                 if ((ul & 0xffffffff00000000) == 0){
1351                                         uint ui = (uint) ul;
1352
1353                                         if ((ui & 0x80000000) != 0)
1354                                                 val = new UIntLiteral (ui, Location);
1355                                         else
1356                                                 val = new IntLiteral ((int) ui, Location);
1357                                 } else {
1358                                         if ((ul & 0x8000000000000000) != 0)
1359                                                 val = new ULongLiteral (ul, Location);
1360                                         else
1361                                                 val = new LongLiteral ((long) ul, Location);
1362                                 }
1363                         }
1364                         return Token.LITERAL;
1365                 }
1366
1367                 //
1368                 // given `c' as the next char in the input decide whether
1369                 // we need to convert to a special type, and then choose
1370                 // the best representation for the integer
1371                 //
1372                 int adjust_int (int c)
1373                 {
1374                         try {
1375                                 if (number_pos > 9){
1376                                         ulong ul = (uint) (number_builder [0] - '0');
1377
1378                                         for (int i = 1; i < number_pos; i++){
1379                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1380                                         }
1381                                         return integer_type_suffix (ul, c);
1382                                 } else {
1383                                         uint ui = (uint) (number_builder [0] - '0');
1384
1385                                         for (int i = 1; i < number_pos; i++){
1386                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1387                                         }
1388                                         return integer_type_suffix (ui, c);
1389                                 }
1390                         } catch (OverflowException) {
1391                                 Error_NumericConstantTooLong ();
1392                                 val = new IntLiteral (0, Location);
1393                                 return Token.LITERAL;
1394                         }
1395                         catch (FormatException) {
1396                                 Report.Error (1013, Location, "Invalid number");
1397                                 val = new IntLiteral (0, Location);
1398                                 return Token.LITERAL;
1399                         }
1400                 }
1401
1402                 int adjust_real (TypeCode t)
1403                 {
1404                         string s = new String (number_builder, 0, number_pos);
1405                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1406
1407                         switch (t){
1408                         case TypeCode.Decimal:
1409                                 try {
1410                                         val = new DecimalLiteral (decimal.Parse (s, styles, csharp_format_info), Location);
1411                                 } catch (OverflowException) {
1412                                         val = new DecimalLiteral (0, Location);
1413                                         Report.Error (594, Location, error_details, "decimal");
1414                                 }
1415                                 break;
1416                         case TypeCode.Single:
1417                                 try {
1418                                         val = new FloatLiteral (float.Parse (s, styles, csharp_format_info), Location);
1419                                 } catch (OverflowException) {
1420                                         val = new FloatLiteral (0, Location);
1421                                         Report.Error (594, Location, error_details, "float");
1422                                 }
1423                                 break;
1424                         default:
1425                                 try {
1426                                         val = new DoubleLiteral (double.Parse (s, styles, csharp_format_info), Location);
1427                                 } catch (OverflowException) {
1428                                         val = new DoubleLiteral (0, Location);
1429                                         Report.Error (594, Location, error_details, "double");
1430                                 }
1431                                 break;
1432                         }
1433
1434                         return Token.LITERAL;
1435                 }
1436
1437                 int handle_hex ()
1438                 {
1439                         int d;
1440                         ulong ul;
1441
1442                         get_char ();
1443                         while ((d = peek_char ()) != -1){
1444                                 if (is_hex (d)){
1445                                         number_builder [number_pos++] = (char) d;
1446                                         get_char ();
1447                                 } else
1448                                         break;
1449                         }
1450
1451                         string s = new String (number_builder, 0, number_pos);
1452                         try {
1453                                 if (number_pos <= 8)
1454                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1455                                 else
1456                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1457                         } catch (OverflowException){
1458                                 Error_NumericConstantTooLong ();
1459                                 val = new IntLiteral (0, Location);
1460                                 return Token.LITERAL;
1461                         }
1462                         catch (FormatException) {
1463                                 Report.Error (1013, Location, "Invalid number");
1464                                 val = new IntLiteral (0, Location);
1465                                 return Token.LITERAL;
1466                         }
1467
1468                         return integer_type_suffix (ul, peek_char ());
1469                 }
1470
1471                 //
1472                 // Invoked if we know we have .digits or digits
1473                 //
1474                 int is_number (int c)
1475                 {
1476                         bool is_real = false;
1477
1478                         number_pos = 0;
1479
1480                         if (c >= '0' && c <= '9'){
1481                                 if (c == '0'){
1482                                         int peek = peek_char ();
1483
1484                                         if (peek == 'x' || peek == 'X')
1485                                                 return handle_hex ();
1486                                 }
1487                                 decimal_digits (c);
1488                                 c = get_char ();
1489                         }
1490
1491                         //
1492                         // We need to handle the case of
1493                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1494                         //
1495                         if (c == '.'){
1496                                 if (decimal_digits ('.')){
1497                                         is_real = true;
1498                                         c = get_char ();
1499                                 } else {
1500                                         putback ('.');
1501                                         number_pos--;
1502                                         return adjust_int (-1);
1503                                 }
1504                         }
1505
1506                         if (c == 'e' || c == 'E'){
1507                                 is_real = true;
1508                                 if (number_pos == max_number_size)
1509                                         Error_NumericConstantTooLong ();
1510                                 number_builder [number_pos++] = 'e';
1511                                 c = get_char ();
1512
1513                                 if (c == '+'){
1514                                         if (number_pos == max_number_size)
1515                                                 Error_NumericConstantTooLong ();
1516                                         number_builder [number_pos++] = '+';
1517                                         c = -1;
1518                                 } else if (c == '-') {
1519                                         if (number_pos == max_number_size)
1520                                                 Error_NumericConstantTooLong ();
1521                                         number_builder [number_pos++] = '-';
1522                                         c = -1;
1523                                 } else {
1524                                         if (number_pos == max_number_size)
1525                                                 Error_NumericConstantTooLong ();
1526                                         number_builder [number_pos++] = '+';
1527                                 }
1528
1529                                 decimal_digits (c);
1530                                 c = get_char ();
1531                         }
1532
1533                         var type = real_type_suffix (c);
1534                         if (type == TypeCode.Empty && !is_real){
1535                                 putback (c);
1536                                 return adjust_int (c);
1537                         }
1538
1539                         is_real = true;
1540
1541                         if (type == TypeCode.Empty){
1542                                 putback (c);
1543                         }
1544
1545                         if (is_real)
1546                                 return adjust_real (type);
1547
1548                         throw new Exception ("Is Number should never reach this point");
1549                 }
1550
1551                 //
1552                 // Accepts exactly count (4 or 8) hex, no more no less
1553                 //
1554                 int getHex (int count, out int surrogate, out bool error)
1555                 {
1556                         int i;
1557                         int total = 0;
1558                         int c;
1559                         int top = count != -1 ? count : 4;
1560
1561                         get_char ();
1562                         error = false;
1563                         surrogate = 0;
1564                         for (i = 0; i < top; i++){
1565                                 c = get_char ();
1566
1567                                 if (c >= '0' && c <= '9')
1568                                         c = (int) c - (int) '0';
1569                                 else if (c >= 'A' && c <= 'F')
1570                                         c = (int) c - (int) 'A' + 10;
1571                                 else if (c >= 'a' && c <= 'f')
1572                                         c = (int) c - (int) 'a' + 10;
1573                                 else {
1574                                         error = true;
1575                                         return 0;
1576                                 }
1577
1578                                 total = (total * 16) + c;
1579                                 if (count == -1){
1580                                         int p = peek_char ();
1581                                         if (p == -1)
1582                                                 break;
1583                                         if (!is_hex ((char)p))
1584                                                 break;
1585                                 }
1586                         }
1587
1588                         if (top == 8) {
1589                                 if (total > 0x0010FFFF) {
1590                                         error = true;
1591                                         return 0;
1592                                 }
1593
1594                                 if (total >= 0x00010000) {
1595                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1596                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1597                                 }
1598                         }
1599
1600                         return total;
1601                 }
1602
1603                 int escape (int c, out int surrogate)
1604                 {
1605                         bool error;
1606                         int d;
1607                         int v;
1608
1609                         d = peek_char ();
1610                         if (c != '\\') {
1611                                 surrogate = 0;
1612                                 return c;
1613                         }
1614
1615                         switch (d){
1616                         case 'a':
1617                                 v = '\a'; break;
1618                         case 'b':
1619                                 v = '\b'; break;
1620                         case 'n':
1621                                 v = '\n'; break;
1622                         case 't':
1623                                 v = '\t'; break;
1624                         case 'v':
1625                                 v = '\v'; break;
1626                         case 'r':
1627                                 v = '\r'; break;
1628                         case '\\':
1629                                 v = '\\'; break;
1630                         case 'f':
1631                                 v = '\f'; break;
1632                         case '0':
1633                                 v = 0; break;
1634                         case '"':
1635                                 v = '"'; break;
1636                         case '\'':
1637                                 v = '\''; break;
1638                         case 'x':
1639                                 v = getHex (-1, out surrogate, out error);
1640                                 if (error)
1641                                         goto default;
1642                                 return v;
1643                         case 'u':
1644                         case 'U':
1645                                 return EscapeUnicode (d, out surrogate);
1646                         default:
1647                                 surrogate = 0;
1648                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1649                                 return d;
1650                         }
1651
1652                         get_char ();
1653                         surrogate = 0;
1654                         return v;
1655                 }
1656
1657                 int EscapeUnicode (int ch, out int surrogate)
1658                 {
1659                         bool error;
1660                         if (ch == 'U') {
1661                                 ch = getHex (8, out surrogate, out error);
1662                         } else {
1663                                 ch = getHex (4, out surrogate, out error);
1664                         }
1665
1666                         if (error)
1667                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1668
1669                         return ch;
1670                 }
1671
1672                 int get_char ()
1673                 {
1674                         int x;
1675                         if (putback_char != -1) {
1676                                 x = putback_char;
1677                                 putback_char = -1;
1678                         } else
1679                                 x = reader.Read ();
1680                         if (x == '\n') {
1681                                 advance_line ();
1682                         } else {
1683                                 col++;
1684                         }
1685                         return x;
1686                 }
1687
1688                 void advance_line ()
1689                 {
1690                         line++;
1691                         ref_line++;
1692                         previous_col = col;
1693                         col = 0;
1694                 }
1695
1696                 int peek_char ()
1697                 {
1698                         if (putback_char == -1)
1699                                 putback_char = reader.Read ();
1700                         return putback_char;
1701                 }
1702
1703                 int peek_char2 ()
1704                 {
1705                         if (putback_char != -1)
1706                                 return putback_char;
1707                         return reader.Peek ();
1708                 }
1709
1710                 void putback (int c)
1711                 {
1712                         if (putback_char != -1){
1713                                 Console.WriteLine ("Col: " + col);
1714                                 Console.WriteLine ("Row: " + line);
1715                                 Console.WriteLine ("Name: " + ref_name.Name);
1716                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1717                                 throw new Exception ("This should not happen putback on putback");
1718                         }
1719                         if (c == '\n' || col == 0) {
1720                                 // It won't happen though.
1721                                 line--;
1722                                 ref_line--;
1723                                 col = previous_col;
1724                         }
1725                         else
1726                                 col--;
1727                         putback_char = c;
1728                 }
1729
1730                 public bool advance ()
1731                 {
1732                         return peek_char () != -1 || CompleteOnEOF;
1733                 }
1734
1735                 public Object Value {
1736                         get {
1737                                 return val;
1738                         }
1739                 }
1740
1741                 public Object value ()
1742                 {
1743                         return val;
1744                 }
1745
1746                 public int token ()
1747                 {
1748                         current_token = xtoken ();
1749                         return current_token;
1750                 }
1751
1752                 int TokenizePreprocessorIdentifier (out int c)
1753                 {
1754                         // skip over white space
1755                         do {
1756                                 c = get_char ();
1757                         } while (c == '\r' || c == ' ' || c == '\t');
1758
1759
1760                         int pos = 0;
1761                         while (c != -1 && c >= 'a' && c <= 'z') {
1762                                 id_builder[pos++] = (char) c;
1763                                 c = get_char ();
1764                                 if (c == '\\') {
1765                                         int peek = peek_char ();
1766                                         if (peek == 'U' || peek == 'u') {
1767                                                 int surrogate;
1768                                                 c = EscapeUnicode (c, out surrogate);
1769                                                 if (surrogate != 0) {
1770                                                         if (is_identifier_part_character ((char) c)) {
1771                                                                 id_builder[pos++] = (char) c;
1772                                                         }
1773                                                         c = surrogate;
1774                                                 }
1775                                         }
1776                                 }
1777                         }
1778
1779                         return pos;
1780                 }
1781
1782                 PreprocessorDirective get_cmd_arg (out string arg)
1783                 {
1784                         int c;
1785
1786                         tokens_seen = false;
1787                         arg = "";
1788
1789                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorIdentifier (out c));
1790
1791                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
1792                                 return cmd;
1793
1794                         // skip over white space
1795                         while (c == '\r' || c == ' ' || c == '\t')
1796                                 c = get_char ();
1797
1798                         static_cmd_arg.Length = 0;
1799                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
1800
1801                         while (c != -1 && c != '\n' && c != '\r') {
1802                                 if (c == '\\' && has_identifier_argument >= 0) {
1803                                         if (has_identifier_argument != 0) {
1804                                                 has_identifier_argument = 1;
1805
1806                                                 int peek = peek_char ();
1807                                                 if (peek == 'U' || peek == 'u') {
1808                                                         int surrogate;
1809                                                         c = EscapeUnicode (c, out surrogate);
1810                                                         if (surrogate != 0) {
1811                                                                 if (is_identifier_part_character ((char) c))
1812                                                                         static_cmd_arg.Append ((char) c);
1813                                                                 c = surrogate;
1814                                                         }
1815                                                 }
1816                                         } else {
1817                                                 has_identifier_argument = -1;
1818                                         }
1819                                 }
1820                                 static_cmd_arg.Append ((char) c);
1821                                 c = get_char ();
1822                         }
1823
1824                         if (static_cmd_arg.Length != 0) {
1825                                 arg = static_cmd_arg.ToString ();
1826
1827                                 // Eat any trailing whitespaces and single-line comments
1828                                 if (arg.IndexOf ("//") != -1) {
1829                                         arg = arg.Substring (0, arg.IndexOf ("//"));
1830                                 }
1831
1832                                 arg = arg.Trim (simple_whitespaces);
1833                         }
1834
1835                         return cmd;
1836                 }
1837
1838                 //
1839                 // Handles the #line directive
1840                 //
1841                 bool PreProcessLine (string arg)
1842                 {
1843                         if (arg.Length == 0)
1844                                 return false;
1845
1846                         if (arg == "default"){
1847                                 ref_line = line;
1848                                 ref_name = file_name;
1849                                 hidden = false;
1850                                 Location.Push (file_name, ref_name);
1851                                 return true;
1852                         } else if (arg == "hidden"){
1853                                 hidden = true;
1854                                 return true;
1855                         }
1856
1857                         try {
1858                                 int pos;
1859
1860                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1861                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1862                                         pos++;
1863
1864                                         char [] quotes = { '\"' };
1865
1866                                         string name = arg.Substring (pos). Trim (quotes);
1867                                         ref_name = Location.LookupFile (file_name, name);
1868                                         file_name.AddFile (ref_name);
1869                                         hidden = false;
1870                                         Location.Push (file_name, ref_name);
1871                                 } else {
1872                                         ref_line = System.Int32.Parse (arg);
1873                                         hidden = false;
1874                                 }
1875                         } catch {
1876                                 return false;
1877                         }
1878
1879                         return true;
1880                 }
1881
1882                 //
1883                 // Handles #define and #undef
1884                 //
1885                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
1886                 {
1887                         if (ident.Length == 0 || ident == "true" || ident == "false"){
1888                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
1889                                 return;
1890                         }
1891
1892                         if (ident.IndexOfAny (simple_whitespaces) != -1){
1893                                 Error_EndLineExpected ();
1894                                 return;
1895                         }
1896
1897                         if (!is_identifier_start_character (ident [0]))
1898                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
1899
1900                         foreach (char c in ident.Substring (1)){
1901                                 if (!is_identifier_part_character (c)){
1902                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
1903                                         return;
1904                                 }
1905                         }
1906
1907                         if (!caller_is_taking)
1908                                 return;
1909
1910                         if (is_define) {
1911                                 //
1912                                 // #define ident
1913                                 //
1914                                 if (RootContext.IsConditionalDefined (ident))
1915                                         return;
1916
1917                                 file_name.AddDefine (ident);
1918                         } else {
1919                                 //
1920                                 // #undef ident
1921                                 //
1922                                 file_name.AddUndefine (ident);
1923                         }
1924                 }
1925
1926                 byte read_hex (out bool error)
1927                 {
1928                         int total;
1929                         int c = get_char ();
1930
1931                         if ((c >= '0') && (c <= '9'))
1932                                 total = (int) c - (int) '0';
1933                         else if ((c >= 'A') && (c <= 'F'))
1934                                 total = (int) c - (int) 'A' + 10;
1935                         else if ((c >= 'a') && (c <= 'f'))
1936                                 total = (int) c - (int) 'a' + 10;
1937                         else {
1938                                 error = true;
1939                                 return 0;
1940                         }
1941
1942                         total *= 16;
1943                         c = get_char ();
1944
1945                         if ((c >= '0') && (c <= '9'))
1946                                 total += (int) c - (int) '0';
1947                         else if ((c >= 'A') && (c <= 'F'))
1948                                 total += (int) c - (int) 'A' + 10;
1949                         else if ((c >= 'a') && (c <= 'f'))
1950                                 total += (int) c - (int) 'a' + 10;
1951                         else {
1952                                 error = true;
1953                                 return 0;
1954                         }
1955
1956                         error = false;
1957                         return (byte) total;
1958                 }
1959
1960                 //
1961                 // Parses #pragma checksum
1962                 //
1963                 bool ParsePragmaChecksum ()
1964                 {
1965                         //
1966                         // The syntax is ` "foo.txt" "{guid}" "hash"'
1967                         //
1968                         int c = get_char ();
1969
1970                         if (c != '"')
1971                                 return false;
1972
1973                         string_builder.Length = 0;
1974                         while (c != -1 && c != '\n') {
1975                                 c = get_char ();
1976                                 if (c == '"') {
1977                                         c = get_char ();
1978                                         break;
1979                                 }
1980
1981                                 string_builder.Append ((char) c);
1982                         }
1983
1984                         if (string_builder.Length == 0) {
1985                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
1986                         }
1987
1988                         // TODO: Any white-spaces count
1989                         if (c != ' ')
1990                                 return false;
1991
1992                         SourceFile file = Location.LookupFile (file_name, string_builder.ToString ());
1993
1994                         if (get_char () != '"' || get_char () != '{')
1995                                 return false;
1996
1997                         bool error;
1998                         byte[] guid_bytes = new byte [16];
1999                         int i = 0;
2000
2001                         for (; i < 4; i++) {
2002                                 guid_bytes [i] = read_hex (out error);
2003                                 if (error)
2004                                         return false;
2005                         }
2006
2007                         if (get_char () != '-')
2008                                 return false;
2009
2010                         for (; i < 10; i++) {
2011                                 guid_bytes [i] = read_hex (out error);
2012                                 if (error)
2013                                         return false;
2014
2015                                 guid_bytes [i++] = read_hex (out error);
2016                                 if (error)
2017                                         return false;
2018
2019                                 if (get_char () != '-')
2020                                         return false;
2021                         }
2022
2023                         for (; i < 16; i++) {
2024                                 guid_bytes [i] = read_hex (out error);
2025                                 if (error)
2026                                         return false;
2027                         }
2028
2029                         if (get_char () != '}' || get_char () != '"')
2030                                 return false;
2031
2032                         // TODO: Any white-spaces count
2033                         c = get_char ();
2034                         if (c != ' ')
2035                                 return false;
2036
2037                         if (get_char () != '"')
2038                                 return false;
2039
2040                         // Any length of checksum
2041                         List<byte> checksum_bytes = new List<byte> (16);
2042
2043                         c = peek_char ();
2044                         while (c != '"' && c != -1) {
2045                                 checksum_bytes.Add (read_hex (out error));
2046                                 if (error)
2047                                         return false;
2048
2049                                 c = peek_char ();
2050                         }
2051
2052                         if (c == '/') {
2053                                 ReadSingleLineComment ();
2054                         } else if (get_char () != '"') {
2055                                 return false;
2056                         }
2057
2058                         file.SetChecksum (guid_bytes, checksum_bytes.ToArray ());
2059                         ref_name.AutoGenerated = true;
2060                         return true;
2061                 }
2062
2063                 bool IsTokenIdentifierEqual (char[] identifier)
2064                 {
2065                         for (int i = 0; i < identifier.Length; ++i) {
2066                                 if (identifier[i] != id_builder[i])
2067                                         return false;
2068                         }
2069
2070                         return true;
2071                 }
2072
2073                 int TokenizePragmaNumber (ref int c)
2074                 {
2075                         number_pos = 0;
2076
2077                         int number;
2078
2079                         if (c >= '0' && c <= '9') {
2080                                 decimal_digits (c);
2081                                 uint ui = (uint) (number_builder[0] - '0');
2082
2083                                 try {
2084                                         for (int i = 1; i < number_pos; i++) {
2085                                                 ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2086                                         }
2087
2088                                         number = (int) ui;
2089                                 } catch (OverflowException) {
2090                                         Error_NumericConstantTooLong ();
2091                                         number = -1;
2092                                 }
2093
2094
2095                                 c = get_char ();
2096
2097                                 // skip over white space
2098                                 while (c == '\r' || c == ' ' || c == '\t')
2099                                         c = get_char ();
2100
2101                                 if (c == ',') {
2102                                         c = get_char ();
2103                                 }
2104
2105                                 // skip over white space
2106                                 while (c == '\r' || c == ' ' || c == '\t')
2107                                         c = get_char ();
2108                         } else {
2109                                 number = -1;
2110                                 if (c == '/') {
2111                                         ReadSingleLineComment ();
2112                                 } else {
2113                                         Report.Warning (1692, 1, Location, "Invalid number");
2114
2115                                         // Read everything till the end of the line or file
2116                                         do {
2117                                                 c = get_char ();
2118                                         } while (c != -1 && c != '\n');
2119                                 }
2120                         }
2121
2122                         return number;
2123                 }
2124
2125                 void ReadSingleLineComment ()
2126                 {
2127                         if (peek_char () != '/')
2128                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2129
2130                         // Read everything till the end of the line or file
2131                         int c;
2132                         do {
2133                                 c = get_char ();
2134                         } while (c != -1 && c != '\n');
2135                 }
2136
2137                 /// <summary>
2138                 /// Handles #pragma directive
2139                 /// </summary>
2140                 void ParsePragmaDirective (string arg)
2141                 {
2142                         int c;
2143                         int length = TokenizePreprocessorIdentifier (out c);
2144                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2145                                 length = TokenizePreprocessorIdentifier (out c);
2146
2147                                 //
2148                                 // #pragma warning disable
2149                                 // #pragma warning restore
2150                                 //
2151                                 if (length == pragma_warning_disable.Length) {
2152                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2153                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2154                                                 // skip over white space
2155                                                 while (c == '\r' || c == ' ' || c == '\t')
2156                                                         c = get_char ();
2157
2158                                                 var loc = Location;
2159
2160                                                 if (c == '\n' || c == '/') {
2161                                                         if (c == '/')
2162                                                                 ReadSingleLineComment ();
2163
2164                                                         //
2165                                                         // Disable/Restore all warnings
2166                                                         //
2167                                                         if (disable) {
2168                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2169                                                         } else {
2170                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2171                                                         }
2172                                                 } else {
2173                                                         //
2174                                                         // Disable/Restore a warning or group of warnings
2175                                                         //
2176                                                         int code;
2177                                                         do {
2178                                                                 code = TokenizePragmaNumber (ref c);
2179                                                                 if (code > 0) {
2180                                                                         if (disable) {
2181                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, Report);
2182                                                                         } else {
2183                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, Report);
2184                                                                         }
2185                                                                 }
2186                                                         } while (code >= 0 && c != '\n');
2187                                                 }
2188
2189                                                 return;
2190                                         }
2191                                 }
2192
2193                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2194                                 return;
2195                         }
2196
2197                         //
2198                         // #pragma checksum
2199                         //
2200                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2201                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2202                                         Report.Warning (1695, 1, Location,
2203                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2204                                 }
2205
2206                                 return;
2207                         }
2208
2209                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2210                 }
2211
2212                 bool eval_val (string s)
2213                 {
2214                         if (s == "true")
2215                                 return true;
2216                         if (s == "false")
2217                                 return false;
2218
2219                         return file_name.IsConditionalDefined (s);
2220                 }
2221
2222                 bool pp_primary (ref string s)
2223                 {
2224                         s = s.Trim ();
2225                         int len = s.Length;
2226
2227                         if (len > 0){
2228                                 char c = s [0];
2229
2230                                 if (c == '('){
2231                                         s = s.Substring (1);
2232                                         bool val = pp_expr (ref s, false);
2233                                         if (s.Length > 0 && s [0] == ')'){
2234                                                 s = s.Substring (1);
2235                                                 return val;
2236                                         }
2237                                         Error_InvalidDirective ();
2238                                         return false;
2239                                 }
2240
2241                                 if (is_identifier_start_character (c)){
2242                                         int j = 1;
2243
2244                                         while (j < len){
2245                                                 c = s [j];
2246
2247                                                 if (is_identifier_part_character (c)){
2248                                                         j++;
2249                                                         continue;
2250                                                 }
2251                                                 bool v = eval_val (s.Substring (0, j));
2252                                                 s = s.Substring (j);
2253                                                 return v;
2254                                         }
2255                                         bool vv = eval_val (s);
2256                                         s = "";
2257                                         return vv;
2258                                 }
2259                         }
2260                         Error_InvalidDirective ();
2261                         return false;
2262                 }
2263
2264                 bool pp_unary (ref string s)
2265                 {
2266                         s = s.Trim ();
2267                         int len = s.Length;
2268
2269                         if (len > 0){
2270                                 if (s [0] == '!'){
2271                                         if (len > 1 && s [1] == '='){
2272                                                 Error_InvalidDirective ();
2273                                                 return false;
2274                                         }
2275                                         s = s.Substring (1);
2276                                         return ! pp_primary (ref s);
2277                                 } else
2278                                         return pp_primary (ref s);
2279                         } else {
2280                                 Error_InvalidDirective ();
2281                                 return false;
2282                         }
2283                 }
2284
2285                 bool pp_eq (ref string s)
2286                 {
2287                         bool va = pp_unary (ref s);
2288
2289                         s = s.Trim ();
2290                         int len = s.Length;
2291                         if (len > 0){
2292                                 if (s [0] == '='){
2293                                         if (len > 2 && s [1] == '='){
2294                                                 s = s.Substring (2);
2295                                                 return va == pp_unary (ref s);
2296                                         } else {
2297                                                 Error_InvalidDirective ();
2298                                                 return false;
2299                                         }
2300                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2301                                         s = s.Substring (2);
2302
2303                                         return va != pp_unary (ref s);
2304
2305                                 }
2306                         }
2307
2308                         return va;
2309
2310                 }
2311
2312                 bool pp_and (ref string s)
2313                 {
2314                         bool va = pp_eq (ref s);
2315
2316                         s = s.Trim ();
2317                         int len = s.Length;
2318                         if (len > 0){
2319                                 if (s [0] == '&'){
2320                                         if (len > 2 && s [1] == '&'){
2321                                                 s = s.Substring (2);
2322                                                 return (va & pp_and (ref s));
2323                                         } else {
2324                                                 Error_InvalidDirective ();
2325                                                 return false;
2326                                         }
2327                                 }
2328                         }
2329                         return va;
2330                 }
2331
2332                 //
2333                 // Evaluates an expression for `#if' or `#elif'
2334                 //
2335                 bool pp_expr (ref string s, bool isTerm)
2336                 {
2337                         bool va = pp_and (ref s);
2338                         s = s.Trim ();
2339                         int len = s.Length;
2340                         if (len > 0){
2341                                 char c = s [0];
2342
2343                                 if (c == '|'){
2344                                         if (len > 2 && s [1] == '|'){
2345                                                 s = s.Substring (2);
2346                                                 return va | pp_expr (ref s, isTerm);
2347                                         } else {
2348                                                 Error_InvalidDirective ();
2349                                                 return false;
2350                                         }
2351                                 }
2352                                 if (isTerm) {
2353                                         Error_EndLineExpected ();
2354                                         return false;
2355                                 }
2356                         }
2357
2358                         return va;
2359                 }
2360
2361                 bool eval (string s)
2362                 {
2363                         bool v = pp_expr (ref s, true);
2364                         s = s.Trim ();
2365                         if (s.Length != 0){
2366                                 return false;
2367                         }
2368
2369                         return v;
2370                 }
2371
2372                 void Error_NumericConstantTooLong ()
2373                 {
2374                         Report.Error (1021, Location, "Integral constant is too large");
2375                 }
2376
2377                 void Error_InvalidDirective ()
2378                 {
2379                         Report.Error (1517, Location, "Invalid preprocessor directive");
2380                 }
2381
2382                 void Error_UnexpectedDirective (string extra)
2383                 {
2384                         Report.Error (
2385                                 1028, Location,
2386                                 "Unexpected processor directive ({0})", extra);
2387                 }
2388
2389                 void Error_TokensSeen ()
2390                 {
2391                         Report.Error (1032, Location,
2392                                 "Cannot define or undefine preprocessor symbols after first token in file");
2393                 }
2394
2395                 void Eror_WrongPreprocessorLocation ()
2396                 {
2397                         Report.Error (1040, Location,
2398                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2399                 }
2400
2401                 void Error_EndLineExpected ()
2402                 {
2403                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2404                 }
2405
2406                 //
2407                 // if true, then the code continues processing the code
2408                 // if false, the code stays in a loop until another directive is
2409                 // reached.
2410                 // When caller_is_taking is false we ignore all directives except the ones
2411                 // which can help us to identify where the #if block ends
2412                 bool ParsePreprocessingDirective (bool caller_is_taking)
2413                 {
2414                         string arg;
2415                         bool region_directive = false;
2416
2417                         var directive = get_cmd_arg (out arg);
2418
2419                         //
2420                         // The first group of pre-processing instructions is always processed
2421                         //
2422                         switch (directive) {
2423                         case PreprocessorDirective.Region:
2424                                 region_directive = true;
2425                                 arg = "true";
2426                                 goto case PreprocessorDirective.If;
2427
2428                         case PreprocessorDirective.Endregion:
2429                                 if (ifstack == null || ifstack.Count == 0){
2430                                         Error_UnexpectedDirective ("no #region for this #endregion");
2431                                         return true;
2432                                 }
2433                                 int pop = ifstack.Pop ();
2434
2435                                 if ((pop & REGION) == 0)
2436                                         Report.Error (1027, Location, "Expected `#endif' directive");
2437
2438                                 return caller_is_taking;
2439
2440                         case PreprocessorDirective.If:
2441                                 if (ifstack == null)
2442                                         ifstack = new Stack<int> (2);
2443
2444                                 int flags = region_directive ? REGION : 0;
2445                                 if (ifstack.Count == 0){
2446                                         flags |= PARENT_TAKING;
2447                                 } else {
2448                                         int state = ifstack.Peek ();
2449                                         if ((state & TAKING) != 0) {
2450                                                 flags |= PARENT_TAKING;
2451                                         }
2452                                 }
2453
2454                                 if (caller_is_taking && eval (arg)) {
2455                                         ifstack.Push (flags | TAKING);
2456                                         return true;
2457                                 }
2458                                 ifstack.Push (flags);
2459                                 return false;
2460
2461                         case PreprocessorDirective.Endif:
2462                                 if (ifstack == null || ifstack.Count == 0){
2463                                         Error_UnexpectedDirective ("no #if for this #endif");
2464                                         return true;
2465                                 } else {
2466                                         pop = ifstack.Pop ();
2467
2468                                         if ((pop & REGION) != 0)
2469                                                 Report.Error (1038, Location, "#endregion directive expected");
2470
2471                                         if (arg.Length != 0) {
2472                                                 Error_EndLineExpected ();
2473                                         }
2474
2475                                         if (ifstack.Count == 0)
2476                                                 return true;
2477
2478                                         int state = ifstack.Peek ();
2479                                         return (state & TAKING) != 0;
2480                                 }
2481
2482                         case PreprocessorDirective.Elif:
2483                                 if (ifstack == null || ifstack.Count == 0){
2484                                         Error_UnexpectedDirective ("no #if for this #elif");
2485                                         return true;
2486                                 } else {
2487                                         int state = ifstack.Pop ();
2488
2489                                         if ((state & REGION) != 0) {
2490                                                 Report.Error (1038, Location, "#endregion directive expected");
2491                                                 return true;
2492                                         }
2493
2494                                         if ((state & ELSE_SEEN) != 0){
2495                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2496                                                 return true;
2497                                         }
2498
2499                                         if ((state & TAKING) != 0) {
2500                                                 ifstack.Push (0);
2501                                                 return false;
2502                                         }
2503
2504                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2505                                                 ifstack.Push (state | TAKING);
2506                                                 return true;
2507                                         }
2508
2509                                         ifstack.Push (state);
2510                                         return false;
2511                                 }
2512
2513                         case PreprocessorDirective.Else:
2514                                 if (ifstack == null || ifstack.Count == 0){
2515                                         Error_UnexpectedDirective ("no #if for this #else");
2516                                         return true;
2517                                 } else {
2518                                         int state = ifstack.Peek ();
2519
2520                                         if ((state & REGION) != 0) {
2521                                                 Report.Error (1038, Location, "#endregion directive expected");
2522                                                 return true;
2523                                         }
2524
2525                                         if ((state & ELSE_SEEN) != 0){
2526                                                 Error_UnexpectedDirective ("#else within #else");
2527                                                 return true;
2528                                         }
2529
2530                                         ifstack.Pop ();
2531
2532                                         if (arg.Length != 0) {
2533                                                 Error_EndLineExpected ();
2534                                                 return true;
2535                                         }
2536
2537                                         bool ret = false;
2538                                         if ((state & PARENT_TAKING) != 0) {
2539                                                 ret = (state & TAKING) == 0;
2540
2541                                                 if (ret)
2542                                                         state |= TAKING;
2543                                                 else
2544                                                         state &= ~TAKING;
2545                                         }
2546
2547                                         ifstack.Push (state | ELSE_SEEN);
2548
2549                                         return ret;
2550                                 }
2551                         case PreprocessorDirective.Define:
2552                                 if (any_token_seen){
2553                                         Error_TokensSeen ();
2554                                         return caller_is_taking;
2555                                 }
2556                                 PreProcessDefinition (true, arg, caller_is_taking);
2557                                 return caller_is_taking;
2558
2559                         case PreprocessorDirective.Undef:
2560                                 if (any_token_seen){
2561                                         Error_TokensSeen ();
2562                                         return caller_is_taking;
2563                                 }
2564                                 PreProcessDefinition (false, arg, caller_is_taking);
2565                                 return caller_is_taking;
2566
2567                         case PreprocessorDirective.Invalid:
2568                                 Report.Error (1024, Location, "Wrong preprocessor directive");
2569                                 return true;
2570                         }
2571
2572                         //
2573                         // These are only processed if we are in a `taking' block
2574                         //
2575                         if (!caller_is_taking)
2576                                 return false;
2577
2578                         switch (directive){
2579                         case PreprocessorDirective.Error:
2580                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2581                                 return true;
2582
2583                         case PreprocessorDirective.Warning:
2584                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2585                                 return true;
2586
2587                         case PreprocessorDirective.Pragma:
2588                                 if (RootContext.Version == LanguageVersion.ISO_1) {
2589                                         Report.FeatureIsNotAvailable (Location, "#pragma");
2590                                 }
2591
2592                                 ParsePragmaDirective (arg);
2593                                 return true;
2594
2595                         case PreprocessorDirective.Line:
2596                                 if (!PreProcessLine (arg))
2597                                         Report.Error (
2598                                                 1576, Location,
2599                                                 "The line number specified for #line directive is missing or invalid");
2600                                 return caller_is_taking;
2601                         }
2602
2603                         throw new NotImplementedException (directive.ToString ());
2604                 }
2605
2606                 private int consume_string (bool quoted)
2607                 {
2608                         int c;
2609                         string_builder.Length = 0;
2610
2611                         while (true){
2612                                 c = get_char ();
2613                                 if (c == '"') {
2614                                         if (quoted && peek_char () == '"') {
2615                                                 string_builder.Append ((char) c);
2616                                                 get_char ();
2617                                                 continue;
2618                                         }
2619
2620                                         val = new StringLiteral (string_builder.ToString (), Location);
2621                                         return Token.LITERAL;
2622                                 }
2623
2624                                 if (c == '\n') {
2625                                         if (!quoted)
2626                                                 Report.Error (1010, Location, "Newline in constant");
2627                                 } else if (c == '\\' && !quoted) {
2628                                         int surrogate;
2629                                         c = escape (c, out surrogate);
2630                                         if (c == -1)
2631                                                 return Token.ERROR;
2632                                         if (surrogate != 0) {
2633                                                 string_builder.Append ((char) c);
2634                                                 c = surrogate;
2635                                         }
2636                                 } else if (c == -1) {
2637                                         Report.Error (1039, Location, "Unterminated string literal");
2638                                         return Token.EOF;
2639                                 }
2640
2641                                 string_builder.Append ((char) c);
2642                         }
2643                 }
2644
2645                 private int consume_identifier (int s)
2646                 {
2647                         int res = consume_identifier (s, false);
2648
2649                         if (doc_state == XmlCommentState.Allowed)
2650                                 doc_state = XmlCommentState.NotAllowed;
2651
2652                         return res;
2653                 }
2654
2655                 int consume_identifier (int c, bool quoted)
2656                 {
2657                         //
2658                         // This method is very performance sensitive. It accounts
2659                         // for approximately 25% of all parser time
2660                         //
2661
2662                         int pos = 0;
2663                         int column = col;
2664
2665                         if (c == '\\') {
2666                                 int surrogate;
2667                                 c = escape (c, out surrogate);
2668                                 if (surrogate != 0) {
2669                                         id_builder [pos++] = (char) c;
2670                                         c = surrogate;
2671                                 }
2672                         }
2673
2674                         id_builder [pos++] = (char) c;
2675
2676                         try {
2677                                 while (true) {
2678                                         c = reader.Read ();
2679
2680                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
2681                                                 id_builder [pos++] = (char) c;
2682                                                 continue;
2683                                         }
2684
2685                                         if (c < 0x80) {
2686                                                 if (c == '\\') {
2687                                                         int surrogate;
2688                                                         c = escape (c, out surrogate);
2689                                                         if (surrogate != 0) {
2690                                                                 if (is_identifier_part_character ((char) c))
2691                                                                         id_builder[pos++] = (char) c;
2692                                                                 c = surrogate;
2693                                                         }
2694
2695                                                         continue;
2696                                                 }
2697                                         } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
2698                                                 id_builder [pos++] = (char) c;
2699                                                 continue;
2700                                         }
2701
2702                                         putback_char = c;
2703                                         break;
2704                                 }
2705                         } catch (IndexOutOfRangeException) {
2706                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2707                                 --pos;
2708                                 col += pos;
2709                         }
2710
2711                         col += pos - 1;
2712
2713                         //
2714                         // Optimization: avoids doing the keyword lookup
2715                         // on uppercase letters
2716                         //
2717                         if (id_builder [0] >= '_' && !quoted) {
2718                                 int keyword = GetKeyword (id_builder, pos);
2719                                 if (keyword != -1) {
2720                                         val = LocatedToken.Create (null, ref_line, column);
2721                                         return keyword;
2722                                 }
2723                         }
2724
2725                         //
2726                         // Keep identifiers in an array of hashtables to avoid needless
2727                         // allocations
2728                         //
2729                         var identifiers_group = identifiers [pos];
2730                         string s;
2731                         if (identifiers_group != null) {
2732                                 if (identifiers_group.TryGetValue (id_builder, out s)) {
2733                                         val = LocatedToken.Create (s, ref_line, column);
2734                                         if (quoted)
2735                                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2736                                         return Token.IDENTIFIER;
2737                                 }
2738                         } else {
2739                                 // TODO: this should be number of files dependant
2740                                 // corlib compilation peaks at 1000 and System.Core at 150
2741                                 int capacity = pos > 20 ? 10 : 100;
2742                                 identifiers_group = new Dictionary<char[],string> (capacity, new IdentifiersComparer (pos));
2743                                 identifiers [pos] = identifiers_group;
2744                         }
2745
2746                         char [] chars = new char [pos];
2747                         Array.Copy (id_builder, chars, pos);
2748
2749                         s = new string (id_builder, 0, pos);
2750                         identifiers_group.Add (chars, s);
2751
2752                         val = LocatedToken.Create (s, ref_line, column);
2753                         if (quoted)
2754                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2755
2756                         return Token.IDENTIFIER;
2757                 }
2758
2759                 public int xtoken ()
2760                 {
2761                         int d, c;
2762
2763                         // Whether we have seen comments on the current line
2764                         bool comments_seen = false;
2765                         while ((c = get_char ()) != -1) {
2766                                 switch (c) {
2767                                 case '\t':
2768                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
2769                                         continue;
2770
2771                                 case ' ':
2772                                 case '\f':
2773                                 case '\v':
2774                                 case 0xa0:
2775                                 case 0:
2776                                 case 0xFEFF:    // Ignore BOM anywhere in the file
2777                                         continue;
2778
2779 /*                              This is required for compatibility with .NET
2780                                 case 0xEF:
2781                                         if (peek_char () == 0xBB) {
2782                                                 PushPosition ();
2783                                                 get_char ();
2784                                                 if (get_char () == 0xBF)
2785                                                         continue;
2786                                                 PopPosition ();
2787                                         }
2788                                         break;
2789 */
2790                                 case '\r':
2791                                         if (peek_char () != '\n')
2792                                                 advance_line ();
2793                                         else
2794                                                 get_char ();
2795
2796                                         any_token_seen |= tokens_seen;
2797                                         tokens_seen = false;
2798                                         comments_seen = false;
2799                                         continue;
2800
2801                                 case '\\':
2802                                         tokens_seen = true;
2803                                         return consume_identifier (c);
2804
2805                                 case '{':
2806                                         val = LocatedToken.Create (ref_line, col);
2807                                         return Token.OPEN_BRACE;
2808                                 case '}':
2809                                         val = LocatedToken.Create (ref_line, col);
2810                                         return Token.CLOSE_BRACE;
2811                                 case '[':
2812                                         // To block doccomment inside attribute declaration.
2813                                         if (doc_state == XmlCommentState.Allowed)
2814                                                 doc_state = XmlCommentState.NotAllowed;
2815
2816                                         val = LocatedToken.Create (ref_line, col);
2817
2818                                         if (parsing_block == 0 || lambda_arguments_parsing)
2819                                                 return Token.OPEN_BRACKET;
2820
2821                                         int next = peek_char ();
2822                                         switch (next) {
2823                                         case ']':
2824                                         case ',':
2825                                                 return Token.OPEN_BRACKET;
2826
2827                                         case ' ':
2828                                         case '\f':
2829                                         case '\v':
2830                                         case '\r':
2831                                         case '\n':
2832                                         case '/':
2833                                                 next = peek_token ();
2834                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
2835                                                         return Token.OPEN_BRACKET;
2836
2837                                                 return Token.OPEN_BRACKET_EXPR;
2838                                         default:
2839                                                 return Token.OPEN_BRACKET_EXPR;
2840                                         }
2841                                 case ']':
2842                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2843                                         return Token.CLOSE_BRACKET;
2844                                 case '(':
2845                                         val = LocatedToken.Create (ref_line, col);
2846                                         //
2847                                         // An expression versions of parens can appear in block context only
2848                                         //
2849                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
2850
2851                                                 //
2852                                                 // Optmize most common case where we know that parens
2853                                                 // is not special
2854                                                 //
2855                                                 switch (current_token) {
2856                                                 case Token.IDENTIFIER:
2857                                                 case Token.IF:
2858                                                 case Token.FOR:
2859                                                 case Token.FOREACH:
2860                                                 case Token.TYPEOF:
2861                                                 case Token.WHILE:
2862                                                 case Token.USING:
2863                                                 case Token.DEFAULT:
2864                                                 case Token.DELEGATE:
2865                                                 case Token.OP_GENERICS_GT:
2866                                                         return Token.OPEN_PARENS;
2867                                                 }
2868
2869                                                 // Optimize using peek
2870                                                 int xx = peek_char ();
2871                                                 switch (xx) {
2872                                                 case '(':
2873                                                 case '\'':
2874                                                 case '"':
2875                                                 case '0':
2876                                                 case '1':
2877                                                         return Token.OPEN_PARENS;
2878                                                 }
2879
2880                                                 lambda_arguments_parsing = true;
2881                                                 PushPosition ();
2882                                                 d = TokenizeOpenParens ();
2883                                                 PopPosition ();
2884                                                 lambda_arguments_parsing = false;
2885                                                 return d;
2886                                         }
2887
2888                                         return Token.OPEN_PARENS;
2889                                 case ')':
2890                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2891                                         return Token.CLOSE_PARENS;
2892                                 case ',':
2893                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2894                                         return Token.COMMA;
2895                                 case ';':
2896                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2897                                         return Token.SEMICOLON;
2898                                 case '~':
2899                                         val = LocatedToken.Create (ref_line, col);
2900                                         return Token.TILDE;
2901                                 case '?':
2902                                         val = LocatedToken.Create (ref_line, col);
2903                                         return TokenizePossibleNullableType ();
2904                                 case '<':
2905                                         val = LocatedToken.Create (ref_line, col);
2906                                         if (parsing_generic_less_than++ > 0)
2907                                                 return Token.OP_GENERICS_LT;
2908
2909                                         return TokenizeLessThan ();
2910
2911                                 case '>':
2912                                         val = LocatedToken.Create (ref_line, col);
2913                                         d = peek_char ();
2914
2915                                         if (d == '='){
2916                                                 get_char ();
2917                                                 return Token.OP_GE;
2918                                         }
2919
2920                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
2921                                                 parsing_generic_less_than--;
2922                                                 return Token.OP_GENERICS_GT;
2923                                         }
2924
2925                                         if (d == '>') {
2926                                                 get_char ();
2927                                                 d = peek_char ();
2928
2929                                                 if (d == '=') {
2930                                                         get_char ();
2931                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
2932                                                 }
2933                                                 return Token.OP_SHIFT_RIGHT;
2934                                         }
2935
2936                                         return Token.OP_GT;
2937
2938                                 case '+':
2939                                         val = LocatedToken.Create (ref_line, col);
2940                                         d = peek_char ();
2941                                         if (d == '+') {
2942                                                 d = Token.OP_INC;
2943                                         } else if (d == '=') {
2944                                                 d = Token.OP_ADD_ASSIGN;
2945                                         } else {
2946                                                 return Token.PLUS;
2947                                         }
2948                                         get_char ();
2949                                         return d;
2950
2951                                 case '-':
2952                                         val = LocatedToken.Create (ref_line, col);
2953                                         d = peek_char ();
2954                                         if (d == '-') {
2955                                                 d = Token.OP_DEC;
2956                                         } else if (d == '=')
2957                                                 d = Token.OP_SUB_ASSIGN;
2958                                         else if (d == '>')
2959                                                 d = Token.OP_PTR;
2960                                         else {
2961                                                 return Token.MINUS;
2962                                         }
2963                                         get_char ();
2964                                         return d;
2965
2966                                 case '!':
2967                                         val = LocatedToken.Create (ref_line, col);
2968                                         if (peek_char () == '='){
2969                                                 get_char ();
2970                                                 return Token.OP_NE;
2971                                         }
2972                                         return Token.BANG;
2973
2974                                 case '=':
2975                                         val = LocatedToken.Create (ref_line, col);
2976                                         d = peek_char ();
2977                                         if (d == '='){
2978                                                 get_char ();
2979                                                 return Token.OP_EQ;
2980                                         }
2981                                         if (d == '>'){
2982                                                 get_char ();
2983                                                 return Token.ARROW;
2984                                         }
2985
2986                                         return Token.ASSIGN;
2987
2988                                 case '&':
2989                                         val = LocatedToken.Create (ref_line, col);
2990                                         d = peek_char ();
2991                                         if (d == '&'){
2992                                                 get_char ();
2993                                                 return Token.OP_AND;
2994                                         }
2995                                         if (d == '='){
2996                                                 get_char ();
2997                                                 return Token.OP_AND_ASSIGN;
2998                                         }
2999                                         return Token.BITWISE_AND;
3000
3001                                 case '|':
3002                                         val = LocatedToken.Create (ref_line, col);
3003                                         d = peek_char ();
3004                                         if (d == '|'){
3005                                                 get_char ();
3006                                                 return Token.OP_OR;
3007                                         }
3008                                         if (d == '='){
3009                                                 get_char ();
3010                                                 return Token.OP_OR_ASSIGN;
3011                                         }
3012                                         return Token.BITWISE_OR;
3013
3014                                 case '*':
3015                                         val = LocatedToken.Create (ref_line, col);
3016                                         if (peek_char () == '='){
3017                                                 get_char ();
3018                                                 return Token.OP_MULT_ASSIGN;
3019                                         }
3020                                         return Token.STAR;
3021
3022                                 case '/':
3023                                         d = peek_char ();
3024                                         if (d == '='){
3025                                                 val = LocatedToken.Create (ref_line, col);
3026                                                 get_char ();
3027                                                 return Token.OP_DIV_ASSIGN;
3028                                         }
3029
3030                                         // Handle double-slash comments.
3031                                         if (d == '/'){
3032                                                 get_char ();
3033                                                 if (RootContext.Documentation != null && peek_char () == '/') {
3034                                                         get_char ();
3035                                                         // Don't allow ////.
3036                                                         if ((d = peek_char ()) != '/') {
3037                                                                 update_comment_location ();
3038                                                                 if (doc_state == XmlCommentState.Allowed)
3039                                                                         handle_one_line_xml_comment ();
3040                                                                 else if (doc_state == XmlCommentState.NotAllowed)
3041                                                                         warn_incorrect_doc_comment ();
3042                                                         }
3043                                                 }
3044                                                 while ((d = get_char ()) != -1 && (d != '\n') && d != '\r');
3045
3046                                                 any_token_seen |= tokens_seen;
3047                                                 tokens_seen = false;
3048                                                 comments_seen = false;
3049                                                 continue;
3050                                         } else if (d == '*'){
3051                                                 get_char ();
3052                                                 bool docAppend = false;
3053                                                 if (RootContext.Documentation != null && peek_char () == '*') {
3054                                                         get_char ();
3055                                                         update_comment_location ();
3056                                                         // But when it is /**/, just do nothing.
3057                                                         if (peek_char () == '/') {
3058                                                                 get_char ();
3059                                                                 continue;
3060                                                         }
3061                                                         if (doc_state == XmlCommentState.Allowed)
3062                                                                 docAppend = true;
3063                                                         else if (doc_state == XmlCommentState.NotAllowed)
3064                                                                 warn_incorrect_doc_comment ();
3065                                                 }
3066
3067                                                 int current_comment_start = 0;
3068                                                 if (docAppend) {
3069                                                         current_comment_start = xml_comment_buffer.Length;
3070                                                         xml_comment_buffer.Append (Environment.NewLine);
3071                                                 }
3072
3073                                                 while ((d = get_char ()) != -1){
3074                                                         if (d == '*' && peek_char () == '/'){
3075                                                                 get_char ();
3076                                                                 comments_seen = true;
3077                                                                 break;
3078                                                         }
3079                                                         if (docAppend)
3080                                                                 xml_comment_buffer.Append ((char) d);
3081
3082                                                         if (d == '\n'){
3083                                                                 any_token_seen |= tokens_seen;
3084                                                                 tokens_seen = false;
3085                                                                 //
3086                                                                 // Reset 'comments_seen' just to be consistent.
3087                                                                 // It doesn't matter either way, here.
3088                                                                 //
3089                                                                 comments_seen = false;
3090                                                         }
3091                                                 }
3092                                                 if (!comments_seen)
3093                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3094
3095                                                 if (docAppend)
3096                                                         update_formatted_doc_comment (current_comment_start);
3097                                                 continue;
3098                                         }
3099                                         val = LocatedToken.Create (ref_line, col);
3100                                         return Token.DIV;
3101
3102                                 case '%':
3103                                         val = LocatedToken.Create (ref_line, col);
3104                                         if (peek_char () == '='){
3105                                                 get_char ();
3106                                                 return Token.OP_MOD_ASSIGN;
3107                                         }
3108                                         return Token.PERCENT;
3109
3110                                 case '^':
3111                                         val = LocatedToken.Create (ref_line, col);
3112                                         if (peek_char () == '='){
3113                                                 get_char ();
3114                                                 return Token.OP_XOR_ASSIGN;
3115                                         }
3116                                         return Token.CARRET;
3117
3118                                 case ':':
3119                                         val = LocatedToken.Create (ref_line, col);
3120                                         if (peek_char () == ':') {
3121                                                 get_char ();
3122                                                 return Token.DOUBLE_COLON;
3123                                         }
3124                                         return Token.COLON;
3125
3126                                 case '0': case '1': case '2': case '3': case '4':
3127                                 case '5': case '6': case '7': case '8': case '9':
3128                                         tokens_seen = true;
3129                                         return is_number (c);
3130
3131                                 case '\n': // white space
3132                                         any_token_seen |= tokens_seen;
3133                                         tokens_seen = false;
3134                                         comments_seen = false;
3135                                         continue;
3136
3137                                 case '.':
3138                                         tokens_seen = true;
3139                                         d = peek_char ();
3140                                         if (d >= '0' && d <= '9')
3141                                                 return is_number (c);
3142
3143                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3144                                         return Token.DOT;
3145
3146                                 case '#':
3147                                         if (tokens_seen || comments_seen) {
3148                                                 Eror_WrongPreprocessorLocation ();
3149                                                 return Token.ERROR;
3150                                         }
3151
3152                                         if (ParsePreprocessingDirective (true))
3153                                                 continue;
3154
3155                                         bool directive_expected = false;
3156                                         while ((c = get_char ()) != -1) {
3157                                                 if (col == 1) {
3158                                                         directive_expected = true;
3159                                                 } else if (!directive_expected) {
3160                                                         // TODO: Implement comment support for disabled code and uncomment this code
3161 //                                                      if (c == '#') {
3162 //                                                              Eror_WrongPreprocessorLocation ();
3163 //                                                              return Token.ERROR;
3164 //                                                      }
3165                                                         continue;
3166                                                 }
3167
3168                                                 if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
3169                                                         continue;
3170
3171                                                 if (c == '#') {
3172                                                         if (ParsePreprocessingDirective (false))
3173                                                                 break;
3174                                                 }
3175                                                 directive_expected = false;
3176                                         }
3177
3178                                         if (c != -1) {
3179                                                 tokens_seen = false;
3180                                                 continue;
3181                                         }
3182
3183                                         return Token.EOF;
3184
3185                                 case '"':
3186                                         return consume_string (false);
3187
3188                                 case '\'':
3189                                         return TokenizeBackslash ();
3190
3191                                 case '@':
3192                                         c = get_char ();
3193                                         if (c == '"') {
3194                                                 tokens_seen = true;
3195                                                 return consume_string (true);
3196                                         }
3197
3198                                         if (is_identifier_start_character (c)){
3199                                                 return consume_identifier (c, true);
3200                                         }
3201
3202                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3203                                         return Token.ERROR;
3204
3205                                 case EvalStatementParserCharacter:
3206                                         return Token.EVAL_STATEMENT_PARSER;
3207                                 case EvalCompilationUnitParserCharacter:
3208                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3209                                 case EvalUsingDeclarationsParserCharacter:
3210                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3211                                 }
3212
3213                                 if (is_identifier_start_character (c)) {
3214                                         tokens_seen = true;
3215                                         return consume_identifier (c);
3216                                 }
3217
3218                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3219                         }
3220
3221                         if (CompleteOnEOF){
3222                                 if (generated)
3223                                         return Token.COMPLETE_COMPLETION;
3224
3225                                 generated = true;
3226                                 return Token.GENERATE_COMPLETION;
3227                         }
3228
3229
3230                         return Token.EOF;
3231                 }
3232
3233                 int TokenizeBackslash ()
3234                 {
3235                         int c = get_char ();
3236                         tokens_seen = true;
3237                         if (c == '\'') {
3238                                 val = new CharLiteral ((char) c, Location);
3239                                 Report.Error (1011, Location, "Empty character literal");
3240                                 return Token.LITERAL;
3241                         }
3242
3243                         if (c == '\r' || c == '\n') {
3244                                 Report.Error (1010, Location, "Newline in constant");
3245                                 return Token.ERROR;
3246                         }
3247
3248                         int d;
3249                         c = escape (c, out d);
3250                         if (c == -1)
3251                                 return Token.ERROR;
3252                         if (d != 0)
3253                                 throw new NotImplementedException ();
3254
3255                         val = new CharLiteral ((char) c, Location);
3256                         c = get_char ();
3257
3258                         if (c != '\'') {
3259                                 Report.Error (1012, Location, "Too many characters in character literal");
3260
3261                                 // Try to recover, read until newline or next "'"
3262                                 while ((c = get_char ()) != -1) {
3263                                         if (c == '\n' || c == '\'')
3264                                                 break;
3265                                 }
3266                         }
3267
3268                         return Token.LITERAL;
3269                 }
3270
3271                 int TokenizeLessThan ()
3272                 {
3273                         int d;
3274                         if (handle_typeof) {
3275                                 PushPosition ();
3276                                 if (parse_generic_dimension (out d)) {
3277                                         val = d;
3278                                         DiscardPosition ();
3279                                         return Token.GENERIC_DIMENSION;
3280                                 }
3281                                 PopPosition ();
3282                         }
3283
3284                         // Save current position and parse next token.
3285                         PushPosition ();
3286                         if (parse_less_than ()) {
3287                                 if (parsing_generic_declaration && token () != Token.DOT) {
3288                                         d = Token.OP_GENERICS_LT_DECL;
3289                                 } else {
3290                                         d = Token.OP_GENERICS_LT;
3291                                 }
3292                                 PopPosition ();
3293                                 return d;
3294                         }
3295
3296                         PopPosition ();
3297                         parsing_generic_less_than = 0;
3298
3299                         d = peek_char ();
3300                         if (d == '<') {
3301                                 get_char ();
3302                                 d = peek_char ();
3303
3304                                 if (d == '=') {
3305                                         get_char ();
3306                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3307                                 }
3308                                 return Token.OP_SHIFT_LEFT;
3309                         }
3310
3311                         if (d == '=') {
3312                                 get_char ();
3313                                 return Token.OP_LE;
3314                         }
3315                         return Token.OP_LT;
3316                 }
3317
3318                 //
3319                 // Handles one line xml comment
3320                 //
3321                 private void handle_one_line_xml_comment ()
3322                 {
3323                         int c;
3324                         while ((c = peek_char ()) == ' ')
3325                                 get_char (); // skip heading whitespaces.
3326                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3327                                 xml_comment_buffer.Append ((char) get_char ());
3328                         }
3329                         if (c == '\r' || c == '\n')
3330                                 xml_comment_buffer.Append (Environment.NewLine);
3331                 }
3332
3333                 //
3334                 // Remove heading "*" in Javadoc-like xml documentation.
3335                 //
3336                 private void update_formatted_doc_comment (int current_comment_start)
3337                 {
3338                         int length = xml_comment_buffer.Length - current_comment_start;
3339                         string [] lines = xml_comment_buffer.ToString (
3340                                 current_comment_start,
3341                                 length).Replace ("\r", "").Split ('\n');
3342
3343                         // The first line starts with /**, thus it is not target
3344                         // for the format check.
3345                         for (int i = 1; i < lines.Length; i++) {
3346                                 string s = lines [i];
3347                                 int idx = s.IndexOf ('*');
3348                                 string head = null;
3349                                 if (idx < 0) {
3350                                         if (i < lines.Length - 1)
3351                                                 return;
3352                                         head = s;
3353                                 } else
3354                                         head = s.Substring (0, idx);
3355                                 foreach (char c in head)
3356                                         if (c != ' ')
3357                                                 return;
3358                                 lines [i] = s.Substring (idx + 1);
3359                         }
3360                         xml_comment_buffer.Remove (current_comment_start, length);
3361                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3362                 }
3363
3364                 //
3365                 // Updates current comment location.
3366                 //
3367                 private void update_comment_location ()
3368                 {
3369                         if (current_comment_location.IsNull) {
3370                                 // "-2" is for heading "//" or "/*"
3371                                 current_comment_location =
3372                                         new Location (ref_line, hidden ? -1 : col - 2);
3373                         }
3374                 }
3375
3376                 //
3377                 // Checks if there was incorrect doc comments and raise
3378                 // warnings.
3379                 //
3380                 public void check_incorrect_doc_comment ()
3381                 {
3382                         if (xml_comment_buffer.Length > 0)
3383                                 warn_incorrect_doc_comment ();
3384                 }
3385
3386                 //
3387                 // Raises a warning when tokenizer found incorrect doccomment
3388                 // markup.
3389                 //
3390                 private void warn_incorrect_doc_comment ()
3391                 {
3392                         if (doc_state != XmlCommentState.Error) {
3393                                 doc_state = XmlCommentState.Error;
3394                                 // in csc, it is 'XML comment is not placed on
3395                                 // a valid language element'. But that does not
3396                                 // make sense.
3397                                 Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
3398                         }
3399                 }
3400
3401                 //
3402                 // Consumes the saved xml comment lines (if any)
3403                 // as for current target member or type.
3404                 //
3405                 public string consume_doc_comment ()
3406                 {
3407                         if (xml_comment_buffer.Length > 0) {
3408                                 string ret = xml_comment_buffer.ToString ();
3409                                 reset_doc_comment ();
3410                                 return ret;
3411                         }
3412                         return null;
3413                 }
3414
3415                 Report Report {
3416                         get { return context.Report; }
3417                 }
3418
3419                 void reset_doc_comment ()
3420                 {
3421                         xml_comment_buffer.Length = 0;
3422                         current_comment_location = Location.Null;
3423                 }
3424
3425                 public void cleanup ()
3426                 {
3427                         if (ifstack != null && ifstack.Count >= 1) {
3428                                 int state = ifstack.Pop ();
3429                                 if ((state & REGION) != 0)
3430                                         Report.Error (1038, Location, "#endregion directive expected");
3431                                 else
3432                                         Report.Error (1027, Location, "Expected `#endif' directive");
3433                         }
3434                 }
3435         }
3436
3437         //
3438         // Indicates whether it accepts XML documentation or not.
3439         //
3440         public enum XmlCommentState {
3441                 // comment is allowed in this state.
3442                 Allowed,
3443                 // comment is not allowed in this state.
3444                 NotAllowed,
3445                 // once comments appeared when it is NotAllowed, then the
3446                 // state is changed to it, until the state is changed to
3447                 // .Allowed.
3448                 Error
3449         }
3450 }
3451