mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 //
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20
  21 namespace Mono.CSharp
  22 {
  23         /// <summary>
  24         ///    Tokenizer for C# source code.
  25         /// </summary>
  26
  27         public class Tokenizer : yyParser.yyInput
  28         {
  29                 class KeywordEntry<T>
  30                 {
  31                         public readonly T Token;
  32                         public KeywordEntry<T> Next;
  33                         public readonly char[] Value;
  34
  35                         public KeywordEntry (string value, T token)
  36                         {
  37                                 this.Value = value.ToCharArray ();
  38                                 this.Token = token;
  39                         }
  40                 }
  41
  42                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  43                 {
  44                         readonly int length;
  45
  46                         public IdentifiersComparer (int length)
  47                         {
  48                                 this.length = length;
  49                         }
  50
  51                         public bool Equals (char[] x, char[] y)
  52                         {
  53                                 for (int i = 0; i < length; ++i)
  54                                         if (x [i] != y [i])
  55                                                 return false;
  56
  57                                 return true;
  58                         }
  59
  60                         public int GetHashCode (char[] obj)
  61                         {
  62                                 int h = 0;
  63                                 for (int i = 0; i < length; ++i)
  64                                         h = (h << 5) - h + obj [i];
  65
  66                                 return h;
  67                         }
  68                 }
  69
  70                 //
  71                 // This class has to be used in the parser only, it reuses token
  72                 // details after each parse
  73                 //
  74                 public class LocatedToken
  75                 {
  76                         int row, column;
  77                         string value;
  78
  79                         static LocatedToken[] buffer;
  80                         static int pos;
  81
  82                         private LocatedToken ()
  83                         {
  84                         }
  85
  86                         public static LocatedToken Create (int row, int column)
  87                         {
  88                                 return Create (null, row, column);
  89                         }
  90
  91                         public static LocatedToken Create (string value, int row, int column)
  92                         {
  93                                 //
  94                                 // TODO: I am not very happy about the logic but it's the best
  95                                 // what I could come up with for now.
  96                                 // Ideally we should be using just tiny buffer (256 elements) which
  97                                 // is enough to hold all details for currect stack and recycle elements
  98                                 // poped from the stack but there is a trick needed to recycle
  99                                 // them properly.
 100                                 //
 101                                 LocatedToken entry;
 102                                 if (pos >= buffer.Length) {
 103                                         entry = new LocatedToken ();
 104                                 } else {
 105                                         entry = buffer [pos];
 106                                         if (entry == null) {
 107                                                 entry = new LocatedToken ();
 108                                                 buffer [pos] = entry;
 109                                         }
 110
 111                                         ++pos;
 112                                 }
 113                                 entry.value = value;
 114                                 entry.row = row;
 115                                 entry.column = column;
 116                                 return entry;
 117                         }
 118
 119                         //
 120                         // Used for token not required by expression evaluator
 121                         //
 122                         [Conditional ("FULL_AST")]
 123                         public static void CreateOptional (int row, int col, ref object token)
 124                         {
 125                                 token = Create (row, col);
 126                         }
 127
 128                         public static void Initialize ()
 129                         {
 130                                 if (buffer == null)
 131                                         buffer = new LocatedToken [10000];
 132                                 pos = 0;
 133                         }
 134
 135                         public Location Location {
 136                                 get { return new Location (row, column); }
 137                         }
 138
 139                         public string Value {
 140                                 get { return value; }
 141                         }
 142                 }
 143
 144                 enum PreprocessorDirective
 145                 {
 146                         Invalid = 0,
 147
 148                         Region = 1,
 149                         Endregion = 2,
 150                         If = 3 | RequiresArgument,
 151                         Endif = 4,
 152                         Elif = 5 | RequiresArgument,
 153                         Else = 6,
 154                         Define = 7 | RequiresArgument,
 155                         Undef = 8 | RequiresArgument,
 156                         Error = 9,
 157                         Warning = 10,
 158                         Pragma = 11 | CustomArgumentsParsing,
 159                         Line = 12,
 160
 161                         CustomArgumentsParsing = 1 << 10,
 162                         RequiresArgument = 1 << 11
 163                 }
 164
 165                 SeekableStreamReader reader;
 166                 SourceFile ref_name;
 167                 CompilationSourceFile file_name;
 168                 CompilerContext context;
 169                 bool hidden = false;
 170                 int ref_line = 1;
 171                 int line = 1;
 172                 int col = 0;
 173                 int previous_col;
 174                 int current_token;
 175                 int tab_size;
 176                 bool handle_get_set = false;
 177                 bool handle_remove_add = false;
 178                 bool handle_where = false;
 179                 bool handle_typeof = false;
 180                 bool lambda_arguments_parsing;
 181                 List<Location> escaped_identifiers;
 182                 int parsing_generic_less_than;
 183                 readonly bool doc_processing;
 184
 185                 //
 186                 // Used mainly for parser optimizations. Some expressions for instance
 187                 // can appear only in block (including initializer, base initializer)
 188                 // scope only
 189                 //
 190                 public int parsing_block;
 191                 internal bool query_parsing;
 192
 193                 //
 194                 // When parsing type only, useful for ambiguous nullable types
 195                 //
 196                 public int parsing_type;
 197
 198                 //
 199                 // Set when parsing generic declaration (type or method header)
 200                 //
 201                 public bool parsing_generic_declaration;
 202
 203                 //
 204                 // The value indicates that we have not reach any declaration or
 205                 // namespace yet
 206                 //
 207                 public int parsing_declaration;
 208
 209                 public bool parsing_attribute_section;
 210
 211                 //
 212                 // The special character to inject on streams to trigger the EXPRESSION_PARSE
 213                 // token to be returned.   It just happens to be a Unicode character that
 214                 // would never be part of a program (can not be an identifier).
 215                 //
 216                 // This character is only tested just before the tokenizer is about to report
 217                 // an error;   So on the regular operation mode, this addition will have no
 218                 // impact on the tokenizer's performance.
 219                 //
 220
 221                 public const int EvalStatementParserCharacter = 0x2190;   // Unicode Left Arrow
 222                 public const int EvalCompilationUnitParserCharacter = 0x2191;  // Unicode Arrow
 223                 public const int EvalUsingDeclarationsParserCharacter = 0x2192;  // Unicode Arrow
 224
 225                 //
 226                 // XML documentation buffer. The save point is used to divide
 227                 // comments on types and comments on members.
 228                 //
 229                 StringBuilder xml_comment_buffer;
 230
 231                 //
 232                 // See comment on XmlCommentState enumeration.
 233                 //
 234                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 235
 236                 //
 237                 // Whether tokens have been seen on this line
 238                 //
 239                 bool tokens_seen = false;
 240
 241                 //
 242                 // Set to true once the GENERATE_COMPLETION token has bee
 243                 // returned.   This helps produce one GENERATE_COMPLETION,
 244                 // as many COMPLETE_COMPLETION as necessary to complete the
 245                 // AST tree and one final EOF.
 246                 //
 247                 bool generated;
 248
 249                 //
 250                 // Whether a token has been seen on the file
 251                 // This is needed because `define' is not allowed to be used
 252                 // after a token has been seen.
 253                 //
 254                 bool any_token_seen;
 255
 256                 //
 257                 // Class variables
 258                 //
 259                 static readonly KeywordEntry<int>[][] keywords;
 260                 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 261                 static readonly Dictionary<string, object> keyword_strings;             // TODO: HashSet
 262                 static readonly NumberStyles styles;
 263                 static readonly NumberFormatInfo csharp_format_info;
 264
 265                 // Pragma arguments
 266                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 267                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 268                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 269                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 270
 271                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 272
 273                 public bool PropertyParsing {
 274                         get { return handle_get_set; }
 275                         set { handle_get_set = value; }
 276                 }
 277
 278                 public bool EventParsing {
 279                         get { return handle_remove_add; }
 280                         set { handle_remove_add = value; }
 281                 }
 282
 283                 public bool ConstraintsParsing {
 284                         get { return handle_where; }
 285                         set { handle_where = value; }
 286                 }
 287
 288                 public bool TypeOfParsing {
 289                         get { return handle_typeof; }
 290                         set { handle_typeof = value; }
 291                 }
 292
 293                 public int TabSize {
 294                         get { return tab_size; }
 295                         set { tab_size = value; }
 296                 }
 297
 298                 public XmlCommentState doc_state {
 299                         get { return xml_doc_state; }
 300                         set {
 301                                 if (value == XmlCommentState.Allowed) {
 302                                         check_incorrect_doc_comment ();
 303                                         reset_doc_comment ();
 304                                 }
 305                                 xml_doc_state = value;
 306                         }
 307                 }
 308
 309                 //
 310                 // This is used to trigger completion generation on the parser
 311                 public bool CompleteOnEOF;
 312
 313                 void AddEscapedIdentifier (Location loc)
 314                 {
 315                         if (escaped_identifiers == null)
 316                                 escaped_identifiers = new List<Location> ();
 317
 318                         escaped_identifiers.Add (loc);
 319                 }
 320
 321                 public bool IsEscapedIdentifier (MemberName name)
 322                 {
 323                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 324                 }
 325
 326                 //
 327                 // Values for the associated token returned
 328                 //
 329                 internal int putback_char;      // Used by repl only
 330                 object val;
 331
 332                 //
 333                 // Pre-processor
 334                 //
 335                 const int TAKING        = 1;
 336                 const int ELSE_SEEN     = 4;
 337                 const int PARENT_TAKING = 8;
 338                 const int REGION        = 16;
 339
 340                 //
 341                 // pre-processor if stack state:
 342                 //
 343                 Stack<int> ifstack;
 344
 345                 static System.Text.StringBuilder string_builder;
 346
 347                 const int max_id_size = 512;
 348                 static readonly char [] id_builder = new char [max_id_size];
 349
 350                 public static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 351
 352                 const int max_number_size = 512;
 353                 static char [] number_builder = new char [max_number_size];
 354                 static int number_pos;
 355
 356                 static char[] value_builder = new char[256];
 357
 358                 public int Line {
 359                         get {
 360                                 return ref_line;
 361                         }
 362                 }
 363
 364                 //
 365                 // This is used when the tokenizer needs to save
 366                 // the current position as it needs to do some parsing
 367                 // on its own to deamiguate a token in behalf of the
 368                 // parser.
 369                 //
 370                 Stack<Position> position_stack = new Stack<Position> (2);
 371
 372                 class Position {
 373                         public int position;
 374                         public int line;
 375                         public int ref_line;
 376                         public int col;
 377                         public bool hidden;
 378                         public int putback_char;
 379                         public int previous_col;
 380                         public Stack<int> ifstack;
 381                         public int parsing_generic_less_than;
 382                         public int current_token;
 383                         public object val;
 384
 385                         public Position (Tokenizer t)
 386                         {
 387                                 position = t.reader.Position;
 388                                 line = t.line;
 389                                 ref_line = t.ref_line;
 390                                 col = t.col;
 391                                 hidden = t.hidden;
 392                                 putback_char = t.putback_char;
 393                                 previous_col = t.previous_col;
 394                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 395                                         // There is no simple way to clone Stack<T> all
 396                                         // methods reverse the order
 397                                         var clone = t.ifstack.ToArray ();
 398                                         Array.Reverse (clone);
 399                                         ifstack = new Stack<int> (clone);
 400                                 }
 401                                 parsing_generic_less_than = t.parsing_generic_less_than;
 402                                 current_token = t.current_token;
 403                                 val = t.val;
 404                         }
 405                 }
 406
 407                 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, CompilerContext ctx)
 408                 {
 409                         this.ref_name = file;
 410                         this.file_name = file;
 411                         this.context = ctx;
 412                         reader = input;
 413
 414                         putback_char = -1;
 415
 416                         xml_comment_buffer = new StringBuilder ();
 417                         doc_processing = ctx.Settings.Documentation != null;
 418
 419                         if (Environment.OSVersion.Platform == PlatformID.Win32NT)
 420                                 tab_size = 4;
 421                         else
 422                                 tab_size = 8;
 423
 424                         //
 425                         // FIXME: This could be `Location.Push' but we have to
 426                         // find out why the MS compiler allows this
 427                         //
 428                         Mono.CSharp.Location.Push (file, file);
 429                 }
 430
 431                 public void PushPosition ()
 432                 {
 433                         position_stack.Push (new Position (this));
 434                 }
 435
 436                 public void PopPosition ()
 437                 {
 438                         Position p = position_stack.Pop ();
 439
 440                         reader.Position = p.position;
 441                         ref_line = p.ref_line;
 442                         line = p.line;
 443                         col = p.col;
 444                         hidden = p.hidden;
 445                         putback_char = p.putback_char;
 446                         previous_col = p.previous_col;
 447                         ifstack = p.ifstack;
 448                         parsing_generic_less_than = p.parsing_generic_less_than;
 449                         current_token = p.current_token;
 450                         val = p.val;
 451                 }
 452
 453                 // Do not reset the position, ignore it.
 454                 public void DiscardPosition ()
 455                 {
 456                         position_stack.Pop ();
 457                 }
 458
 459                 static void AddKeyword (string kw, int token)
 460                 {
 461                         keyword_strings.Add (kw, null);
 462
 463                         AddKeyword (keywords, kw, token);
 464                 }
 465
 466                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 467                 {
 468                         AddKeyword (keywords_preprocessor, kw, directive);
 469                 }
 470
 471                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 472                 {
 473                         int length = kw.Length;
 474                         if (keywords[length] == null) {
 475                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 476                         }
 477
 478                         int char_index = kw[0] - '_';
 479                         var kwe = keywords[length][char_index];
 480                         if (kwe == null) {
 481                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 482                                 return;
 483                         }
 484
 485                         while (kwe.Next != null) {
 486                                 kwe = kwe.Next;
 487                         }
 488
 489                         kwe.Next = new KeywordEntry<T> (kw, token);
 490                 }
 491
 492                 //
 493                 // Class initializer
 494                 //
 495                 static Tokenizer ()
 496                 {
 497                         keyword_strings = new Dictionary<string, object> ();
 498
 499                         // 11 is the length of the longest keyword for now
 500                         keywords = new KeywordEntry<int>[11][];
 501
 502                         AddKeyword ("__arglist", Token.ARGLIST);
 503                         AddKeyword ("__makeref", Token.MAKEREF);
 504                         AddKeyword ("__reftype", Token.REFTYPE);
 505                         AddKeyword ("__refvalue", Token.REFVALUE);
 506                         AddKeyword ("abstract", Token.ABSTRACT);
 507                         AddKeyword ("as", Token.AS);
 508                         AddKeyword ("add", Token.ADD);
 509                         AddKeyword ("base", Token.BASE);
 510                         AddKeyword ("bool", Token.BOOL);
 511                         AddKeyword ("break", Token.BREAK);
 512                         AddKeyword ("byte", Token.BYTE);
 513                         AddKeyword ("case", Token.CASE);
 514                         AddKeyword ("catch", Token.CATCH);
 515                         AddKeyword ("char", Token.CHAR);
 516                         AddKeyword ("checked", Token.CHECKED);
 517                         AddKeyword ("class", Token.CLASS);
 518                         AddKeyword ("const", Token.CONST);
 519                         AddKeyword ("continue", Token.CONTINUE);
 520                         AddKeyword ("decimal", Token.DECIMAL);
 521                         AddKeyword ("default", Token.DEFAULT);
 522                         AddKeyword ("delegate", Token.DELEGATE);
 523                         AddKeyword ("do", Token.DO);
 524                         AddKeyword ("double", Token.DOUBLE);
 525                         AddKeyword ("else", Token.ELSE);
 526                         AddKeyword ("enum", Token.ENUM);
 527                         AddKeyword ("event", Token.EVENT);
 528                         AddKeyword ("explicit", Token.EXPLICIT);
 529                         AddKeyword ("extern", Token.EXTERN);
 530                         AddKeyword ("false", Token.FALSE);
 531                         AddKeyword ("finally", Token.FINALLY);
 532                         AddKeyword ("fixed", Token.FIXED);
 533                         AddKeyword ("float", Token.FLOAT);
 534                         AddKeyword ("for", Token.FOR);
 535                         AddKeyword ("foreach", Token.FOREACH);
 536                         AddKeyword ("goto", Token.GOTO);
 537                         AddKeyword ("get", Token.GET);
 538                         AddKeyword ("if", Token.IF);
 539                         AddKeyword ("implicit", Token.IMPLICIT);
 540                         AddKeyword ("in", Token.IN);
 541                         AddKeyword ("int", Token.INT);
 542                         AddKeyword ("interface", Token.INTERFACE);
 543                         AddKeyword ("internal", Token.INTERNAL);
 544                         AddKeyword ("is", Token.IS);
 545                         AddKeyword ("lock", Token.LOCK);
 546                         AddKeyword ("long", Token.LONG);
 547                         AddKeyword ("namespace", Token.NAMESPACE);
 548                         AddKeyword ("new", Token.NEW);
 549                         AddKeyword ("null", Token.NULL);
 550                         AddKeyword ("object", Token.OBJECT);
 551                         AddKeyword ("operator", Token.OPERATOR);
 552                         AddKeyword ("out", Token.OUT);
 553                         AddKeyword ("override", Token.OVERRIDE);
 554                         AddKeyword ("params", Token.PARAMS);
 555                         AddKeyword ("private", Token.PRIVATE);
 556                         AddKeyword ("protected", Token.PROTECTED);
 557                         AddKeyword ("public", Token.PUBLIC);
 558                         AddKeyword ("readonly", Token.READONLY);
 559                         AddKeyword ("ref", Token.REF);
 560                         AddKeyword ("remove", Token.REMOVE);
 561                         AddKeyword ("return", Token.RETURN);
 562                         AddKeyword ("sbyte", Token.SBYTE);
 563                         AddKeyword ("sealed", Token.SEALED);
 564                         AddKeyword ("set", Token.SET);
 565                         AddKeyword ("short", Token.SHORT);
 566                         AddKeyword ("sizeof", Token.SIZEOF);
 567                         AddKeyword ("stackalloc", Token.STACKALLOC);
 568                         AddKeyword ("static", Token.STATIC);
 569                         AddKeyword ("string", Token.STRING);
 570                         AddKeyword ("struct", Token.STRUCT);
 571                         AddKeyword ("switch", Token.SWITCH);
 572                         AddKeyword ("this", Token.THIS);
 573                         AddKeyword ("throw", Token.THROW);
 574                         AddKeyword ("true", Token.TRUE);
 575                         AddKeyword ("try", Token.TRY);
 576                         AddKeyword ("typeof", Token.TYPEOF);
 577                         AddKeyword ("uint", Token.UINT);
 578                         AddKeyword ("ulong", Token.ULONG);
 579                         AddKeyword ("unchecked", Token.UNCHECKED);
 580                         AddKeyword ("unsafe", Token.UNSAFE);
 581                         AddKeyword ("ushort", Token.USHORT);
 582                         AddKeyword ("using", Token.USING);
 583                         AddKeyword ("virtual", Token.VIRTUAL);
 584                         AddKeyword ("void", Token.VOID);
 585                         AddKeyword ("volatile", Token.VOLATILE);
 586                         AddKeyword ("while", Token.WHILE);
 587                         AddKeyword ("partial", Token.PARTIAL);
 588                         AddKeyword ("where", Token.WHERE);
 589                         AddKeyword ("async", Token.ASYNC);
 590
 591                         // LINQ keywords
 592                         AddKeyword ("from", Token.FROM);
 593                         AddKeyword ("join", Token.JOIN);
 594                         AddKeyword ("on", Token.ON);
 595                         AddKeyword ("equals", Token.EQUALS);
 596                         AddKeyword ("select", Token.SELECT);
 597                         AddKeyword ("group", Token.GROUP);
 598                         AddKeyword ("by", Token.BY);
 599                         AddKeyword ("let", Token.LET);
 600                         AddKeyword ("orderby", Token.ORDERBY);
 601                         AddKeyword ("ascending", Token.ASCENDING);
 602                         AddKeyword ("descending", Token.DESCENDING);
 603                         AddKeyword ("into", Token.INTO);
 604
 605                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 606
 607                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 608                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 609                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 610                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 611                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 612                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 613                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 614                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 615                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 616                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 617                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 618                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 619
 620                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 621                         styles = NumberStyles.Float;
 622
 623                         string_builder = new System.Text.StringBuilder ();
 624                 }
 625
 626                 int GetKeyword (char[] id, int id_len)
 627                 {
 628                         //
 629                         // Keywords are stored in an array of arrays grouped by their
 630                         // length and then by the first character
 631                         //
 632                         if (id_len >= keywords.Length || keywords [id_len] == null)
 633                                 return -1;
 634
 635                         int first_index = id [0] - '_';
 636                         if (first_index > 'z' - '_')
 637                                 return -1;
 638
 639                         var kwe = keywords [id_len] [first_index];
 640                         if (kwe == null)
 641                                 return -1;
 642
 643                         int res;
 644                         do {
 645                                 res = kwe.Token;
 646                                 for (int i = 1; i < id_len; ++i) {
 647                                         if (id [i] != kwe.Value [i]) {
 648                                                 res = 0;
 649                                                 kwe = kwe.Next;
 650                                                 break;
 651                                         }
 652                                 }
 653                         } while (res == 0 && kwe != null);
 654
 655                         if (res == 0)
 656                                 return -1;
 657
 658                         int next_token;
 659                         switch (res) {
 660                         case Token.GET:
 661                         case Token.SET:
 662                                 if (!handle_get_set)
 663                                         res = -1;
 664                                 break;
 665                         case Token.REMOVE:
 666                         case Token.ADD:
 667                                 if (!handle_remove_add)
 668                                         res = -1;
 669                                 break;
 670                         case Token.EXTERN:
 671                                 if (parsing_declaration == 0)
 672                                         res = Token.EXTERN_ALIAS;
 673                                 break;
 674                         case Token.DEFAULT:
 675                                 if (peek_token () == Token.COLON) {
 676                                         token ();
 677                                         res = Token.DEFAULT_COLON;
 678                                 }
 679                                 break;
 680                         case Token.WHERE:
 681                                 if (!handle_where && !query_parsing)
 682                                         res = -1;
 683                                 break;
 684                         case Token.FROM:
 685                                 //
 686                                 // A query expression is any expression that starts with `from identifier'
 687                                 // followed by any token except ; , =
 688                                 //
 689                                 if (!query_parsing) {
 690                                         if (lambda_arguments_parsing) {
 691                                                 res = -1;
 692                                                 break;
 693                                         }
 694
 695                                         PushPosition ();
 696                                         // HACK: to disable generics micro-parser, because PushPosition does not
 697                                         // store identifiers array
 698                                         parsing_generic_less_than = 1;
 699                                         switch (xtoken ()) {
 700                                         case Token.IDENTIFIER:
 701                                         case Token.INT:
 702                                         case Token.BOOL:
 703                                         case Token.BYTE:
 704                                         case Token.CHAR:
 705                                         case Token.DECIMAL:
 706                                         case Token.FLOAT:
 707                                         case Token.LONG:
 708                                         case Token.OBJECT:
 709                                         case Token.STRING:
 710                                         case Token.UINT:
 711                                         case Token.ULONG:
 712                                                 next_token = xtoken ();
 713                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 714                                                         goto default;
 715
 716                                                 res = Token.FROM_FIRST;
 717                                                 query_parsing = true;
 718                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 719                                                         Report.FeatureIsNotAvailable (context, Location, "query expressions");
 720                                                 break;
 721                                         case Token.VOID:
 722                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 723                                                 break;
 724                                         default:
 725                                                 PopPosition ();
 726                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 727                                                 // which has been overwritten before we grabbed the identifier
 728                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 729                                                 return -1;
 730                                         }
 731                                         PopPosition ();
 732                                 }
 733                                 break;
 734                         case Token.JOIN:
 735                         case Token.ON:
 736                         case Token.EQUALS:
 737                         case Token.SELECT:
 738                         case Token.GROUP:
 739                         case Token.BY:
 740                         case Token.LET:
 741                         case Token.ORDERBY:
 742                         case Token.ASCENDING:
 743                         case Token.DESCENDING:
 744                         case Token.INTO:
 745                                 if (!query_parsing)
 746                                         res = -1;
 747                                 break;
 748
 749                         case Token.USING:
 750                         case Token.NAMESPACE:
 751                                 // TODO: some explanation needed
 752                                 check_incorrect_doc_comment ();
 753                                 break;
 754
 755                         case Token.PARTIAL:
 756                                 if (parsing_block > 0) {
 757                                         res = -1;
 758                                         break;
 759                                 }
 760
 761                                 // Save current position and parse next token.
 762                                 PushPosition ();
 763
 764                                 next_token = token ();
 765                                 bool ok = (next_token == Token.CLASS) ||
 766                                         (next_token == Token.STRUCT) ||
 767                                         (next_token == Token.INTERFACE) ||
 768                                         (next_token == Token.VOID);
 769
 770                                 PopPosition ();
 771
 772                                 if (ok) {
 773                                         if (next_token == Token.VOID) {
 774                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 775                                                         Report.FeatureIsNotAvailable (context, Location, "partial methods");
 776                                         } else if (context.Settings.Version == LanguageVersion.ISO_1)
 777                                                 Report.FeatureIsNotAvailable (context, Location, "partial types");
 778
 779                                         return res;
 780                                 }
 781
 782                                 if (next_token < Token.LAST_KEYWORD) {
 783                                         Report.Error (267, Location,
 784                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 785                                         return token ();
 786                                 }
 787
 788                                 res = -1;
 789                                 break;
 790
 791                         case Token.ASYNC:
 792                                 if (parsing_block > 0 || context.Settings.Version != LanguageVersion.Future) {
 793                                         res = -1;
 794                                         break;
 795                                 }
 796                                 break;
 797                         }
 798
 799                         return res;
 800                 }
 801
 802                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 803                 {
 804                         //
 805                         // Keywords are stored in an array of arrays grouped by their
 806                         // length and then by the first character
 807                         //
 808                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 809                                 return PreprocessorDirective.Invalid;
 810
 811                         int first_index = id[0] - '_';
 812                         if (first_index > 'z' - '_')
 813                                 return PreprocessorDirective.Invalid;
 814
 815                         var kwe = keywords_preprocessor[id_len][first_index];
 816                         if (kwe == null)
 817                                 return PreprocessorDirective.Invalid;
 818
 819                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 820                         do {
 821                                 res = kwe.Token;
 822                                 for (int i = 1; i < id_len; ++i) {
 823                                         if (id[i] != kwe.Value[i]) {
 824                                                 res = 0;
 825                                                 kwe = kwe.Next;
 826                                                 break;
 827                                         }
 828                                 }
 829                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 830
 831                         return res;
 832                 }
 833
 834                 public Location Location {
 835                         get {
 836                                 return new Location (ref_line, hidden ? -1 : col);
 837                         }
 838                 }
 839
 840                 static bool is_identifier_start_character (int c)
 841                 {
 842                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 843                 }
 844
 845                 static bool is_identifier_part_character (char c)
 846                 {
 847                         if (c >= 'a' && c <= 'z')
 848                                 return true;
 849
 850                         if (c >= 'A' && c <= 'Z')
 851                                 return true;
 852
 853                         if (c == '_' || (c >= '0' && c <= '9'))
 854                                 return true;
 855
 856                         if (c < 0x80)
 857                                 return false;
 858
 859                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 860                 }
 861
 862                 public static bool IsKeyword (string s)
 863                 {
 864                         return keyword_strings.ContainsKey (s);
 865                 }
 866
 867                 //
 868                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 869                 //
 870                 int TokenizeOpenParens ()
 871                 {
 872                         int ptoken;
 873                         current_token = -1;
 874
 875                         int bracket_level = 0;
 876                         bool is_type = false;
 877                         bool can_be_type = false;
 878
 879                         while (true) {
 880                                 ptoken = current_token;
 881                                 token ();
 882
 883                                 switch (current_token) {
 884                                 case Token.CLOSE_PARENS:
 885                                         token ();
 886
 887                                         //
 888                                         // Expression inside parens is lambda, (int i) =>
 889                                         //
 890                                         if (current_token == Token.ARROW)
 891                                                 return Token.OPEN_PARENS_LAMBDA;
 892
 893                                         //
 894                                         // Expression inside parens is single type, (int[])
 895                                         //
 896                                         if (is_type)
 897                                                 return Token.OPEN_PARENS_CAST;
 898
 899                                         //
 900                                         // Expression is possible cast, look at next token, (T)null
 901                                         //
 902                                         if (can_be_type) {
 903                                                 switch (current_token) {
 904                                                 case Token.OPEN_PARENS:
 905                                                 case Token.BANG:
 906                                                 case Token.TILDE:
 907                                                 case Token.IDENTIFIER:
 908                                                 case Token.LITERAL:
 909                                                 case Token.BASE:
 910                                                 case Token.CHECKED:
 911                                                 case Token.DELEGATE:
 912                                                 case Token.FALSE:
 913                                                 case Token.FIXED:
 914                                                 case Token.NEW:
 915                                                 case Token.NULL:
 916                                                 case Token.SIZEOF:
 917                                                 case Token.THIS:
 918                                                 case Token.THROW:
 919                                                 case Token.TRUE:
 920                                                 case Token.TYPEOF:
 921                                                 case Token.UNCHECKED:
 922                                                 case Token.UNSAFE:
 923                                                 case Token.DEFAULT:
 924
 925                                                 //
 926                                                 // These can be part of a member access
 927                                                 //
 928                                                 case Token.INT:
 929                                                 case Token.UINT:
 930                                                 case Token.SHORT:
 931                                                 case Token.USHORT:
 932                                                 case Token.LONG:
 933                                                 case Token.ULONG:
 934                                                 case Token.DOUBLE:
 935                                                 case Token.FLOAT:
 936                                                 case Token.CHAR:
 937                                                 case Token.BYTE:
 938                                                 case Token.DECIMAL:
 939                                                 case Token.BOOL:
 940                                                         return Token.OPEN_PARENS_CAST;
 941                                                 }
 942                                         }
 943                                         return Token.OPEN_PARENS;
 944
 945                                 case Token.DOT:
 946                                 case Token.DOUBLE_COLON:
 947                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
 948                                                 goto default;
 949
 950                                         continue;
 951
 952                                 case Token.IDENTIFIER:
 953                                         switch (ptoken) {
 954                                         case Token.DOT:
 955                                                 if (bracket_level == 0) {
 956                                                         is_type = false;
 957                                                         can_be_type = true;
 958                                                 }
 959
 960                                                 continue;
 961                                         case Token.OP_GENERICS_LT:
 962                                         case Token.COMMA:
 963                                         case Token.DOUBLE_COLON:
 964                                         case -1:
 965                                                 if (bracket_level == 0)
 966                                                         can_be_type = true;
 967                                                 continue;
 968                                         default:
 969                                                 can_be_type = is_type = false;
 970                                                 continue;
 971                                         }
 972
 973                                 case Token.OBJECT:
 974                                 case Token.STRING:
 975                                 case Token.BOOL:
 976                                 case Token.DECIMAL:
 977                                 case Token.FLOAT:
 978                                 case Token.DOUBLE:
 979                                 case Token.SBYTE:
 980                                 case Token.BYTE:
 981                                 case Token.SHORT:
 982                                 case Token.USHORT:
 983                                 case Token.INT:
 984                                 case Token.UINT:
 985                                 case Token.LONG:
 986                                 case Token.ULONG:
 987                                 case Token.CHAR:
 988                                 case Token.VOID:
 989                                         if (bracket_level == 0)
 990                                                 is_type = true;
 991                                         continue;
 992
 993                                 case Token.COMMA:
 994                                         if (bracket_level == 0) {
 995                                                 bracket_level = 100;
 996                                                 can_be_type = is_type = false;
 997                                         }
 998                                         continue;
 999
1000                                 case Token.OP_GENERICS_LT:
1001                                 case Token.OPEN_BRACKET:
1002                                         if (bracket_level++ == 0)
1003                                                 is_type = true;
1004                                         continue;
1005
1006                                 case Token.OP_GENERICS_GT:
1007                                 case Token.CLOSE_BRACKET:
1008                                         --bracket_level;
1009                                         continue;
1010
1011                                 case Token.INTERR_NULLABLE:
1012                                 case Token.STAR:
1013                                         if (bracket_level == 0)
1014                                                 is_type = true;
1015                                         continue;
1016
1017                                 case Token.REF:
1018                                 case Token.OUT:
1019                                         can_be_type = is_type = false;
1020                                         continue;
1021
1022                                 default:
1023                                         return Token.OPEN_PARENS;
1024                                 }
1025                         }
1026                 }
1027
1028                 public static bool IsValidIdentifier (string s)
1029                 {
1030                         if (s == null || s.Length == 0)
1031                                 return false;
1032
1033                         if (!is_identifier_start_character (s [0]))
1034                                 return false;
1035
1036                         for (int i = 1; i < s.Length; i ++)
1037                                 if (! is_identifier_part_character (s [i]))
1038                                         return false;
1039
1040                         return true;
1041                 }
1042
1043                 bool parse_less_than ()
1044                 {
1045                 start:
1046                         int the_token = token ();
1047                         if (the_token == Token.OPEN_BRACKET) {
1048                                 do {
1049                                         the_token = token ();
1050                                 } while (the_token != Token.CLOSE_BRACKET);
1051                                 the_token = token ();
1052                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1053                                 the_token = token ();
1054                         }
1055                         switch (the_token) {
1056                         case Token.IDENTIFIER:
1057                         case Token.OBJECT:
1058                         case Token.STRING:
1059                         case Token.BOOL:
1060                         case Token.DECIMAL:
1061                         case Token.FLOAT:
1062                         case Token.DOUBLE:
1063                         case Token.SBYTE:
1064                         case Token.BYTE:
1065                         case Token.SHORT:
1066                         case Token.USHORT:
1067                         case Token.INT:
1068                         case Token.UINT:
1069                         case Token.LONG:
1070                         case Token.ULONG:
1071                         case Token.CHAR:
1072                         case Token.VOID:
1073                                 break;
1074                         case Token.OP_GENERICS_GT:
1075                                 return true;
1076
1077                         default:
1078                                 return false;
1079                         }
1080                 again:
1081                         the_token = token ();
1082
1083                         if (the_token == Token.OP_GENERICS_GT)
1084                                 return true;
1085                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1086                                 goto start;
1087                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1088                                 goto again;
1089                         else if (the_token == Token.OP_GENERICS_LT) {
1090                                 if (!parse_less_than ())
1091                                         return false;
1092                                 goto again;
1093                         } else if (the_token == Token.OPEN_BRACKET) {
1094                         rank_specifiers:
1095                                 the_token = token ();
1096                                 if (the_token == Token.CLOSE_BRACKET)
1097                                         goto again;
1098                                 else if (the_token == Token.COMMA)
1099                                         goto rank_specifiers;
1100                                 return false;
1101                         }
1102
1103                         return false;
1104                 }
1105
1106                 bool parse_generic_dimension (out int dimension)
1107                 {
1108                         dimension = 1;
1109
1110                 again:
1111                         int the_token = token ();
1112                         if (the_token == Token.OP_GENERICS_GT)
1113                                 return true;
1114                         else if (the_token == Token.COMMA) {
1115                                 dimension++;
1116                                 goto again;
1117                         }
1118
1119                         return false;
1120                 }
1121
1122                 public int peek_token ()
1123                 {
1124                         int the_token;
1125
1126                         PushPosition ();
1127                         the_token = token ();
1128                         PopPosition ();
1129
1130                         return the_token;
1131                 }
1132
1133                 //
1134                 // Tonizes `?' using custom disambiguous rules to return one
1135                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1136                 //
1137                 // Tricky expression look like:
1138                 //
1139                 // Foo ? a = x ? b : c;
1140                 //
1141                 int TokenizePossibleNullableType ()
1142                 {
1143                         if (parsing_block == 0 || parsing_type > 0)
1144                                 return Token.INTERR_NULLABLE;
1145
1146                         int d = peek_char ();
1147                         if (d == '?') {
1148                                 get_char ();
1149                                 return Token.OP_COALESCING;
1150                         }
1151
1152                         switch (current_token) {
1153                         case Token.CLOSE_PARENS:
1154                         case Token.TRUE:
1155                         case Token.FALSE:
1156                         case Token.NULL:
1157                         case Token.LITERAL:
1158                                 return Token.INTERR;
1159                         }
1160
1161                         if (d != ' ') {
1162                                 if (d == ',' || d == ';' || d == '>')
1163                                         return Token.INTERR_NULLABLE;
1164                                 if (d == '*' || (d >= '0' && d <= '9'))
1165                                         return Token.INTERR;
1166                         }
1167
1168                         PushPosition ();
1169                         current_token = Token.NONE;
1170                         int next_token;
1171                         switch (xtoken ()) {
1172                         case Token.LITERAL:
1173                         case Token.TRUE:
1174                         case Token.FALSE:
1175                         case Token.NULL:
1176                         case Token.THIS:
1177                         case Token.NEW:
1178                                 next_token = Token.INTERR;
1179                                 break;
1180
1181                         case Token.SEMICOLON:
1182                         case Token.COMMA:
1183                         case Token.CLOSE_PARENS:
1184                         case Token.OPEN_BRACKET:
1185                         case Token.OP_GENERICS_GT:
1186                                 next_token = Token.INTERR_NULLABLE;
1187                                 break;
1188
1189                         default:
1190                                 next_token = -1;
1191                                 break;
1192                         }
1193
1194                         if (next_token == -1) {
1195                                 switch (xtoken ()) {
1196                                 case Token.COMMA:
1197                                 case Token.SEMICOLON:
1198                                 case Token.OPEN_BRACE:
1199                                 case Token.CLOSE_PARENS:
1200                                 case Token.IN:
1201                                         next_token = Token.INTERR_NULLABLE;
1202                                         break;
1203
1204                                 case Token.COLON:
1205                                         next_token = Token.INTERR;
1206                                         break;
1207
1208                                 default:
1209                                         int ntoken;
1210                                         int interrs = 1;
1211                                         int colons = 0;
1212                                         //
1213                                         // All shorcuts failed, do it hard way
1214                                         //
1215                                         while ((ntoken = xtoken ()) != Token.EOF) {
1216                                                 if (ntoken == Token.SEMICOLON)
1217                                                         break;
1218
1219                                                 if (ntoken == Token.COLON) {
1220                                                         if (++colons == interrs)
1221                                                                 break;
1222                                                         continue;
1223                                                 }
1224
1225                                                 if (ntoken == Token.INTERR) {
1226                                                         ++interrs;
1227                                                         continue;
1228                                                 }
1229                                         }
1230
1231                                         next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1232                                         break;
1233                                 }
1234                         }
1235
1236                         PopPosition ();
1237                         return next_token;
1238                 }
1239
1240                 bool decimal_digits (int c)
1241                 {
1242                         int d;
1243                         bool seen_digits = false;
1244
1245                         if (c != -1){
1246                                 if (number_pos == max_number_size)
1247                                         Error_NumericConstantTooLong ();
1248                                 number_builder [number_pos++] = (char) c;
1249                         }
1250
1251                         //
1252                         // We use peek_char2, because decimal_digits needs to do a
1253                         // 2-character look-ahead (5.ToString for example).
1254                         //
1255                         while ((d = peek_char2 ()) != -1){
1256                                 if (d >= '0' && d <= '9'){
1257                                         if (number_pos == max_number_size)
1258                                                 Error_NumericConstantTooLong ();
1259                                         number_builder [number_pos++] = (char) d;
1260                                         get_char ();
1261                                         seen_digits = true;
1262                                 } else
1263                                         break;
1264                         }
1265
1266                         return seen_digits;
1267                 }
1268
1269                 static bool is_hex (int e)
1270                 {
1271                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1272                 }
1273
1274                 static TypeCode real_type_suffix (int c)
1275                 {
1276                         switch (c){
1277                         case 'F': case 'f':
1278                                 return TypeCode.Single;
1279                         case 'D': case 'd':
1280                                 return TypeCode.Double;
1281                         case 'M': case 'm':
1282                                 return TypeCode.Decimal;
1283                         default:
1284                                 return TypeCode.Empty;
1285                         }
1286                 }
1287
1288                 int integer_type_suffix (ulong ul, int c)
1289                 {
1290                         bool is_unsigned = false;
1291                         bool is_long = false;
1292
1293                         if (c != -1){
1294                                 bool scanning = true;
1295                                 do {
1296                                         switch (c){
1297                                         case 'U': case 'u':
1298                                                 if (is_unsigned)
1299                                                         scanning = false;
1300                                                 is_unsigned = true;
1301                                                 get_char ();
1302                                                 break;
1303
1304                                         case 'l':
1305                                                 if (!is_unsigned){
1306                                                         //
1307                                                         // if we have not seen anything in between
1308                                                         // report this error
1309                                                         //
1310                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1311                                                 }
1312
1313                                                 goto case 'L';
1314
1315                                         case 'L':
1316                                                 if (is_long)
1317                                                         scanning = false;
1318                                                 is_long = true;
1319                                                 get_char ();
1320                                                 break;
1321
1322                                         default:
1323                                                 scanning = false;
1324                                                 break;
1325                                         }
1326                                         c = peek_char ();
1327                                 } while (scanning);
1328                         }
1329
1330                         if (is_long && is_unsigned){
1331                                 val = new ULongLiteral (context.BuiltinTypes, ul, Location);
1332                                 return Token.LITERAL;
1333                         }
1334
1335                         if (is_unsigned){
1336                                 // uint if possible, or ulong else.
1337
1338                                 if ((ul & 0xffffffff00000000) == 0)
1339                                         val = new UIntLiteral (context.BuiltinTypes, (uint) ul, Location);
1340                                 else
1341                                         val = new ULongLiteral (context.BuiltinTypes, ul, Location);
1342                         } else if (is_long){
1343                                 // long if possible, ulong otherwise
1344                                 if ((ul & 0x8000000000000000) != 0)
1345                                         val = new ULongLiteral (context.BuiltinTypes, ul, Location);
1346                                 else
1347                                         val = new LongLiteral (context.BuiltinTypes, (long) ul, Location);
1348                         } else {
1349                                 // int, uint, long or ulong in that order
1350                                 if ((ul & 0xffffffff00000000) == 0){
1351                                         uint ui = (uint) ul;
1352
1353                                         if ((ui & 0x80000000) != 0)
1354                                                 val = new UIntLiteral (context.BuiltinTypes, ui, Location);
1355                                         else
1356                                                 val = new IntLiteral (context.BuiltinTypes, (int) ui, Location);
1357                                 } else {
1358                                         if ((ul & 0x8000000000000000) != 0)
1359                                                 val = new ULongLiteral (context.BuiltinTypes, ul, Location);
1360                                         else
1361                                                 val = new LongLiteral (context.BuiltinTypes, (long) ul, Location);
1362                                 }
1363                         }
1364                         return Token.LITERAL;
1365                 }
1366
1367                 //
1368                 // given `c' as the next char in the input decide whether
1369                 // we need to convert to a special type, and then choose
1370                 // the best representation for the integer
1371                 //
1372                 int adjust_int (int c)
1373                 {
1374                         try {
1375                                 if (number_pos > 9){
1376                                         ulong ul = (uint) (number_builder [0] - '0');
1377
1378                                         for (int i = 1; i < number_pos; i++){
1379                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1380                                         }
1381                                         return integer_type_suffix (ul, c);
1382                                 } else {
1383                                         uint ui = (uint) (number_builder [0] - '0');
1384
1385                                         for (int i = 1; i < number_pos; i++){
1386                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1387                                         }
1388                                         return integer_type_suffix (ui, c);
1389                                 }
1390                         } catch (OverflowException) {
1391                                 Error_NumericConstantTooLong ();
1392                                 val = new IntLiteral (context.BuiltinTypes, 0, Location);
1393                                 return Token.LITERAL;
1394                         }
1395                         catch (FormatException) {
1396                                 Report.Error (1013, Location, "Invalid number");
1397                                 val = new IntLiteral (context.BuiltinTypes, 0, Location);
1398                                 return Token.LITERAL;
1399                         }
1400                 }
1401
1402                 int adjust_real (TypeCode t)
1403                 {
1404                         string s = new String (number_builder, 0, number_pos);
1405                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1406
1407                         switch (t){
1408                         case TypeCode.Decimal:
1409                                 try {
1410                                         val = new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), Location);
1411                                 } catch (OverflowException) {
1412                                         val = new DecimalLiteral (context.BuiltinTypes, 0, Location);
1413                                         Report.Error (594, Location, error_details, "decimal");
1414                                 }
1415                                 break;
1416                         case TypeCode.Single:
1417                                 try {
1418                                         val = new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), Location);
1419                                 } catch (OverflowException) {
1420                                         val = new FloatLiteral (context.BuiltinTypes, 0, Location);
1421                                         Report.Error (594, Location, error_details, "float");
1422                                 }
1423                                 break;
1424                         default:
1425                                 try {
1426                                         val = new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), Location);
1427                                 } catch (OverflowException) {
1428                                         val = new DoubleLiteral (context.BuiltinTypes, 0, Location);
1429                                         Report.Error (594, Location, error_details, "double");
1430                                 }
1431                                 break;
1432                         }
1433
1434                         return Token.LITERAL;
1435                 }
1436
1437                 int handle_hex ()
1438                 {
1439                         int d;
1440                         ulong ul;
1441
1442                         get_char ();
1443                         while ((d = peek_char ()) != -1){
1444                                 if (is_hex (d)){
1445                                         number_builder [number_pos++] = (char) d;
1446                                         get_char ();
1447                                 } else
1448                                         break;
1449                         }
1450
1451                         string s = new String (number_builder, 0, number_pos);
1452                         try {
1453                                 if (number_pos <= 8)
1454                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1455                                 else
1456                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1457                         } catch (OverflowException){
1458                                 Error_NumericConstantTooLong ();
1459                                 val = new IntLiteral (context.BuiltinTypes, 0, Location);
1460                                 return Token.LITERAL;
1461                         }
1462                         catch (FormatException) {
1463                                 Report.Error (1013, Location, "Invalid number");
1464                                 val = new IntLiteral (context.BuiltinTypes, 0, Location);
1465                                 return Token.LITERAL;
1466                         }
1467
1468                         return integer_type_suffix (ul, peek_char ());
1469                 }
1470
1471                 //
1472                 // Invoked if we know we have .digits or digits
1473                 //
1474                 int is_number (int c)
1475                 {
1476                         bool is_real = false;
1477
1478                         number_pos = 0;
1479
1480                         if (c >= '0' && c <= '9'){
1481                                 if (c == '0'){
1482                                         int peek = peek_char ();
1483
1484                                         if (peek == 'x' || peek == 'X')
1485                                                 return handle_hex ();
1486                                 }
1487                                 decimal_digits (c);
1488                                 c = get_char ();
1489                         }
1490
1491                         //
1492                         // We need to handle the case of
1493                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1494                         //
1495                         if (c == '.'){
1496                                 if (decimal_digits ('.')){
1497                                         is_real = true;
1498                                         c = get_char ();
1499                                 } else {
1500                                         putback ('.');
1501                                         number_pos--;
1502                                         return adjust_int (-1);
1503                                 }
1504                         }
1505
1506                         if (c == 'e' || c == 'E'){
1507                                 is_real = true;
1508                                 if (number_pos == max_number_size)
1509                                         Error_NumericConstantTooLong ();
1510                                 number_builder [number_pos++] = 'e';
1511                                 c = get_char ();
1512
1513                                 if (c == '+'){
1514                                         if (number_pos == max_number_size)
1515                                                 Error_NumericConstantTooLong ();
1516                                         number_builder [number_pos++] = '+';
1517                                         c = -1;
1518                                 } else if (c == '-') {
1519                                         if (number_pos == max_number_size)
1520                                                 Error_NumericConstantTooLong ();
1521                                         number_builder [number_pos++] = '-';
1522                                         c = -1;
1523                                 } else {
1524                                         if (number_pos == max_number_size)
1525                                                 Error_NumericConstantTooLong ();
1526                                         number_builder [number_pos++] = '+';
1527                                 }
1528
1529                                 decimal_digits (c);
1530                                 c = get_char ();
1531                         }
1532
1533                         var type = real_type_suffix (c);
1534                         if (type == TypeCode.Empty && !is_real){
1535                                 putback (c);
1536                                 return adjust_int (c);
1537                         }
1538
1539                         is_real = true;
1540
1541                         if (type == TypeCode.Empty){
1542                                 putback (c);
1543                         }
1544
1545                         if (is_real)
1546                                 return adjust_real (type);
1547
1548                         throw new Exception ("Is Number should never reach this point");
1549                 }
1550
1551                 //
1552                 // Accepts exactly count (4 or 8) hex, no more no less
1553                 //
1554                 int getHex (int count, out int surrogate, out bool error)
1555                 {
1556                         int i;
1557                         int total = 0;
1558                         int c;
1559                         int top = count != -1 ? count : 4;
1560
1561                         get_char ();
1562                         error = false;
1563                         surrogate = 0;
1564                         for (i = 0; i < top; i++){
1565                                 c = get_char ();
1566
1567                                 if (c >= '0' && c <= '9')
1568                                         c = (int) c - (int) '0';
1569                                 else if (c >= 'A' && c <= 'F')
1570                                         c = (int) c - (int) 'A' + 10;
1571                                 else if (c >= 'a' && c <= 'f')
1572                                         c = (int) c - (int) 'a' + 10;
1573                                 else {
1574                                         error = true;
1575                                         return 0;
1576                                 }
1577
1578                                 total = (total * 16) + c;
1579                                 if (count == -1){
1580                                         int p = peek_char ();
1581                                         if (p == -1)
1582                                                 break;
1583                                         if (!is_hex ((char)p))
1584                                                 break;
1585                                 }
1586                         }
1587
1588                         if (top == 8) {
1589                                 if (total > 0x0010FFFF) {
1590                                         error = true;
1591                                         return 0;
1592                                 }
1593
1594                                 if (total >= 0x00010000) {
1595                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1596                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1597                                 }
1598                         }
1599
1600                         return total;
1601                 }
1602
1603                 int escape (int c, out int surrogate)
1604                 {
1605                         bool error;
1606                         int d;
1607                         int v;
1608
1609                         d = peek_char ();
1610                         if (c != '\\') {
1611                                 surrogate = 0;
1612                                 return c;
1613                         }
1614
1615                         switch (d){
1616                         case 'a':
1617                                 v = '\a'; break;
1618                         case 'b':
1619                                 v = '\b'; break;
1620                         case 'n':
1621                                 v = '\n'; break;
1622                         case 't':
1623                                 v = '\t'; break;
1624                         case 'v':
1625                                 v = '\v'; break;
1626                         case 'r':
1627                                 v = '\r'; break;
1628                         case '\\':
1629                                 v = '\\'; break;
1630                         case 'f':
1631                                 v = '\f'; break;
1632                         case '0':
1633                                 v = 0; break;
1634                         case '"':
1635                                 v = '"'; break;
1636                         case '\'':
1637                                 v = '\''; break;
1638                         case 'x':
1639                                 v = getHex (-1, out surrogate, out error);
1640                                 if (error)
1641                                         goto default;
1642                                 return v;
1643                         case 'u':
1644                         case 'U':
1645                                 return EscapeUnicode (d, out surrogate);
1646                         default:
1647                                 surrogate = 0;
1648                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1649                                 return d;
1650                         }
1651
1652                         get_char ();
1653                         surrogate = 0;
1654                         return v;
1655                 }
1656
1657                 int EscapeUnicode (int ch, out int surrogate)
1658                 {
1659                         bool error;
1660                         if (ch == 'U') {
1661                                 ch = getHex (8, out surrogate, out error);
1662                         } else {
1663                                 ch = getHex (4, out surrogate, out error);
1664                         }
1665
1666                         if (error)
1667                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1668
1669                         return ch;
1670                 }
1671
1672                 int get_char ()
1673                 {
1674                         int x;
1675                         if (putback_char != -1) {
1676                                 x = putback_char;
1677                                 putback_char = -1;
1678                         } else {
1679                                 x = reader.Read ();
1680                         }
1681
1682                         if (x == '\r') {
1683                                 if (peek_char () == '\n') {
1684                                         putback_char = -1;
1685                                 }
1686
1687                                 x = '\n';
1688                                 advance_line ();
1689                         } else if (x == '\n') {
1690                                 advance_line ();
1691                         } else {
1692                                 col++;
1693                         }
1694                         return x;
1695                 }
1696
1697                 void advance_line ()
1698                 {
1699                         line++;
1700                         ref_line++;
1701                         previous_col = col;
1702                         col = 0;
1703                 }
1704
1705                 int peek_char ()
1706                 {
1707                         if (putback_char == -1)
1708                                 putback_char = reader.Read ();
1709                         return putback_char;
1710                 }
1711
1712                 int peek_char2 ()
1713                 {
1714                         if (putback_char != -1)
1715                                 return putback_char;
1716                         return reader.Peek ();
1717                 }
1718
1719                 void putback (int c)
1720                 {
1721                         if (putback_char != -1){
1722                                 Console.WriteLine ("Col: " + col);
1723                                 Console.WriteLine ("Row: " + line);
1724                                 Console.WriteLine ("Name: " + ref_name.Name);
1725                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1726                                 throw new Exception ("This should not happen putback on putback");
1727                         }
1728                         if (c == '\n' || col == 0) {
1729                                 // It won't happen though.
1730                                 line--;
1731                                 ref_line--;
1732                                 col = previous_col;
1733                         }
1734                         else
1735                                 col--;
1736                         putback_char = c;
1737                 }
1738
1739                 public bool advance ()
1740                 {
1741                         return peek_char () != -1 || CompleteOnEOF;
1742                 }
1743
1744                 public Object Value {
1745                         get {
1746                                 return val;
1747                         }
1748                 }
1749
1750                 public Object value ()
1751                 {
1752                         return val;
1753                 }
1754
1755                 public int token ()
1756                 {
1757                         current_token = xtoken ();
1758                         return current_token;
1759                 }
1760
1761                 int TokenizePreprocessorIdentifier (out int c)
1762                 {
1763                         // skip over white space
1764                         do {
1765                                 c = get_char ();
1766                         } while (c == ' ' || c == '\t');
1767
1768
1769                         int pos = 0;
1770                         while (c != -1 && c >= 'a' && c <= 'z') {
1771                                 id_builder[pos++] = (char) c;
1772                                 c = get_char ();
1773                                 if (c == '\\') {
1774                                         int peek = peek_char ();
1775                                         if (peek == 'U' || peek == 'u') {
1776                                                 int surrogate;
1777                                                 c = EscapeUnicode (c, out surrogate);
1778                                                 if (surrogate != 0) {
1779                                                         if (is_identifier_part_character ((char) c)) {
1780                                                                 id_builder[pos++] = (char) c;
1781                                                         }
1782                                                         c = surrogate;
1783                                                 }
1784                                         }
1785                                 }
1786                         }
1787
1788                         return pos;
1789                 }
1790
1791                 PreprocessorDirective get_cmd_arg (out string arg)
1792                 {
1793                         int c;
1794
1795                         tokens_seen = false;
1796                         arg = "";
1797
1798                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorIdentifier (out c));
1799
1800                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
1801                                 return cmd;
1802
1803                         // skip over white space
1804                         while (c == ' ' || c == '\t')
1805                                 c = get_char ();
1806
1807                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
1808                         int pos = 0;
1809
1810                         while (c != -1 && c != '\n') {
1811                                 if (c == '\\' && has_identifier_argument >= 0) {
1812                                         if (has_identifier_argument != 0) {
1813                                                 has_identifier_argument = 1;
1814
1815                                                 int peek = peek_char ();
1816                                                 if (peek == 'U' || peek == 'u') {
1817                                                         int surrogate;
1818                                                         c = EscapeUnicode (c, out surrogate);
1819                                                         if (surrogate != 0) {
1820                                                                 if (is_identifier_part_character ((char) c)) {
1821                                                                         if (pos == value_builder.Length)
1822                                                                                 Array.Resize (ref value_builder, pos * 2);
1823
1824                                                                         value_builder[pos++] = (char) c;
1825                                                                 }
1826                                                                 c = surrogate;
1827                                                         }
1828                                                 }
1829                                         } else {
1830                                                 has_identifier_argument = -1;
1831                                         }
1832                                 } else if (c == '/' && peek_char () == '/') {
1833                                         //
1834                                         // Eat single-line comments
1835                                         //
1836                                         get_char ();
1837                                         do {
1838                                                 c = get_char ();
1839                                         } while (c != -1 && c != '\n');
1840
1841                                         break;
1842                                 }
1843
1844                                 if (pos == value_builder.Length)
1845                                         Array.Resize (ref value_builder, pos * 2);
1846
1847                                 value_builder[pos++] = (char) c;
1848                                 c = get_char ();
1849                         }
1850
1851                         if (pos != 0) {
1852                                 if (pos > max_id_size)
1853                                         arg = new string (value_builder, 0, pos);
1854                                 else
1855                                         arg = InternIdentifier (value_builder, pos);
1856
1857                                 // Eat any trailing whitespaces
1858                                 arg = arg.Trim (simple_whitespaces);
1859                         }
1860
1861                         return cmd;
1862                 }
1863
1864                 //
1865                 // Handles the #line directive
1866                 //
1867                 bool PreProcessLine (string arg)
1868                 {
1869                         if (arg.Length == 0)
1870                                 return false;
1871
1872                         if (arg == "default"){
1873                                 ref_line = line;
1874                                 ref_name = file_name;
1875                                 hidden = false;
1876                                 Location.Push (file_name, ref_name);
1877                                 return true;
1878                         } else if (arg == "hidden"){
1879                                 hidden = true;
1880                                 return true;
1881                         }
1882
1883                         try {
1884                                 int pos;
1885
1886                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1887                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1888                                         pos++;
1889
1890                                         char [] quotes = { '\"' };
1891
1892                                         string name = arg.Substring (pos). Trim (quotes);
1893                                         ref_name = context.LookupFile (file_name, name);
1894                                         file_name.AddIncludeFile (ref_name);
1895                                         hidden = false;
1896                                         Location.Push (file_name, ref_name);
1897                                 } else {
1898                                         ref_line = System.Int32.Parse (arg);
1899                                         hidden = false;
1900                                 }
1901                         } catch {
1902                                 return false;
1903                         }
1904
1905                         return true;
1906                 }
1907
1908                 //
1909                 // Handles #define and #undef
1910                 //
1911                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
1912                 {
1913                         if (ident.Length == 0 || ident == "true" || ident == "false"){
1914                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
1915                                 return;
1916                         }
1917
1918                         if (ident.IndexOfAny (simple_whitespaces) != -1){
1919                                 Error_EndLineExpected ();
1920                                 return;
1921                         }
1922
1923                         if (!is_identifier_start_character (ident [0]))
1924                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
1925
1926                         foreach (char c in ident.Substring (1)){
1927                                 if (!is_identifier_part_character (c)){
1928                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
1929                                         return;
1930                                 }
1931                         }
1932
1933                         if (!caller_is_taking)
1934                                 return;
1935
1936                         if (is_define) {
1937                                 //
1938                                 // #define ident
1939                                 //
1940                                 if (context.Settings.IsConditionalSymbolDefined (ident))
1941                                         return;
1942
1943                                 file_name.AddDefine (ident);
1944                         } else {
1945                                 //
1946                                 // #undef ident
1947                                 //
1948                                 file_name.AddUndefine (ident);
1949                         }
1950                 }
1951
1952                 byte read_hex (out bool error)
1953                 {
1954                         int total;
1955                         int c = get_char ();
1956
1957                         if ((c >= '0') && (c <= '9'))
1958                                 total = (int) c - (int) '0';
1959                         else if ((c >= 'A') && (c <= 'F'))
1960                                 total = (int) c - (int) 'A' + 10;
1961                         else if ((c >= 'a') && (c <= 'f'))
1962                                 total = (int) c - (int) 'a' + 10;
1963                         else {
1964                                 error = true;
1965                                 return 0;
1966                         }
1967
1968                         total *= 16;
1969                         c = get_char ();
1970
1971                         if ((c >= '0') && (c <= '9'))
1972                                 total += (int) c - (int) '0';
1973                         else if ((c >= 'A') && (c <= 'F'))
1974                                 total += (int) c - (int) 'A' + 10;
1975                         else if ((c >= 'a') && (c <= 'f'))
1976                                 total += (int) c - (int) 'a' + 10;
1977                         else {
1978                                 error = true;
1979                                 return 0;
1980                         }
1981
1982                         error = false;
1983                         return (byte) total;
1984                 }
1985
1986                 //
1987                 // Parses #pragma checksum
1988                 //
1989                 bool ParsePragmaChecksum ()
1990                 {
1991                         //
1992                         // The syntax is ` "foo.txt" "{guid}" "hash"'
1993                         //
1994                         int c = get_char ();
1995
1996                         if (c != '"')
1997                                 return false;
1998
1999                         string_builder.Length = 0;
2000                         while (c != -1 && c != '\n') {
2001                                 c = get_char ();
2002                                 if (c == '"') {
2003                                         c = get_char ();
2004                                         break;
2005                                 }
2006
2007                                 string_builder.Append ((char) c);
2008                         }
2009
2010                         if (string_builder.Length == 0) {
2011                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
2012                         }
2013
2014                         // TODO: Any white-spaces count
2015                         if (c != ' ')
2016                                 return false;
2017
2018                         SourceFile file = context.LookupFile (file_name, string_builder.ToString ());
2019
2020                         if (get_char () != '"' || get_char () != '{')
2021                                 return false;
2022
2023                         bool error;
2024                         byte[] guid_bytes = new byte [16];
2025                         int i = 0;
2026
2027                         for (; i < 4; i++) {
2028                                 guid_bytes [i] = read_hex (out error);
2029                                 if (error)
2030                                         return false;
2031                         }
2032
2033                         if (get_char () != '-')
2034                                 return false;
2035
2036                         for (; i < 10; i++) {
2037                                 guid_bytes [i] = read_hex (out error);
2038                                 if (error)
2039                                         return false;
2040
2041                                 guid_bytes [i++] = read_hex (out error);
2042                                 if (error)
2043                                         return false;
2044
2045                                 if (get_char () != '-')
2046                                         return false;
2047                         }
2048
2049                         for (; i < 16; i++) {
2050                                 guid_bytes [i] = read_hex (out error);
2051                                 if (error)
2052                                         return false;
2053                         }
2054
2055                         if (get_char () != '}' || get_char () != '"')
2056                                 return false;
2057
2058                         // TODO: Any white-spaces count
2059                         c = get_char ();
2060                         if (c != ' ')
2061                                 return false;
2062
2063                         if (get_char () != '"')
2064                                 return false;
2065
2066                         // Any length of checksum
2067                         List<byte> checksum_bytes = new List<byte> (16);
2068
2069                         c = peek_char ();
2070                         while (c != '"' && c != -1) {
2071                                 checksum_bytes.Add (read_hex (out error));
2072                                 if (error)
2073                                         return false;
2074
2075                                 c = peek_char ();
2076                         }
2077
2078                         if (c == '/') {
2079                                 ReadSingleLineComment ();
2080                         } else if (get_char () != '"') {
2081                                 return false;
2082                         }
2083
2084                         file.SetChecksum (guid_bytes, checksum_bytes.ToArray ());
2085                         ref_name.AutoGenerated = true;
2086                         return true;
2087                 }
2088
2089                 bool IsTokenIdentifierEqual (char[] identifier)
2090                 {
2091                         for (int i = 0; i < identifier.Length; ++i) {
2092                                 if (identifier[i] != id_builder[i])
2093                                         return false;
2094                         }
2095
2096                         return true;
2097                 }
2098
2099                 int TokenizePragmaNumber (ref int c)
2100                 {
2101                         number_pos = 0;
2102
2103                         int number;
2104
2105                         if (c >= '0' && c <= '9') {
2106                                 decimal_digits (c);
2107                                 uint ui = (uint) (number_builder[0] - '0');
2108
2109                                 try {
2110                                         for (int i = 1; i < number_pos; i++) {
2111                                                 ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2112                                         }
2113
2114                                         number = (int) ui;
2115                                 } catch (OverflowException) {
2116                                         Error_NumericConstantTooLong ();
2117                                         number = -1;
2118                                 }
2119
2120
2121                                 c = get_char ();
2122
2123                                 // skip over white space
2124                                 while (c == ' ' || c == '\t')
2125                                         c = get_char ();
2126
2127                                 if (c == ',') {
2128                                         c = get_char ();
2129                                 }
2130
2131                                 // skip over white space
2132                                 while (c == ' ' || c == '\t')
2133                                         c = get_char ();
2134                         } else {
2135                                 number = -1;
2136                                 if (c == '/') {
2137                                         ReadSingleLineComment ();
2138                                 } else {
2139                                         Report.Warning (1692, 1, Location, "Invalid number");
2140
2141                                         // Read everything till the end of the line or file
2142                                         do {
2143                                                 c = get_char ();
2144                                         } while (c != -1 && c != '\n');
2145                                 }
2146                         }
2147
2148                         return number;
2149                 }
2150
2151                 void ReadSingleLineComment ()
2152                 {
2153                         if (peek_char () != '/')
2154                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2155
2156                         // Read everything till the end of the line or file
2157                         int c;
2158                         do {
2159                                 c = get_char ();
2160                         } while (c != -1 && c != '\n');
2161                 }
2162
2163                 /// <summary>
2164                 /// Handles #pragma directive
2165                 /// </summary>
2166                 void ParsePragmaDirective (string arg)
2167                 {
2168                         int c;
2169                         int length = TokenizePreprocessorIdentifier (out c);
2170                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2171                                 length = TokenizePreprocessorIdentifier (out c);
2172
2173                                 //
2174                                 // #pragma warning disable
2175                                 // #pragma warning restore
2176                                 //
2177                                 if (length == pragma_warning_disable.Length) {
2178                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2179                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2180                                                 // skip over white space
2181                                                 while (c == ' ' || c == '\t')
2182                                                         c = get_char ();
2183
2184                                                 var loc = Location;
2185
2186                                                 if (c == '\n' || c == '/') {
2187                                                         if (c == '/')
2188                                                                 ReadSingleLineComment ();
2189
2190                                                         //
2191                                                         // Disable/Restore all warnings
2192                                                         //
2193                                                         if (disable) {
2194                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2195                                                         } else {
2196                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2197                                                         }
2198                                                 } else {
2199                                                         //
2200                                                         // Disable/Restore a warning or group of warnings
2201                                                         //
2202                                                         int code;
2203                                                         do {
2204                                                                 code = TokenizePragmaNumber (ref c);
2205                                                                 if (code > 0) {
2206                                                                         if (disable) {
2207                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, Report);
2208                                                                         } else {
2209                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, Report);
2210                                                                         }
2211                                                                 }
2212                                                         } while (code >= 0 && c != '\n' && c != -1);
2213                                                 }
2214
2215                                                 return;
2216                                         }
2217                                 }
2218
2219                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2220                                 return;
2221                         }
2222
2223                         //
2224                         // #pragma checksum
2225                         //
2226                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2227                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2228                                         Report.Warning (1695, 1, Location,
2229                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2230                                 }
2231
2232                                 return;
2233                         }
2234
2235                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2236                 }
2237
2238                 bool eval_val (string s)
2239                 {
2240                         if (s == "true")
2241                                 return true;
2242                         if (s == "false")
2243                                 return false;
2244
2245                         return file_name.IsConditionalDefined (context, s);
2246                 }
2247
2248                 bool pp_primary (ref string s)
2249                 {
2250                         s = s.Trim ();
2251                         int len = s.Length;
2252
2253                         if (len > 0){
2254                                 char c = s [0];
2255
2256                                 if (c == '('){
2257                                         s = s.Substring (1);
2258                                         bool val = pp_expr (ref s, false);
2259                                         if (s.Length > 0 && s [0] == ')'){
2260                                                 s = s.Substring (1);
2261                                                 return val;
2262                                         }
2263                                         Error_InvalidDirective ();
2264                                         return false;
2265                                 }
2266
2267                                 if (is_identifier_start_character (c)){
2268                                         int j = 1;
2269
2270                                         while (j < len){
2271                                                 c = s [j];
2272
2273                                                 if (is_identifier_part_character (c)){
2274                                                         j++;
2275                                                         continue;
2276                                                 }
2277                                                 bool v = eval_val (s.Substring (0, j));
2278                                                 s = s.Substring (j);
2279                                                 return v;
2280                                         }
2281                                         bool vv = eval_val (s);
2282                                         s = "";
2283                                         return vv;
2284                                 }
2285                         }
2286                         Error_InvalidDirective ();
2287                         return false;
2288                 }
2289
2290                 bool pp_unary (ref string s)
2291                 {
2292                         s = s.Trim ();
2293                         int len = s.Length;
2294
2295                         if (len > 0){
2296                                 if (s [0] == '!'){
2297                                         if (len > 1 && s [1] == '='){
2298                                                 Error_InvalidDirective ();
2299                                                 return false;
2300                                         }
2301                                         s = s.Substring (1);
2302                                         return ! pp_primary (ref s);
2303                                 } else
2304                                         return pp_primary (ref s);
2305                         } else {
2306                                 Error_InvalidDirective ();
2307                                 return false;
2308                         }
2309                 }
2310
2311                 bool pp_eq (ref string s)
2312                 {
2313                         bool va = pp_unary (ref s);
2314
2315                         s = s.Trim ();
2316                         int len = s.Length;
2317                         if (len > 0){
2318                                 if (s [0] == '='){
2319                                         if (len > 2 && s [1] == '='){
2320                                                 s = s.Substring (2);
2321                                                 return va == pp_unary (ref s);
2322                                         } else {
2323                                                 Error_InvalidDirective ();
2324                                                 return false;
2325                                         }
2326                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2327                                         s = s.Substring (2);
2328
2329                                         return va != pp_unary (ref s);
2330
2331                                 }
2332                         }
2333
2334                         return va;
2335
2336                 }
2337
2338                 bool pp_and (ref string s)
2339                 {
2340                         bool va = pp_eq (ref s);
2341
2342                         s = s.Trim ();
2343                         int len = s.Length;
2344                         if (len > 0){
2345                                 if (s [0] == '&'){
2346                                         if (len > 2 && s [1] == '&'){
2347                                                 s = s.Substring (2);
2348                                                 return (va & pp_and (ref s));
2349                                         } else {
2350                                                 Error_InvalidDirective ();
2351                                                 return false;
2352                                         }
2353                                 }
2354                         }
2355                         return va;
2356                 }
2357
2358                 //
2359                 // Evaluates an expression for `#if' or `#elif'
2360                 //
2361                 bool pp_expr (ref string s, bool isTerm)
2362                 {
2363                         bool va = pp_and (ref s);
2364                         s = s.Trim ();
2365                         int len = s.Length;
2366                         if (len > 0){
2367                                 char c = s [0];
2368
2369                                 if (c == '|'){
2370                                         if (len > 2 && s [1] == '|'){
2371                                                 s = s.Substring (2);
2372                                                 return va | pp_expr (ref s, isTerm);
2373                                         } else {
2374                                                 Error_InvalidDirective ();
2375                                                 return false;
2376                                         }
2377                                 }
2378                                 if (isTerm) {
2379                                         Error_EndLineExpected ();
2380                                         return false;
2381                                 }
2382                         }
2383
2384                         return va;
2385                 }
2386
2387                 bool eval (string s)
2388                 {
2389                         bool v = pp_expr (ref s, true);
2390                         s = s.Trim ();
2391                         if (s.Length != 0){
2392                                 return false;
2393                         }
2394
2395                         return v;
2396                 }
2397
2398                 void Error_NumericConstantTooLong ()
2399                 {
2400                         Report.Error (1021, Location, "Integral constant is too large");
2401                 }
2402
2403                 void Error_InvalidDirective ()
2404                 {
2405                         Report.Error (1517, Location, "Invalid preprocessor directive");
2406                 }
2407
2408                 void Error_UnexpectedDirective (string extra)
2409                 {
2410                         Report.Error (
2411                                 1028, Location,
2412                                 "Unexpected processor directive ({0})", extra);
2413                 }
2414
2415                 void Error_TokensSeen ()
2416                 {
2417                         Report.Error (1032, Location,
2418                                 "Cannot define or undefine preprocessor symbols after first token in file");
2419                 }
2420
2421                 void Eror_WrongPreprocessorLocation ()
2422                 {
2423                         Report.Error (1040, Location,
2424                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2425                 }
2426
2427                 void Error_EndLineExpected ()
2428                 {
2429                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2430                 }
2431
2432                 //
2433                 // Raises a warning when tokenizer found documentation comment
2434                 // on unexpected place
2435                 //
2436                 void WarningMisplacedComment (Location loc)
2437                 {
2438                         if (doc_state != XmlCommentState.Error) {
2439                                 doc_state = XmlCommentState.Error;
2440                                 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element");
2441                         }
2442                 }
2443
2444                 //
2445                 // if true, then the code continues processing the code
2446                 // if false, the code stays in a loop until another directive is
2447                 // reached.
2448                 // When caller_is_taking is false we ignore all directives except the ones
2449                 // which can help us to identify where the #if block ends
2450                 bool ParsePreprocessingDirective (bool caller_is_taking)
2451                 {
2452                         string arg;
2453                         bool region_directive = false;
2454
2455                         var directive = get_cmd_arg (out arg);
2456
2457                         //
2458                         // The first group of pre-processing instructions is always processed
2459                         //
2460                         switch (directive) {
2461                         case PreprocessorDirective.Region:
2462                                 region_directive = true;
2463                                 arg = "true";
2464                                 goto case PreprocessorDirective.If;
2465
2466                         case PreprocessorDirective.Endregion:
2467                                 if (ifstack == null || ifstack.Count == 0){
2468                                         Error_UnexpectedDirective ("no #region for this #endregion");
2469                                         return true;
2470                                 }
2471                                 int pop = ifstack.Pop ();
2472
2473                                 if ((pop & REGION) == 0)
2474                                         Report.Error (1027, Location, "Expected `#endif' directive");
2475
2476                                 return caller_is_taking;
2477
2478                         case PreprocessorDirective.If:
2479                                 if (ifstack == null)
2480                                         ifstack = new Stack<int> (2);
2481
2482                                 int flags = region_directive ? REGION : 0;
2483                                 if (ifstack.Count == 0){
2484                                         flags |= PARENT_TAKING;
2485                                 } else {
2486                                         int state = ifstack.Peek ();
2487                                         if ((state & TAKING) != 0) {
2488                                                 flags |= PARENT_TAKING;
2489                                         }
2490                                 }
2491
2492                                 if (eval (arg) && caller_is_taking) {
2493                                         ifstack.Push (flags | TAKING);
2494                                         return true;
2495                                 }
2496                                 ifstack.Push (flags);
2497                                 return false;
2498
2499                         case PreprocessorDirective.Endif:
2500                                 if (ifstack == null || ifstack.Count == 0){
2501                                         Error_UnexpectedDirective ("no #if for this #endif");
2502                                         return true;
2503                                 } else {
2504                                         pop = ifstack.Pop ();
2505
2506                                         if ((pop & REGION) != 0)
2507                                                 Report.Error (1038, Location, "#endregion directive expected");
2508
2509                                         if (arg.Length != 0) {
2510                                                 Error_EndLineExpected ();
2511                                         }
2512
2513                                         if (ifstack.Count == 0)
2514                                                 return true;
2515
2516                                         int state = ifstack.Peek ();
2517                                         return (state & TAKING) != 0;
2518                                 }
2519
2520                         case PreprocessorDirective.Elif:
2521                                 if (ifstack == null || ifstack.Count == 0){
2522                                         Error_UnexpectedDirective ("no #if for this #elif");
2523                                         return true;
2524                                 } else {
2525                                         int state = ifstack.Pop ();
2526
2527                                         if ((state & REGION) != 0) {
2528                                                 Report.Error (1038, Location, "#endregion directive expected");
2529                                                 return true;
2530                                         }
2531
2532                                         if ((state & ELSE_SEEN) != 0){
2533                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2534                                                 return true;
2535                                         }
2536
2537                                         if ((state & TAKING) != 0) {
2538                                                 ifstack.Push (0);
2539                                                 return false;
2540                                         }
2541
2542                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2543                                                 ifstack.Push (state | TAKING);
2544                                                 return true;
2545                                         }
2546
2547                                         ifstack.Push (state);
2548                                         return false;
2549                                 }
2550
2551                         case PreprocessorDirective.Else:
2552                                 if (ifstack == null || ifstack.Count == 0){
2553                                         Error_UnexpectedDirective ("no #if for this #else");
2554                                         return true;
2555                                 } else {
2556                                         int state = ifstack.Peek ();
2557
2558                                         if ((state & REGION) != 0) {
2559                                                 Report.Error (1038, Location, "#endregion directive expected");
2560                                                 return true;
2561                                         }
2562
2563                                         if ((state & ELSE_SEEN) != 0){
2564                                                 Error_UnexpectedDirective ("#else within #else");
2565                                                 return true;
2566                                         }
2567
2568                                         ifstack.Pop ();
2569
2570                                         if (arg.Length != 0) {
2571                                                 Error_EndLineExpected ();
2572                                                 return true;
2573                                         }
2574
2575                                         bool ret = false;
2576                                         if ((state & PARENT_TAKING) != 0) {
2577                                                 ret = (state & TAKING) == 0;
2578
2579                                                 if (ret)
2580                                                         state |= TAKING;
2581                                                 else
2582                                                         state &= ~TAKING;
2583                                         }
2584
2585                                         ifstack.Push (state | ELSE_SEEN);
2586
2587                                         return ret;
2588                                 }
2589                         case PreprocessorDirective.Define:
2590                                 if (any_token_seen){
2591                                         Error_TokensSeen ();
2592                                         return caller_is_taking;
2593                                 }
2594                                 PreProcessDefinition (true, arg, caller_is_taking);
2595                                 return caller_is_taking;
2596
2597                         case PreprocessorDirective.Undef:
2598                                 if (any_token_seen){
2599                                         Error_TokensSeen ();
2600                                         return caller_is_taking;
2601                                 }
2602                                 PreProcessDefinition (false, arg, caller_is_taking);
2603                                 return caller_is_taking;
2604
2605                         case PreprocessorDirective.Invalid:
2606                                 Report.Error (1024, Location, "Wrong preprocessor directive");
2607                                 return true;
2608                         }
2609
2610                         //
2611                         // These are only processed if we are in a `taking' block
2612                         //
2613                         if (!caller_is_taking)
2614                                 return false;
2615
2616                         switch (directive){
2617                         case PreprocessorDirective.Error:
2618                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2619                                 return true;
2620
2621                         case PreprocessorDirective.Warning:
2622                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2623                                 return true;
2624
2625                         case PreprocessorDirective.Pragma:
2626                                 if (context.Settings.Version == LanguageVersion.ISO_1) {
2627                                         Report.FeatureIsNotAvailable (context, Location, "#pragma");
2628                                 }
2629
2630                                 ParsePragmaDirective (arg);
2631                                 return true;
2632
2633                         case PreprocessorDirective.Line:
2634                                 if (!PreProcessLine (arg))
2635                                         Report.Error (
2636                                                 1576, Location,
2637                                                 "The line number specified for #line directive is missing or invalid");
2638                                 return caller_is_taking;
2639                         }
2640
2641                         throw new NotImplementedException (directive.ToString ());
2642                 }
2643
2644                 private int consume_string (bool quoted)
2645                 {
2646                         int c;
2647                         int pos = 0;
2648                         Location start_location = Location;
2649                         while (true){
2650                                 c = get_char ();
2651                                 if (c == '"') {
2652                                         if (quoted && peek_char () == '"') {
2653                                                 if (pos == value_builder.Length)
2654                                                         Array.Resize (ref value_builder, pos * 2);
2655
2656                                                 value_builder[pos++] = (char) c;
2657                                                 get_char ();
2658                                                 continue;
2659                                         }
2660
2661                                         string s;
2662                                         if (pos == 0)
2663                                                 s = string.Empty;
2664                                         else if (pos <= 4)
2665                                                 s = InternIdentifier (value_builder, pos);
2666                                         else
2667                                                 s = new string (value_builder, 0, pos);
2668
2669                                         val = new StringLiteral (context.BuiltinTypes, s, start_location);
2670                                         return Token.LITERAL;
2671                                 }
2672
2673                                 if (c == '\n') {
2674                                         if (!quoted)
2675                                                 Report.Error (1010, Location, "Newline in constant");
2676                                 } else if (c == '\\' && !quoted) {
2677                                         int surrogate;
2678                                         c = escape (c, out surrogate);
2679                                         if (c == -1)
2680                                                 return Token.ERROR;
2681                                         if (surrogate != 0) {
2682                                                 if (pos == value_builder.Length)
2683                                                         Array.Resize (ref value_builder, pos * 2);
2684
2685                                                 value_builder[pos++] = (char) c;
2686                                                 c = surrogate;
2687                                         }
2688                                 } else if (c == -1) {
2689                                         Report.Error (1039, Location, "Unterminated string literal");
2690                                         return Token.EOF;
2691                                 }
2692
2693                                 if (pos == value_builder.Length)
2694                                         Array.Resize (ref value_builder, pos * 2);
2695
2696                                 value_builder[pos++] = (char) c;
2697                         }
2698                 }
2699
2700                 private int consume_identifier (int s)
2701                 {
2702                         int res = consume_identifier (s, false);
2703
2704                         if (doc_state == XmlCommentState.Allowed)
2705                                 doc_state = XmlCommentState.NotAllowed;
2706
2707                         return res;
2708                 }
2709
2710                 int consume_identifier (int c, bool quoted)
2711                 {
2712                         //
2713                         // This method is very performance sensitive. It accounts
2714                         // for approximately 25% of all parser time
2715                         //
2716
2717                         int pos = 0;
2718                         int column = col;
2719
2720                         if (c == '\\') {
2721                                 int surrogate;
2722                                 c = escape (c, out surrogate);
2723                                 if (surrogate != 0) {
2724                                         id_builder [pos++] = (char) c;
2725                                         c = surrogate;
2726                                 }
2727                         }
2728
2729                         id_builder [pos++] = (char) c;
2730
2731                         try {
2732                                 while (true) {
2733                                         c = reader.Read ();
2734
2735                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
2736                                                 id_builder [pos++] = (char) c;
2737                                                 continue;
2738                                         }
2739
2740                                         if (c < 0x80) {
2741                                                 if (c == '\\') {
2742                                                         int surrogate;
2743                                                         c = escape (c, out surrogate);
2744                                                         if (surrogate != 0) {
2745                                                                 if (is_identifier_part_character ((char) c))
2746                                                                         id_builder[pos++] = (char) c;
2747                                                                 c = surrogate;
2748                                                         }
2749
2750                                                         continue;
2751                                                 }
2752                                         } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
2753                                                 id_builder [pos++] = (char) c;
2754                                                 continue;
2755                                         }
2756
2757                                         putback_char = c;
2758                                         break;
2759                                 }
2760                         } catch (IndexOutOfRangeException) {
2761                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2762                                 --pos;
2763                                 col += pos;
2764                         }
2765
2766                         col += pos - 1;
2767
2768                         //
2769                         // Optimization: avoids doing the keyword lookup
2770                         // on uppercase letters
2771                         //
2772                         if (id_builder [0] >= '_' && !quoted) {
2773                                 int keyword = GetKeyword (id_builder, pos);
2774                                 if (keyword != -1) {
2775                                         val = LocatedToken.Create (null, ref_line, column);
2776                                         return keyword;
2777                                 }
2778                         }
2779
2780                         string s = InternIdentifier (id_builder, pos);
2781                         val = LocatedToken.Create (s, ref_line, column);
2782                         if (quoted && parsing_attribute_section)
2783                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2784
2785                         return Token.IDENTIFIER;
2786                 }
2787
2788                 static string InternIdentifier (char[] charBuffer, int length)
2789                 {
2790                         //
2791                         // Keep identifiers in an array of hashtables to avoid needless
2792                         // allocations
2793                         //
2794                         var identifiers_group = identifiers[length];
2795                         string s;
2796                         if (identifiers_group != null) {
2797                                 if (identifiers_group.TryGetValue (charBuffer, out s)) {
2798                                         return s;
2799                                 }
2800                         } else {
2801                                 // TODO: this should be number of files dependant
2802                                 // corlib compilation peaks at 1000 and System.Core at 150
2803                                 int capacity = length > 20 ? 10 : 100;
2804                                 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length));
2805                                 identifiers[length] = identifiers_group;
2806                         }
2807
2808                         char[] chars = new char[length];
2809                         Array.Copy (charBuffer, chars, length);
2810
2811                         s = new string (charBuffer, 0, length);
2812                         identifiers_group.Add (chars, s);
2813                         return s;
2814                 }
2815
2816                 public int xtoken ()
2817                 {
2818                         int d, c;
2819
2820                         // Whether we have seen comments on the current line
2821                         bool comments_seen = false;
2822                         while ((c = get_char ()) != -1) {
2823                                 switch (c) {
2824                                 case '\t':
2825                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
2826                                         continue;
2827
2828                                 case ' ':
2829                                 case '\f':
2830                                 case '\v':
2831                                 case 0xa0:
2832                                 case 0:
2833                                 case 0xFEFF:    // Ignore BOM anywhere in the file
2834                                         continue;
2835
2836 /*                              This is required for compatibility with .NET
2837                                 case 0xEF:
2838                                         if (peek_char () == 0xBB) {
2839                                                 PushPosition ();
2840                                                 get_char ();
2841                                                 if (get_char () == 0xBF)
2842                                                         continue;
2843                                                 PopPosition ();
2844                                         }
2845                                         break;
2846 */
2847                                 case '\\':
2848                                         tokens_seen = true;
2849                                         return consume_identifier (c);
2850
2851                                 case '{':
2852                                         val = LocatedToken.Create (ref_line, col);
2853                                         return Token.OPEN_BRACE;
2854                                 case '}':
2855                                         val = LocatedToken.Create (ref_line, col);
2856                                         return Token.CLOSE_BRACE;
2857                                 case '[':
2858                                         // To block doccomment inside attribute declaration.
2859                                         if (doc_state == XmlCommentState.Allowed)
2860                                                 doc_state = XmlCommentState.NotAllowed;
2861
2862                                         val = LocatedToken.Create (ref_line, col);
2863
2864                                         if (parsing_block == 0 || lambda_arguments_parsing)
2865                                                 return Token.OPEN_BRACKET;
2866
2867                                         int next = peek_char ();
2868                                         switch (next) {
2869                                         case ']':
2870                                         case ',':
2871                                                 return Token.OPEN_BRACKET;
2872
2873                                         case ' ':
2874                                         case '\f':
2875                                         case '\v':
2876                                         case '\r':
2877                                         case '\n':
2878                                         case '/':
2879                                                 next = peek_token ();
2880                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
2881                                                         return Token.OPEN_BRACKET;
2882
2883                                                 return Token.OPEN_BRACKET_EXPR;
2884                                         default:
2885                                                 return Token.OPEN_BRACKET_EXPR;
2886                                         }
2887                                 case ']':
2888                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2889                                         return Token.CLOSE_BRACKET;
2890                                 case '(':
2891                                         val = LocatedToken.Create (ref_line, col);
2892                                         //
2893                                         // An expression versions of parens can appear in block context only
2894                                         //
2895                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
2896
2897                                                 //
2898                                                 // Optmize most common case where we know that parens
2899                                                 // is not special
2900                                                 //
2901                                                 switch (current_token) {
2902                                                 case Token.IDENTIFIER:
2903                                                 case Token.IF:
2904                                                 case Token.FOR:
2905                                                 case Token.FOREACH:
2906                                                 case Token.TYPEOF:
2907                                                 case Token.WHILE:
2908                                                 case Token.USING:
2909                                                 case Token.DEFAULT:
2910                                                 case Token.DELEGATE:
2911                                                 case Token.OP_GENERICS_GT:
2912                                                         return Token.OPEN_PARENS;
2913                                                 }
2914
2915                                                 // Optimize using peek
2916                                                 int xx = peek_char ();
2917                                                 switch (xx) {
2918                                                 case '(':
2919                                                 case '\'':
2920                                                 case '"':
2921                                                 case '0':
2922                                                 case '1':
2923                                                         return Token.OPEN_PARENS;
2924                                                 }
2925
2926                                                 lambda_arguments_parsing = true;
2927                                                 PushPosition ();
2928                                                 d = TokenizeOpenParens ();
2929                                                 PopPosition ();
2930                                                 lambda_arguments_parsing = false;
2931                                                 return d;
2932                                         }
2933
2934                                         return Token.OPEN_PARENS;
2935                                 case ')':
2936                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2937                                         return Token.CLOSE_PARENS;
2938                                 case ',':
2939                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2940                                         return Token.COMMA;
2941                                 case ';':
2942                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2943                                         return Token.SEMICOLON;
2944                                 case '~':
2945                                         val = LocatedToken.Create (ref_line, col);
2946                                         return Token.TILDE;
2947                                 case '?':
2948                                         val = LocatedToken.Create (ref_line, col);
2949                                         return TokenizePossibleNullableType ();
2950                                 case '<':
2951                                         val = LocatedToken.Create (ref_line, col);
2952                                         if (parsing_generic_less_than++ > 0)
2953                                                 return Token.OP_GENERICS_LT;
2954
2955                                         return TokenizeLessThan ();
2956
2957                                 case '>':
2958                                         val = LocatedToken.Create (ref_line, col);
2959                                         d = peek_char ();
2960
2961                                         if (d == '='){
2962                                                 get_char ();
2963                                                 return Token.OP_GE;
2964                                         }
2965
2966                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
2967                                                 parsing_generic_less_than--;
2968                                                 return Token.OP_GENERICS_GT;
2969                                         }
2970
2971                                         if (d == '>') {
2972                                                 get_char ();
2973                                                 d = peek_char ();
2974
2975                                                 if (d == '=') {
2976                                                         get_char ();
2977                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
2978                                                 }
2979                                                 return Token.OP_SHIFT_RIGHT;
2980                                         }
2981
2982                                         return Token.OP_GT;
2983
2984                                 case '+':
2985                                         val = LocatedToken.Create (ref_line, col);
2986                                         d = peek_char ();
2987                                         if (d == '+') {
2988                                                 d = Token.OP_INC;
2989                                         } else if (d == '=') {
2990                                                 d = Token.OP_ADD_ASSIGN;
2991                                         } else {
2992                                                 return Token.PLUS;
2993                                         }
2994                                         get_char ();
2995                                         return d;
2996
2997                                 case '-':
2998                                         val = LocatedToken.Create (ref_line, col);
2999                                         d = peek_char ();
3000                                         if (d == '-') {
3001                                                 d = Token.OP_DEC;
3002                                         } else if (d == '=')
3003                                                 d = Token.OP_SUB_ASSIGN;
3004                                         else if (d == '>')
3005                                                 d = Token.OP_PTR;
3006                                         else {
3007                                                 return Token.MINUS;
3008                                         }
3009                                         get_char ();
3010                                         return d;
3011
3012                                 case '!':
3013                                         val = LocatedToken.Create (ref_line, col);
3014                                         if (peek_char () == '='){
3015                                                 get_char ();
3016                                                 return Token.OP_NE;
3017                                         }
3018                                         return Token.BANG;
3019
3020                                 case '=':
3021                                         val = LocatedToken.Create (ref_line, col);
3022                                         d = peek_char ();
3023                                         if (d == '='){
3024                                                 get_char ();
3025                                                 return Token.OP_EQ;
3026                                         }
3027                                         if (d == '>'){
3028                                                 get_char ();
3029                                                 return Token.ARROW;
3030                                         }
3031
3032                                         return Token.ASSIGN;
3033
3034                                 case '&':
3035                                         val = LocatedToken.Create (ref_line, col);
3036                                         d = peek_char ();
3037                                         if (d == '&'){
3038                                                 get_char ();
3039                                                 return Token.OP_AND;
3040                                         }
3041                                         if (d == '='){
3042                                                 get_char ();
3043                                                 return Token.OP_AND_ASSIGN;
3044                                         }
3045                                         return Token.BITWISE_AND;
3046
3047                                 case '|':
3048                                         val = LocatedToken.Create (ref_line, col);
3049                                         d = peek_char ();
3050                                         if (d == '|'){
3051                                                 get_char ();
3052                                                 return Token.OP_OR;
3053                                         }
3054                                         if (d == '='){
3055                                                 get_char ();
3056                                                 return Token.OP_OR_ASSIGN;
3057                                         }
3058                                         return Token.BITWISE_OR;
3059
3060                                 case '*':
3061                                         val = LocatedToken.Create (ref_line, col);
3062                                         if (peek_char () == '='){
3063                                                 get_char ();
3064                                                 return Token.OP_MULT_ASSIGN;
3065                                         }
3066                                         return Token.STAR;
3067
3068                                 case '/':
3069                                         d = peek_char ();
3070                                         if (d == '='){
3071                                                 val = LocatedToken.Create (ref_line, col);
3072                                                 get_char ();
3073                                                 return Token.OP_DIV_ASSIGN;
3074                                         }
3075
3076                                         // Handle double-slash comments.
3077                                         if (d == '/'){
3078                                                 get_char ();
3079                                                 if (doc_processing && peek_char () == '/') {
3080                                                         get_char ();
3081                                                         // Don't allow ////.
3082                                                         if ((d = peek_char ()) != '/') {
3083                                                                 if (doc_state == XmlCommentState.Allowed)
3084                                                                         handle_one_line_xml_comment ();
3085                                                                 else if (doc_state == XmlCommentState.NotAllowed)
3086                                                                         WarningMisplacedComment (Location - 3);
3087                                                         }
3088                                                 }
3089
3090                                                 while ((d = get_char ()) != -1 && d != '\n');
3091
3092                                                 any_token_seen |= tokens_seen;
3093                                                 tokens_seen = false;
3094                                                 comments_seen = false;
3095                                                 continue;
3096                                         } else if (d == '*'){
3097                                                 get_char ();
3098                                                 bool docAppend = false;
3099                                                 if (doc_processing && peek_char () == '*') {
3100                                                         get_char ();
3101                                                         // But when it is /**/, just do nothing.
3102                                                         if (peek_char () == '/') {
3103                                                                 get_char ();
3104                                                                 continue;
3105                                                         }
3106                                                         if (doc_state == XmlCommentState.Allowed)
3107                                                                 docAppend = true;
3108                                                         else if (doc_state == XmlCommentState.NotAllowed) {
3109                                                                 WarningMisplacedComment (Location - 2);
3110                                                         }
3111                                                 }
3112
3113                                                 int current_comment_start = 0;
3114                                                 if (docAppend) {
3115                                                         current_comment_start = xml_comment_buffer.Length;
3116                                                         xml_comment_buffer.Append (Environment.NewLine);
3117                                                 }
3118
3119                                                 while ((d = get_char ()) != -1){
3120                                                         if (d == '*' && peek_char () == '/'){
3121                                                                 get_char ();
3122                                                                 comments_seen = true;
3123                                                                 break;
3124                                                         }
3125                                                         if (docAppend)
3126                                                                 xml_comment_buffer.Append ((char) d);
3127
3128                                                         if (d == '\n'){
3129                                                                 any_token_seen |= tokens_seen;
3130                                                                 tokens_seen = false;
3131                                                                 //
3132                                                                 // Reset 'comments_seen' just to be consistent.
3133                                                                 // It doesn't matter either way, here.
3134                                                                 //
3135                                                                 comments_seen = false;
3136                                                         }
3137                                                 }
3138                                                 if (!comments_seen)
3139                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3140
3141                                                 if (docAppend)
3142                                                         update_formatted_doc_comment (current_comment_start);
3143                                                 continue;
3144                                         }
3145                                         val = LocatedToken.Create (ref_line, col);
3146                                         return Token.DIV;
3147
3148                                 case '%':
3149                                         val = LocatedToken.Create (ref_line, col);
3150                                         if (peek_char () == '='){
3151                                                 get_char ();
3152                                                 return Token.OP_MOD_ASSIGN;
3153                                         }
3154                                         return Token.PERCENT;
3155
3156                                 case '^':
3157                                         val = LocatedToken.Create (ref_line, col);
3158                                         if (peek_char () == '='){
3159                                                 get_char ();
3160                                                 return Token.OP_XOR_ASSIGN;
3161                                         }
3162                                         return Token.CARRET;
3163
3164                                 case ':':
3165                                         val = LocatedToken.Create (ref_line, col);
3166                                         if (peek_char () == ':') {
3167                                                 get_char ();
3168                                                 return Token.DOUBLE_COLON;
3169                                         }
3170                                         return Token.COLON;
3171
3172                                 case '0': case '1': case '2': case '3': case '4':
3173                                 case '5': case '6': case '7': case '8': case '9':
3174                                         tokens_seen = true;
3175                                         return is_number (c);
3176
3177                                 case '\n': // white space
3178                                         any_token_seen |= tokens_seen;
3179                                         tokens_seen = false;
3180                                         comments_seen = false;
3181                                         continue;
3182
3183                                 case '.':
3184                                         tokens_seen = true;
3185                                         d = peek_char ();
3186                                         if (d >= '0' && d <= '9')
3187                                                 return is_number (c);
3188
3189                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3190                                         return Token.DOT;
3191
3192                                 case '#':
3193                                         if (tokens_seen || comments_seen) {
3194                                                 Eror_WrongPreprocessorLocation ();
3195                                                 return Token.ERROR;
3196                                         }
3197
3198                                         if (ParsePreprocessingDirective (true))
3199                                                 continue;
3200
3201                                         bool directive_expected = false;
3202                                         while ((c = get_char ()) != -1) {
3203                                                 if (col == 1) {
3204                                                         directive_expected = true;
3205                                                 } else if (!directive_expected) {
3206                                                         // TODO: Implement comment support for disabled code and uncomment this code
3207 //                                                      if (c == '#') {
3208 //                                                              Eror_WrongPreprocessorLocation ();
3209 //                                                              return Token.ERROR;
3210 //                                                      }
3211                                                         continue;
3212                                                 }
3213
3214                                                 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' )
3215                                                         continue;
3216
3217                                                 if (c == '#') {
3218                                                         if (ParsePreprocessingDirective (false))
3219                                                                 break;
3220                                                 }
3221                                                 directive_expected = false;
3222                                         }
3223
3224                                         if (c != -1) {
3225                                                 tokens_seen = false;
3226                                                 continue;
3227                                         }
3228
3229                                         return Token.EOF;
3230
3231                                 case '"':
3232                                         return consume_string (false);
3233
3234                                 case '\'':
3235                                         return TokenizeBackslash ();
3236
3237                                 case '@':
3238                                         c = get_char ();
3239                                         if (c == '"') {
3240                                                 tokens_seen = true;
3241                                                 return consume_string (true);
3242                                         }
3243
3244                                         if (is_identifier_start_character (c)){
3245                                                 return consume_identifier (c, true);
3246                                         }
3247
3248                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3249                                         return Token.ERROR;
3250
3251                                 case EvalStatementParserCharacter:
3252                                         return Token.EVAL_STATEMENT_PARSER;
3253                                 case EvalCompilationUnitParserCharacter:
3254                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3255                                 case EvalUsingDeclarationsParserCharacter:
3256                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3257                                 }
3258
3259                                 if (is_identifier_start_character (c)) {
3260                                         tokens_seen = true;
3261                                         return consume_identifier (c);
3262                                 }
3263
3264                                 if (char.IsWhiteSpace ((char) c))
3265                                         continue;
3266
3267                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3268                         }
3269
3270                         if (CompleteOnEOF){
3271                                 if (generated)
3272                                         return Token.COMPLETE_COMPLETION;
3273
3274                                 generated = true;
3275                                 return Token.GENERATE_COMPLETION;
3276                         }
3277
3278
3279                         return Token.EOF;
3280                 }
3281
3282                 int TokenizeBackslash ()
3283                 {
3284                         int c = get_char ();
3285                         tokens_seen = true;
3286                         if (c == '\'') {
3287                                 val = new CharLiteral (context.BuiltinTypes, (char) c, Location);
3288                                 Report.Error (1011, Location, "Empty character literal");
3289                                 return Token.LITERAL;
3290                         }
3291
3292                         if (c == '\n') {
3293                                 Report.Error (1010, Location, "Newline in constant");
3294                                 return Token.ERROR;
3295                         }
3296
3297                         int d;
3298                         c = escape (c, out d);
3299                         if (c == -1)
3300                                 return Token.ERROR;
3301                         if (d != 0)
3302                                 throw new NotImplementedException ();
3303
3304                         val = new CharLiteral (context.BuiltinTypes, (char) c, Location);
3305                         c = get_char ();
3306
3307                         if (c != '\'') {
3308                                 Report.Error (1012, Location, "Too many characters in character literal");
3309
3310                                 // Try to recover, read until newline or next "'"
3311                                 while ((c = get_char ()) != -1) {
3312                                         if (c == '\n' || c == '\'')
3313                                                 break;
3314                                 }
3315                         }
3316
3317                         return Token.LITERAL;
3318                 }
3319
3320                 int TokenizeLessThan ()
3321                 {
3322                         int d;
3323                         if (handle_typeof) {
3324                                 PushPosition ();
3325                                 if (parse_generic_dimension (out d)) {
3326                                         val = d;
3327                                         DiscardPosition ();
3328                                         return Token.GENERIC_DIMENSION;
3329                                 }
3330                                 PopPosition ();
3331                         }
3332
3333                         // Save current position and parse next token.
3334                         PushPosition ();
3335                         if (parse_less_than ()) {
3336                                 if (parsing_generic_declaration && token () != Token.DOT) {
3337                                         d = Token.OP_GENERICS_LT_DECL;
3338                                 } else {
3339                                         d = Token.OP_GENERICS_LT;
3340                                 }
3341                                 PopPosition ();
3342                                 return d;
3343                         }
3344
3345                         PopPosition ();
3346                         parsing_generic_less_than = 0;
3347
3348                         d = peek_char ();
3349                         if (d == '<') {
3350                                 get_char ();
3351                                 d = peek_char ();
3352
3353                                 if (d == '=') {
3354                                         get_char ();
3355                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3356                                 }
3357                                 return Token.OP_SHIFT_LEFT;
3358                         }
3359
3360                         if (d == '=') {
3361                                 get_char ();
3362                                 return Token.OP_LE;
3363                         }
3364                         return Token.OP_LT;
3365                 }
3366
3367                 //
3368                 // Handles one line xml comment
3369                 //
3370                 private void handle_one_line_xml_comment ()
3371                 {
3372                         int c;
3373                         while ((c = peek_char ()) == ' ')
3374                                 get_char (); // skip heading whitespaces.
3375                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3376                                 xml_comment_buffer.Append ((char) get_char ());
3377                         }
3378                         if (c == '\r' || c == '\n')
3379                                 xml_comment_buffer.Append (Environment.NewLine);
3380                 }
3381
3382                 //
3383                 // Remove heading "*" in Javadoc-like xml documentation.
3384                 //
3385                 private void update_formatted_doc_comment (int current_comment_start)
3386                 {
3387                         int length = xml_comment_buffer.Length - current_comment_start;
3388                         string [] lines = xml_comment_buffer.ToString (
3389                                 current_comment_start,
3390                                 length).Replace ("\r", "").Split ('\n');
3391
3392                         // The first line starts with /**, thus it is not target
3393                         // for the format check.
3394                         for (int i = 1; i < lines.Length; i++) {
3395                                 string s = lines [i];
3396                                 int idx = s.IndexOf ('*');
3397                                 string head = null;
3398                                 if (idx < 0) {
3399                                         if (i < lines.Length - 1)
3400                                                 return;
3401                                         head = s;
3402                                 } else
3403                                         head = s.Substring (0, idx);
3404                                 foreach (char c in head)
3405                                         if (c != ' ')
3406                                                 return;
3407                                 lines [i] = s.Substring (idx + 1);
3408                         }
3409                         xml_comment_buffer.Remove (current_comment_start, length);
3410                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3411                 }
3412
3413                 //
3414                 // Checks if there was incorrect doc comments and raise
3415                 // warnings.
3416                 //
3417                 public void check_incorrect_doc_comment ()
3418                 {
3419                         if (xml_comment_buffer.Length > 0)
3420                                 WarningMisplacedComment (Location);
3421                 }
3422
3423                 //
3424                 // Consumes the saved xml comment lines (if any)
3425                 // as for current target member or type.
3426                 //
3427                 public string consume_doc_comment ()
3428                 {
3429                         if (xml_comment_buffer.Length > 0) {
3430                                 string ret = xml_comment_buffer.ToString ();
3431                                 reset_doc_comment ();
3432                                 return ret;
3433                         }
3434                         return null;
3435                 }
3436
3437                 Report Report {
3438                         get { return context.Report; }
3439                 }
3440
3441                 void reset_doc_comment ()
3442                 {
3443                         xml_comment_buffer.Length = 0;
3444                 }
3445
3446                 public void cleanup ()
3447                 {
3448                         if (ifstack != null && ifstack.Count >= 1) {
3449                                 int state = ifstack.Pop ();
3450                                 if ((state & REGION) != 0)
3451                                         Report.Error (1038, Location, "#endregion directive expected");
3452                                 else
3453                                         Report.Error (1027, Location, "Expected `#endif' directive");
3454                         }
3455                 }
3456         }
3457
3458         //
3459         // Indicates whether it accepts XML documentation or not.
3460         //
3461         public enum XmlCommentState {
3462                 // comment is allowed in this state.
3463                 Allowed,
3464                 // comment is not allowed in this state.
3465                 NotAllowed,
3466                 // once comments appeared when it is NotAllowed, then the
3467                 // state is changed to it, until the state is changed to
3468                 // .Allowed.
3469                 Error
3470         }
3471 }
3472