mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 //
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20
  21 namespace Mono.CSharp
  22 {
  23         /// <summary>
  24         ///    Tokenizer for C# source code.
  25         /// </summary>
  26
  27         public class Tokenizer : yyParser.yyInput
  28         {
  29                 class KeywordEntry<T>
  30                 {
  31                         public readonly T Token;
  32                         public KeywordEntry<T> Next;
  33                         public readonly char[] Value;
  34
  35                         public KeywordEntry (string value, T token)
  36                         {
  37                                 this.Value = value.ToCharArray ();
  38                                 this.Token = token;
  39                         }
  40                 }
  41
  42                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  43                 {
  44                         readonly int length;
  45
  46                         public IdentifiersComparer (int length)
  47                         {
  48                                 this.length = length;
  49                         }
  50
  51                         public bool Equals (char[] x, char[] y)
  52                         {
  53                                 for (int i = 0; i < length; ++i)
  54                                         if (x [i] != y [i])
  55                                                 return false;
  56
  57                                 return true;
  58                         }
  59
  60                         public int GetHashCode (char[] obj)
  61                         {
  62                                 int h = 0;
  63                                 for (int i = 0; i < length; ++i)
  64                                         h = (h << 5) - h + obj [i];
  65
  66                                 return h;
  67                         }
  68                 }
  69
  70                 //
  71                 // This class has to be used in the parser only, it reuses token
  72                 // details after each parse
  73                 //
  74                 public class LocatedToken
  75                 {
  76                         int row, column;
  77                         string value;
  78
  79                         static LocatedToken[] buffer;
  80                         static int pos;
  81
  82                         private LocatedToken ()
  83                         {
  84                         }
  85
  86                         public static LocatedToken Create (int row, int column)
  87                         {
  88                                 return Create (null, row, column);
  89                         }
  90
  91                         public static LocatedToken Create (string value, int row, int column)
  92                         {
  93                                 //
  94                                 // TODO: I am not very happy about the logic but it's the best
  95                                 // what I could come up with for now.
  96                                 // Ideally we should be using just tiny buffer (256 elements) which
  97                                 // is enough to hold all details for currect stack and recycle elements
  98                                 // poped from the stack but there is a trick needed to recycle
  99                                 // them properly.
 100                                 //
 101                                 LocatedToken entry;
 102                                 if (pos >= buffer.Length) {
 103                                         entry = new LocatedToken ();
 104                                 } else {
 105                                         entry = buffer [pos];
 106                                         if (entry == null) {
 107                                                 entry = new LocatedToken ();
 108                                                 buffer [pos] = entry;
 109                                         }
 110
 111                                         ++pos;
 112                                 }
 113                                 entry.value = value;
 114                                 entry.row = row;
 115                                 entry.column = column;
 116                                 return entry;
 117                         }
 118
 119                         //
 120                         // Used for token not required by expression evaluator
 121                         //
 122                         [Conditional ("FULL_AST")]
 123                         public static void CreateOptional (int row, int col, ref object token)
 124                         {
 125                                 token = Create (row, col);
 126                         }
 127
 128                         public static void Initialize ()
 129                         {
 130                                 if (buffer == null)
 131                                         buffer = new LocatedToken [10000];
 132                                 pos = 0;
 133                         }
 134
 135                         public Location Location {
 136                                 get { return new Location (row, column); }
 137                         }
 138
 139                         public string Value {
 140                                 get { return value; }
 141                         }
 142                 }
 143
 144                 enum PreprocessorDirective
 145                 {
 146                         Invalid = 0,
 147
 148                         Region = 1,
 149                         Endregion = 2,
 150                         If = 3 | RequiresArgument,
 151                         Endif = 4,
 152                         Elif = 5 | RequiresArgument,
 153                         Else = 6,
 154                         Define = 7 | RequiresArgument,
 155                         Undef = 8 | RequiresArgument,
 156                         Error = 9,
 157                         Warning = 10,
 158                         Pragma = 11 | CustomArgumentsParsing,
 159                         Line = 12,
 160
 161                         CustomArgumentsParsing = 1 << 10,
 162                         RequiresArgument = 1 << 11
 163                 }
 164
 165                 SeekableStreamReader reader;
 166                 SourceFile ref_name;
 167                 CompilationSourceFile file_name;
 168                 CompilerContext context;
 169                 bool hidden = false;
 170                 int ref_line = 1;
 171                 int line = 1;
 172                 int col = 0;
 173                 int previous_col;
 174                 int current_token;
 175                 int tab_size;
 176                 bool handle_get_set = false;
 177                 bool handle_remove_add = false;
 178                 bool handle_where = false;
 179                 bool handle_typeof = false;
 180                 bool lambda_arguments_parsing;
 181                 List<Location> escaped_identifiers;
 182                 int parsing_generic_less_than;
 183                 readonly bool doc_processing;
 184
 185                 //
 186                 // Used mainly for parser optimizations. Some expressions for instance
 187                 // can appear only in block (including initializer, base initializer)
 188                 // scope only
 189                 //
 190                 public int parsing_block;
 191                 internal bool query_parsing;
 192
 193                 //
 194                 // When parsing type only, useful for ambiguous nullable types
 195                 //
 196                 public int parsing_type;
 197
 198                 //
 199                 // Set when parsing generic declaration (type or method header)
 200                 //
 201                 public bool parsing_generic_declaration;
 202                 public bool parsing_generic_declaration_doc;
 203
 204                 //
 205                 // The value indicates that we have not reach any declaration or
 206                 // namespace yet
 207                 //
 208                 public int parsing_declaration;
 209
 210                 public bool parsing_attribute_section;
 211
 212                 //
 213                 // The special characters to inject on streams to run the unit parser
 214                 // in the special expression mode. Using private characters from
 215                 // Plane Sixteen (U+100000 to U+10FFFD)
 216                 //
 217                 // This character is only tested just before the tokenizer is about to report
 218                 // an error;   So on the regular operation mode, this addition will have no
 219                 // impact on the tokenizer's performance.
 220                 //
 221
 222                 public const int EvalStatementParserCharacter = 0x100000;
 223                 public const int EvalCompilationUnitParserCharacter = 0x100001;
 224                 public const int EvalUsingDeclarationsParserCharacter = 0x100002;
 225                 public const int DocumentationXref = 0x100003;
 226
 227                 //
 228                 // XML documentation buffer. The save point is used to divide
 229                 // comments on types and comments on members.
 230                 //
 231                 StringBuilder xml_comment_buffer;
 232
 233                 //
 234                 // See comment on XmlCommentState enumeration.
 235                 //
 236                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 237
 238                 //
 239                 // Whether tokens have been seen on this line
 240                 //
 241                 bool tokens_seen = false;
 242
 243                 //
 244                 // Set to true once the GENERATE_COMPLETION token has bee
 245                 // returned.   This helps produce one GENERATE_COMPLETION,
 246                 // as many COMPLETE_COMPLETION as necessary to complete the
 247                 // AST tree and one final EOF.
 248                 //
 249                 bool generated;
 250
 251                 //
 252                 // Whether a token has been seen on the file
 253                 // This is needed because `define' is not allowed to be used
 254                 // after a token has been seen.
 255                 //
 256                 bool any_token_seen;
 257
 258                 //
 259                 // Class variables
 260                 //
 261                 static readonly KeywordEntry<int>[][] keywords;
 262                 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 263                 static readonly Dictionary<string, object> keyword_strings;             // TODO: HashSet
 264                 static readonly NumberStyles styles;
 265                 static readonly NumberFormatInfo csharp_format_info;
 266
 267                 // Pragma arguments
 268                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 269                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 270                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 271                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 272
 273                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 274
 275                 public bool PropertyParsing {
 276                         get { return handle_get_set; }
 277                         set { handle_get_set = value; }
 278                 }
 279
 280                 public bool EventParsing {
 281                         get { return handle_remove_add; }
 282                         set { handle_remove_add = value; }
 283                 }
 284
 285                 public bool ConstraintsParsing {
 286                         get { return handle_where; }
 287                         set { handle_where = value; }
 288                 }
 289
 290                 public bool TypeOfParsing {
 291                         get { return handle_typeof; }
 292                         set { handle_typeof = value; }
 293                 }
 294
 295                 public int TabSize {
 296                         get { return tab_size; }
 297                         set { tab_size = value; }
 298                 }
 299
 300                 public XmlCommentState doc_state {
 301                         get { return xml_doc_state; }
 302                         set {
 303                                 if (value == XmlCommentState.Allowed) {
 304                                         check_incorrect_doc_comment ();
 305                                         reset_doc_comment ();
 306                                 }
 307                                 xml_doc_state = value;
 308                         }
 309                 }
 310
 311                 //
 312                 // This is used to trigger completion generation on the parser
 313                 public bool CompleteOnEOF;
 314
 315                 void AddEscapedIdentifier (Location loc)
 316                 {
 317                         if (escaped_identifiers == null)
 318                                 escaped_identifiers = new List<Location> ();
 319
 320                         escaped_identifiers.Add (loc);
 321                 }
 322
 323                 public bool IsEscapedIdentifier (MemberName name)
 324                 {
 325                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 326                 }
 327
 328                 //
 329                 // Values for the associated token returned
 330                 //
 331                 internal int putback_char;      // Used by repl only
 332                 object val;
 333
 334                 //
 335                 // Pre-processor
 336                 //
 337                 const int TAKING        = 1;
 338                 const int ELSE_SEEN     = 4;
 339                 const int PARENT_TAKING = 8;
 340                 const int REGION        = 16;
 341
 342                 //
 343                 // pre-processor if stack state:
 344                 //
 345                 Stack<int> ifstack;
 346
 347                 static System.Text.StringBuilder string_builder;
 348
 349                 const int max_id_size = 512;
 350                 static readonly char [] id_builder = new char [max_id_size];
 351
 352                 public static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 353
 354                 const int max_number_size = 512;
 355                 static char [] number_builder = new char [max_number_size];
 356                 static int number_pos;
 357
 358                 static char[] value_builder = new char[256];
 359
 360                 public int Line {
 361                         get {
 362                                 return ref_line;
 363                         }
 364                 }
 365
 366                 //
 367                 // This is used when the tokenizer needs to save
 368                 // the current position as it needs to do some parsing
 369                 // on its own to deamiguate a token in behalf of the
 370                 // parser.
 371                 //
 372                 Stack<Position> position_stack = new Stack<Position> (2);
 373
 374                 class Position {
 375                         public int position;
 376                         public int line;
 377                         public int ref_line;
 378                         public int col;
 379                         public bool hidden;
 380                         public int putback_char;
 381                         public int previous_col;
 382                         public Stack<int> ifstack;
 383                         public int parsing_generic_less_than;
 384                         public int current_token;
 385                         public object val;
 386
 387                         public Position (Tokenizer t)
 388                         {
 389                                 position = t.reader.Position;
 390                                 line = t.line;
 391                                 ref_line = t.ref_line;
 392                                 col = t.col;
 393                                 hidden = t.hidden;
 394                                 putback_char = t.putback_char;
 395                                 previous_col = t.previous_col;
 396                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 397                                         // There is no simple way to clone Stack<T> all
 398                                         // methods reverse the order
 399                                         var clone = t.ifstack.ToArray ();
 400                                         Array.Reverse (clone);
 401                                         ifstack = new Stack<int> (clone);
 402                                 }
 403                                 parsing_generic_less_than = t.parsing_generic_less_than;
 404                                 current_token = t.current_token;
 405                                 val = t.val;
 406                         }
 407                 }
 408
 409                 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, CompilerContext ctx)
 410                 {
 411                         this.ref_name = file;
 412                         this.file_name = file;
 413                         this.context = ctx;
 414                         reader = input;
 415
 416                         putback_char = -1;
 417
 418                         xml_comment_buffer = new StringBuilder ();
 419                         doc_processing = ctx.Settings.DocumentationFile != null;
 420
 421                         if (Environment.OSVersion.Platform == PlatformID.Win32NT)
 422                                 tab_size = 4;
 423                         else
 424                                 tab_size = 8;
 425
 426                         //
 427                         // FIXME: This could be `Location.Push' but we have to
 428                         // find out why the MS compiler allows this
 429                         //
 430                         Mono.CSharp.Location.Push (file, file);
 431                 }
 432
 433                 public void PushPosition ()
 434                 {
 435                         position_stack.Push (new Position (this));
 436                 }
 437
 438                 public void PopPosition ()
 439                 {
 440                         Position p = position_stack.Pop ();
 441
 442                         reader.Position = p.position;
 443                         ref_line = p.ref_line;
 444                         line = p.line;
 445                         col = p.col;
 446                         hidden = p.hidden;
 447                         putback_char = p.putback_char;
 448                         previous_col = p.previous_col;
 449                         ifstack = p.ifstack;
 450                         parsing_generic_less_than = p.parsing_generic_less_than;
 451                         current_token = p.current_token;
 452                         val = p.val;
 453                 }
 454
 455                 // Do not reset the position, ignore it.
 456                 public void DiscardPosition ()
 457                 {
 458                         position_stack.Pop ();
 459                 }
 460
 461                 static void AddKeyword (string kw, int token)
 462                 {
 463                         keyword_strings.Add (kw, null);
 464
 465                         AddKeyword (keywords, kw, token);
 466                 }
 467
 468                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 469                 {
 470                         AddKeyword (keywords_preprocessor, kw, directive);
 471                 }
 472
 473                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 474                 {
 475                         int length = kw.Length;
 476                         if (keywords[length] == null) {
 477                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 478                         }
 479
 480                         int char_index = kw[0] - '_';
 481                         var kwe = keywords[length][char_index];
 482                         if (kwe == null) {
 483                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 484                                 return;
 485                         }
 486
 487                         while (kwe.Next != null) {
 488                                 kwe = kwe.Next;
 489                         }
 490
 491                         kwe.Next = new KeywordEntry<T> (kw, token);
 492                 }
 493
 494                 //
 495                 // Class initializer
 496                 //
 497                 static Tokenizer ()
 498                 {
 499                         keyword_strings = new Dictionary<string, object> ();
 500
 501                         // 11 is the length of the longest keyword for now
 502                         keywords = new KeywordEntry<int>[11][];
 503
 504                         AddKeyword ("__arglist", Token.ARGLIST);
 505                         AddKeyword ("__makeref", Token.MAKEREF);
 506                         AddKeyword ("__reftype", Token.REFTYPE);
 507                         AddKeyword ("__refvalue", Token.REFVALUE);
 508                         AddKeyword ("abstract", Token.ABSTRACT);
 509                         AddKeyword ("as", Token.AS);
 510                         AddKeyword ("add", Token.ADD);
 511                         AddKeyword ("base", Token.BASE);
 512                         AddKeyword ("bool", Token.BOOL);
 513                         AddKeyword ("break", Token.BREAK);
 514                         AddKeyword ("byte", Token.BYTE);
 515                         AddKeyword ("case", Token.CASE);
 516                         AddKeyword ("catch", Token.CATCH);
 517                         AddKeyword ("char", Token.CHAR);
 518                         AddKeyword ("checked", Token.CHECKED);
 519                         AddKeyword ("class", Token.CLASS);
 520                         AddKeyword ("const", Token.CONST);
 521                         AddKeyword ("continue", Token.CONTINUE);
 522                         AddKeyword ("decimal", Token.DECIMAL);
 523                         AddKeyword ("default", Token.DEFAULT);
 524                         AddKeyword ("delegate", Token.DELEGATE);
 525                         AddKeyword ("do", Token.DO);
 526                         AddKeyword ("double", Token.DOUBLE);
 527                         AddKeyword ("else", Token.ELSE);
 528                         AddKeyword ("enum", Token.ENUM);
 529                         AddKeyword ("event", Token.EVENT);
 530                         AddKeyword ("explicit", Token.EXPLICIT);
 531                         AddKeyword ("extern", Token.EXTERN);
 532                         AddKeyword ("false", Token.FALSE);
 533                         AddKeyword ("finally", Token.FINALLY);
 534                         AddKeyword ("fixed", Token.FIXED);
 535                         AddKeyword ("float", Token.FLOAT);
 536                         AddKeyword ("for", Token.FOR);
 537                         AddKeyword ("foreach", Token.FOREACH);
 538                         AddKeyword ("goto", Token.GOTO);
 539                         AddKeyword ("get", Token.GET);
 540                         AddKeyword ("if", Token.IF);
 541                         AddKeyword ("implicit", Token.IMPLICIT);
 542                         AddKeyword ("in", Token.IN);
 543                         AddKeyword ("int", Token.INT);
 544                         AddKeyword ("interface", Token.INTERFACE);
 545                         AddKeyword ("internal", Token.INTERNAL);
 546                         AddKeyword ("is", Token.IS);
 547                         AddKeyword ("lock", Token.LOCK);
 548                         AddKeyword ("long", Token.LONG);
 549                         AddKeyword ("namespace", Token.NAMESPACE);
 550                         AddKeyword ("new", Token.NEW);
 551                         AddKeyword ("null", Token.NULL);
 552                         AddKeyword ("object", Token.OBJECT);
 553                         AddKeyword ("operator", Token.OPERATOR);
 554                         AddKeyword ("out", Token.OUT);
 555                         AddKeyword ("override", Token.OVERRIDE);
 556                         AddKeyword ("params", Token.PARAMS);
 557                         AddKeyword ("private", Token.PRIVATE);
 558                         AddKeyword ("protected", Token.PROTECTED);
 559                         AddKeyword ("public", Token.PUBLIC);
 560                         AddKeyword ("readonly", Token.READONLY);
 561                         AddKeyword ("ref", Token.REF);
 562                         AddKeyword ("remove", Token.REMOVE);
 563                         AddKeyword ("return", Token.RETURN);
 564                         AddKeyword ("sbyte", Token.SBYTE);
 565                         AddKeyword ("sealed", Token.SEALED);
 566                         AddKeyword ("set", Token.SET);
 567                         AddKeyword ("short", Token.SHORT);
 568                         AddKeyword ("sizeof", Token.SIZEOF);
 569                         AddKeyword ("stackalloc", Token.STACKALLOC);
 570                         AddKeyword ("static", Token.STATIC);
 571                         AddKeyword ("string", Token.STRING);
 572                         AddKeyword ("struct", Token.STRUCT);
 573                         AddKeyword ("switch", Token.SWITCH);
 574                         AddKeyword ("this", Token.THIS);
 575                         AddKeyword ("throw", Token.THROW);
 576                         AddKeyword ("true", Token.TRUE);
 577                         AddKeyword ("try", Token.TRY);
 578                         AddKeyword ("typeof", Token.TYPEOF);
 579                         AddKeyword ("uint", Token.UINT);
 580                         AddKeyword ("ulong", Token.ULONG);
 581                         AddKeyword ("unchecked", Token.UNCHECKED);
 582                         AddKeyword ("unsafe", Token.UNSAFE);
 583                         AddKeyword ("ushort", Token.USHORT);
 584                         AddKeyword ("using", Token.USING);
 585                         AddKeyword ("virtual", Token.VIRTUAL);
 586                         AddKeyword ("void", Token.VOID);
 587                         AddKeyword ("volatile", Token.VOLATILE);
 588                         AddKeyword ("while", Token.WHILE);
 589                         AddKeyword ("partial", Token.PARTIAL);
 590                         AddKeyword ("where", Token.WHERE);
 591
 592                         // LINQ keywords
 593                         AddKeyword ("from", Token.FROM);
 594                         AddKeyword ("join", Token.JOIN);
 595                         AddKeyword ("on", Token.ON);
 596                         AddKeyword ("equals", Token.EQUALS);
 597                         AddKeyword ("select", Token.SELECT);
 598                         AddKeyword ("group", Token.GROUP);
 599                         AddKeyword ("by", Token.BY);
 600                         AddKeyword ("let", Token.LET);
 601                         AddKeyword ("orderby", Token.ORDERBY);
 602                         AddKeyword ("ascending", Token.ASCENDING);
 603                         AddKeyword ("descending", Token.DESCENDING);
 604                         AddKeyword ("into", Token.INTO);
 605
 606                         // Contextual async keywords
 607                         AddKeyword ("async", Token.ASYNC);
 608                         AddKeyword ("await", Token.AWAIT);
 609
 610                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 611
 612                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 613                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 614                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 615                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 616                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 617                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 618                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 619                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 620                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 621                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 622                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 623                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 624
 625                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 626                         styles = NumberStyles.Float;
 627
 628                         string_builder = new System.Text.StringBuilder ();
 629                 }
 630
 631                 int GetKeyword (char[] id, int id_len)
 632                 {
 633                         //
 634                         // Keywords are stored in an array of arrays grouped by their
 635                         // length and then by the first character
 636                         //
 637                         if (id_len >= keywords.Length || keywords [id_len] == null)
 638                                 return -1;
 639
 640                         int first_index = id [0] - '_';
 641                         if (first_index > 'z' - '_')
 642                                 return -1;
 643
 644                         var kwe = keywords [id_len] [first_index];
 645                         if (kwe == null)
 646                                 return -1;
 647
 648                         int res;
 649                         do {
 650                                 res = kwe.Token;
 651                                 for (int i = 1; i < id_len; ++i) {
 652                                         if (id [i] != kwe.Value [i]) {
 653                                                 res = 0;
 654                                                 kwe = kwe.Next;
 655                                                 break;
 656                                         }
 657                                 }
 658                         } while (res == 0 && kwe != null);
 659
 660                         if (res == 0)
 661                                 return -1;
 662
 663                         int next_token;
 664                         switch (res) {
 665                         case Token.GET:
 666                         case Token.SET:
 667                                 if (!handle_get_set)
 668                                         res = -1;
 669                                 break;
 670                         case Token.REMOVE:
 671                         case Token.ADD:
 672                                 if (!handle_remove_add)
 673                                         res = -1;
 674                                 break;
 675                         case Token.EXTERN:
 676                                 if (parsing_declaration == 0)
 677                                         res = Token.EXTERN_ALIAS;
 678                                 break;
 679                         case Token.DEFAULT:
 680                                 if (peek_token () == Token.COLON) {
 681                                         token ();
 682                                         res = Token.DEFAULT_COLON;
 683                                 }
 684                                 break;
 685                         case Token.WHERE:
 686                                 if (!handle_where && !query_parsing)
 687                                         res = -1;
 688                                 break;
 689                         case Token.FROM:
 690                                 //
 691                                 // A query expression is any expression that starts with `from identifier'
 692                                 // followed by any token except ; , =
 693                                 //
 694                                 if (!query_parsing) {
 695                                         if (lambda_arguments_parsing) {
 696                                                 res = -1;
 697                                                 break;
 698                                         }
 699
 700                                         PushPosition ();
 701                                         // HACK: to disable generics micro-parser, because PushPosition does not
 702                                         // store identifiers array
 703                                         parsing_generic_less_than = 1;
 704                                         switch (xtoken ()) {
 705                                         case Token.IDENTIFIER:
 706                                         case Token.INT:
 707                                         case Token.BOOL:
 708                                         case Token.BYTE:
 709                                         case Token.CHAR:
 710                                         case Token.DECIMAL:
 711                                         case Token.FLOAT:
 712                                         case Token.LONG:
 713                                         case Token.OBJECT:
 714                                         case Token.STRING:
 715                                         case Token.UINT:
 716                                         case Token.ULONG:
 717                                                 next_token = xtoken ();
 718                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 719                                                         goto default;
 720
 721                                                 res = Token.FROM_FIRST;
 722                                                 query_parsing = true;
 723                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 724                                                         Report.FeatureIsNotAvailable (context, Location, "query expressions");
 725                                                 break;
 726                                         case Token.VOID:
 727                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 728                                                 break;
 729                                         default:
 730                                                 PopPosition ();
 731                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 732                                                 // which has been overwritten before we grabbed the identifier
 733                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 734                                                 return -1;
 735                                         }
 736                                         PopPosition ();
 737                                 }
 738                                 break;
 739                         case Token.JOIN:
 740                         case Token.ON:
 741                         case Token.EQUALS:
 742                         case Token.SELECT:
 743                         case Token.GROUP:
 744                         case Token.BY:
 745                         case Token.LET:
 746                         case Token.ORDERBY:
 747                         case Token.ASCENDING:
 748                         case Token.DESCENDING:
 749                         case Token.INTO:
 750                                 if (!query_parsing)
 751                                         res = -1;
 752                                 break;
 753
 754                         case Token.USING:
 755                         case Token.NAMESPACE:
 756                                 // TODO: some explanation needed
 757                                 check_incorrect_doc_comment ();
 758                                 break;
 759
 760                         case Token.PARTIAL:
 761                                 if (parsing_block > 0) {
 762                                         res = -1;
 763                                         break;
 764                                 }
 765
 766                                 // Save current position and parse next token.
 767                                 PushPosition ();
 768
 769                                 next_token = token ();
 770                                 bool ok = (next_token == Token.CLASS) ||
 771                                         (next_token == Token.STRUCT) ||
 772                                         (next_token == Token.INTERFACE) ||
 773                                         (next_token == Token.VOID);
 774
 775                                 PopPosition ();
 776
 777                                 if (ok) {
 778                                         if (next_token == Token.VOID) {
 779                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 780                                                         Report.FeatureIsNotAvailable (context, Location, "partial methods");
 781                                         } else if (context.Settings.Version == LanguageVersion.ISO_1)
 782                                                 Report.FeatureIsNotAvailable (context, Location, "partial types");
 783
 784                                         return res;
 785                                 }
 786
 787                                 if (next_token < Token.LAST_KEYWORD) {
 788                                         Report.Error (267, Location,
 789                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 790                                         return token ();
 791                                 }
 792
 793                                 res = -1;
 794                                 break;
 795
 796                         // TODO: async, it's modifiers context only
 797                         case Token.ASYNC:
 798                                 if (context.Settings.Version != LanguageVersion.Future) {
 799                                         res = -1;
 800                                 }
 801                                 break;
 802
 803                         // TODO: async, it's async block context only
 804                         case Token.AWAIT:
 805                                 if (context.Settings.Version != LanguageVersion.Future) {
 806                                         res = -1;
 807                                 }
 808
 809                                 break;
 810                         }
 811
 812
 813                         return res;
 814                 }
 815
 816                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 817                 {
 818                         //
 819                         // Keywords are stored in an array of arrays grouped by their
 820                         // length and then by the first character
 821                         //
 822                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 823                                 return PreprocessorDirective.Invalid;
 824
 825                         int first_index = id[0] - '_';
 826                         if (first_index > 'z' - '_')
 827                                 return PreprocessorDirective.Invalid;
 828
 829                         var kwe = keywords_preprocessor[id_len][first_index];
 830                         if (kwe == null)
 831                                 return PreprocessorDirective.Invalid;
 832
 833                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 834                         do {
 835                                 res = kwe.Token;
 836                                 for (int i = 1; i < id_len; ++i) {
 837                                         if (id[i] != kwe.Value[i]) {
 838                                                 res = 0;
 839                                                 kwe = kwe.Next;
 840                                                 break;
 841                                         }
 842                                 }
 843                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 844
 845                         return res;
 846                 }
 847
 848                 public Location Location {
 849                         get {
 850                                 return new Location (ref_line, hidden ? -1 : col);
 851                         }
 852                 }
 853
 854                 static bool is_identifier_start_character (int c)
 855                 {
 856                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 857                 }
 858
 859                 static bool is_identifier_part_character (char c)
 860                 {
 861                         if (c >= 'a' && c <= 'z')
 862                                 return true;
 863
 864                         if (c >= 'A' && c <= 'Z')
 865                                 return true;
 866
 867                         if (c == '_' || (c >= '0' && c <= '9'))
 868                                 return true;
 869
 870                         if (c < 0x80)
 871                                 return false;
 872
 873                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 874                 }
 875
 876                 public static bool IsKeyword (string s)
 877                 {
 878                         return keyword_strings.ContainsKey (s);
 879                 }
 880
 881                 //
 882                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 883                 //
 884                 int TokenizeOpenParens ()
 885                 {
 886                         int ptoken;
 887                         current_token = -1;
 888
 889                         int bracket_level = 0;
 890                         bool is_type = false;
 891                         bool can_be_type = false;
 892
 893                         while (true) {
 894                                 ptoken = current_token;
 895                                 token ();
 896
 897                                 switch (current_token) {
 898                                 case Token.CLOSE_PARENS:
 899                                         token ();
 900
 901                                         //
 902                                         // Expression inside parens is lambda, (int i) =>
 903                                         //
 904                                         if (current_token == Token.ARROW)
 905                                                 return Token.OPEN_PARENS_LAMBDA;
 906
 907                                         //
 908                                         // Expression inside parens is single type, (int[])
 909                                         //
 910                                         if (is_type)
 911                                                 return Token.OPEN_PARENS_CAST;
 912
 913                                         //
 914                                         // Expression is possible cast, look at next token, (T)null
 915                                         //
 916                                         if (can_be_type) {
 917                                                 switch (current_token) {
 918                                                 case Token.OPEN_PARENS:
 919                                                 case Token.BANG:
 920                                                 case Token.TILDE:
 921                                                 case Token.IDENTIFIER:
 922                                                 case Token.LITERAL:
 923                                                 case Token.BASE:
 924                                                 case Token.CHECKED:
 925                                                 case Token.DELEGATE:
 926                                                 case Token.FALSE:
 927                                                 case Token.FIXED:
 928                                                 case Token.NEW:
 929                                                 case Token.NULL:
 930                                                 case Token.SIZEOF:
 931                                                 case Token.THIS:
 932                                                 case Token.THROW:
 933                                                 case Token.TRUE:
 934                                                 case Token.TYPEOF:
 935                                                 case Token.UNCHECKED:
 936                                                 case Token.UNSAFE:
 937                                                 case Token.DEFAULT:
 938
 939                                                 //
 940                                                 // These can be part of a member access
 941                                                 //
 942                                                 case Token.INT:
 943                                                 case Token.UINT:
 944                                                 case Token.SHORT:
 945                                                 case Token.USHORT:
 946                                                 case Token.LONG:
 947                                                 case Token.ULONG:
 948                                                 case Token.DOUBLE:
 949                                                 case Token.FLOAT:
 950                                                 case Token.CHAR:
 951                                                 case Token.BYTE:
 952                                                 case Token.DECIMAL:
 953                                                 case Token.BOOL:
 954                                                         return Token.OPEN_PARENS_CAST;
 955                                                 }
 956                                         }
 957                                         return Token.OPEN_PARENS;
 958
 959                                 case Token.DOT:
 960                                 case Token.DOUBLE_COLON:
 961                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
 962                                                 goto default;
 963
 964                                         continue;
 965
 966                                 case Token.IDENTIFIER:
 967                                         switch (ptoken) {
 968                                         case Token.DOT:
 969                                                 if (bracket_level == 0) {
 970                                                         is_type = false;
 971                                                         can_be_type = true;
 972                                                 }
 973
 974                                                 continue;
 975                                         case Token.OP_GENERICS_LT:
 976                                         case Token.COMMA:
 977                                         case Token.DOUBLE_COLON:
 978                                         case -1:
 979                                                 if (bracket_level == 0)
 980                                                         can_be_type = true;
 981                                                 continue;
 982                                         default:
 983                                                 can_be_type = is_type = false;
 984                                                 continue;
 985                                         }
 986
 987                                 case Token.OBJECT:
 988                                 case Token.STRING:
 989                                 case Token.BOOL:
 990                                 case Token.DECIMAL:
 991                                 case Token.FLOAT:
 992                                 case Token.DOUBLE:
 993                                 case Token.SBYTE:
 994                                 case Token.BYTE:
 995                                 case Token.SHORT:
 996                                 case Token.USHORT:
 997                                 case Token.INT:
 998                                 case Token.UINT:
 999                                 case Token.LONG:
1000                                 case Token.ULONG:
1001                                 case Token.CHAR:
1002                                 case Token.VOID:
1003                                         if (bracket_level == 0)
1004                                                 is_type = true;
1005                                         continue;
1006
1007                                 case Token.COMMA:
1008                                         if (bracket_level == 0) {
1009                                                 bracket_level = 100;
1010                                                 can_be_type = is_type = false;
1011                                         }
1012                                         continue;
1013
1014                                 case Token.OP_GENERICS_LT:
1015                                 case Token.OPEN_BRACKET:
1016                                         if (bracket_level++ == 0)
1017                                                 is_type = true;
1018                                         continue;
1019
1020                                 case Token.OP_GENERICS_GT:
1021                                 case Token.CLOSE_BRACKET:
1022                                         --bracket_level;
1023                                         continue;
1024
1025                                 case Token.INTERR_NULLABLE:
1026                                 case Token.STAR:
1027                                         if (bracket_level == 0)
1028                                                 is_type = true;
1029                                         continue;
1030
1031                                 case Token.REF:
1032                                 case Token.OUT:
1033                                         can_be_type = is_type = false;
1034                                         continue;
1035
1036                                 default:
1037                                         return Token.OPEN_PARENS;
1038                                 }
1039                         }
1040                 }
1041
1042                 public static bool IsValidIdentifier (string s)
1043                 {
1044                         if (s == null || s.Length == 0)
1045                                 return false;
1046
1047                         if (!is_identifier_start_character (s [0]))
1048                                 return false;
1049
1050                         for (int i = 1; i < s.Length; i ++)
1051                                 if (! is_identifier_part_character (s [i]))
1052                                         return false;
1053
1054                         return true;
1055                 }
1056
1057                 bool parse_less_than ()
1058                 {
1059                 start:
1060                         int the_token = token ();
1061                         if (the_token == Token.OPEN_BRACKET) {
1062                                 do {
1063                                         the_token = token ();
1064                                 } while (the_token != Token.CLOSE_BRACKET);
1065                                 the_token = token ();
1066                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1067                                 the_token = token ();
1068                         }
1069                         switch (the_token) {
1070                         case Token.IDENTIFIER:
1071                         case Token.OBJECT:
1072                         case Token.STRING:
1073                         case Token.BOOL:
1074                         case Token.DECIMAL:
1075                         case Token.FLOAT:
1076                         case Token.DOUBLE:
1077                         case Token.SBYTE:
1078                         case Token.BYTE:
1079                         case Token.SHORT:
1080                         case Token.USHORT:
1081                         case Token.INT:
1082                         case Token.UINT:
1083                         case Token.LONG:
1084                         case Token.ULONG:
1085                         case Token.CHAR:
1086                         case Token.VOID:
1087                                 break;
1088                         case Token.OP_GENERICS_GT:
1089                         case Token.IN:
1090                         case Token.OUT:
1091                                 return true;
1092
1093                         default:
1094                                 return false;
1095                         }
1096                 again:
1097                         the_token = token ();
1098
1099                         if (the_token == Token.OP_GENERICS_GT)
1100                                 return true;
1101                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1102                                 goto start;
1103                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1104                                 goto again;
1105                         else if (the_token == Token.OP_GENERICS_LT) {
1106                                 if (!parse_less_than ())
1107                                         return false;
1108                                 goto again;
1109                         } else if (the_token == Token.OPEN_BRACKET) {
1110                         rank_specifiers:
1111                                 the_token = token ();
1112                                 if (the_token == Token.CLOSE_BRACKET)
1113                                         goto again;
1114                                 else if (the_token == Token.COMMA)
1115                                         goto rank_specifiers;
1116                                 return false;
1117                         }
1118
1119                         return false;
1120                 }
1121
1122                 bool parse_generic_dimension (out int dimension)
1123                 {
1124                         dimension = 1;
1125
1126                 again:
1127                         int the_token = token ();
1128                         if (the_token == Token.OP_GENERICS_GT)
1129                                 return true;
1130                         else if (the_token == Token.COMMA) {
1131                                 dimension++;
1132                                 goto again;
1133                         }
1134
1135                         return false;
1136                 }
1137
1138                 public int peek_token ()
1139                 {
1140                         int the_token;
1141
1142                         PushPosition ();
1143                         the_token = token ();
1144                         PopPosition ();
1145
1146                         return the_token;
1147                 }
1148
1149                 //
1150                 // Tonizes `?' using custom disambiguous rules to return one
1151                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1152                 //
1153                 // Tricky expression look like:
1154                 //
1155                 // Foo ? a = x ? b : c;
1156                 //
1157                 int TokenizePossibleNullableType ()
1158                 {
1159                         if (parsing_block == 0 || parsing_type > 0)
1160                                 return Token.INTERR_NULLABLE;
1161
1162                         int d = peek_char ();
1163                         if (d == '?') {
1164                                 get_char ();
1165                                 return Token.OP_COALESCING;
1166                         }
1167
1168                         switch (current_token) {
1169                         case Token.CLOSE_PARENS:
1170                         case Token.TRUE:
1171                         case Token.FALSE:
1172                         case Token.NULL:
1173                         case Token.LITERAL:
1174                                 return Token.INTERR;
1175                         }
1176
1177                         if (d != ' ') {
1178                                 if (d == ',' || d == ';' || d == '>')
1179                                         return Token.INTERR_NULLABLE;
1180                                 if (d == '*' || (d >= '0' && d <= '9'))
1181                                         return Token.INTERR;
1182                         }
1183
1184                         PushPosition ();
1185                         current_token = Token.NONE;
1186                         int next_token;
1187                         switch (xtoken ()) {
1188                         case Token.LITERAL:
1189                         case Token.TRUE:
1190                         case Token.FALSE:
1191                         case Token.NULL:
1192                         case Token.THIS:
1193                         case Token.NEW:
1194                                 next_token = Token.INTERR;
1195                                 break;
1196
1197                         case Token.SEMICOLON:
1198                         case Token.COMMA:
1199                         case Token.CLOSE_PARENS:
1200                         case Token.OPEN_BRACKET:
1201                         case Token.OP_GENERICS_GT:
1202                         case Token.INTERR:
1203                                 next_token = Token.INTERR_NULLABLE;
1204                                 break;
1205
1206                         default:
1207                                 next_token = -1;
1208                                 break;
1209                         }
1210
1211                         if (next_token == -1) {
1212                                 switch (xtoken ()) {
1213                                 case Token.COMMA:
1214                                 case Token.SEMICOLON:
1215                                 case Token.OPEN_BRACE:
1216                                 case Token.CLOSE_PARENS:
1217                                 case Token.IN:
1218                                         next_token = Token.INTERR_NULLABLE;
1219                                         break;
1220
1221                                 case Token.COLON:
1222                                         next_token = Token.INTERR;
1223                                         break;
1224
1225                                 default:
1226                                         int ntoken;
1227                                         int interrs = 1;
1228                                         int colons = 0;
1229                                         //
1230                                         // All shorcuts failed, do it hard way
1231                                         //
1232                                         while ((ntoken = xtoken ()) != Token.EOF) {
1233                                                 if (ntoken == Token.SEMICOLON)
1234                                                         break;
1235
1236                                                 if (ntoken == Token.COLON) {
1237                                                         if (++colons == interrs)
1238                                                                 break;
1239                                                         continue;
1240                                                 }
1241
1242                                                 if (ntoken == Token.INTERR) {
1243                                                         ++interrs;
1244                                                         continue;
1245                                                 }
1246                                         }
1247
1248                                         next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1249                                         break;
1250                                 }
1251                         }
1252
1253                         PopPosition ();
1254                         return next_token;
1255                 }
1256
1257                 bool decimal_digits (int c)
1258                 {
1259                         int d;
1260                         bool seen_digits = false;
1261
1262                         if (c != -1){
1263                                 if (number_pos == max_number_size)
1264                                         Error_NumericConstantTooLong ();
1265                                 number_builder [number_pos++] = (char) c;
1266                         }
1267
1268                         //
1269                         // We use peek_char2, because decimal_digits needs to do a
1270                         // 2-character look-ahead (5.ToString for example).
1271                         //
1272                         while ((d = peek_char2 ()) != -1){
1273                                 if (d >= '0' && d <= '9'){
1274                                         if (number_pos == max_number_size)
1275                                                 Error_NumericConstantTooLong ();
1276                                         number_builder [number_pos++] = (char) d;
1277                                         get_char ();
1278                                         seen_digits = true;
1279                                 } else
1280                                         break;
1281                         }
1282
1283                         return seen_digits;
1284                 }
1285
1286                 static bool is_hex (int e)
1287                 {
1288                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1289                 }
1290
1291                 static TypeCode real_type_suffix (int c)
1292                 {
1293                         switch (c){
1294                         case 'F': case 'f':
1295                                 return TypeCode.Single;
1296                         case 'D': case 'd':
1297                                 return TypeCode.Double;
1298                         case 'M': case 'm':
1299                                 return TypeCode.Decimal;
1300                         default:
1301                                 return TypeCode.Empty;
1302                         }
1303                 }
1304
1305                 ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc)
1306                 {
1307                         bool is_unsigned = false;
1308                         bool is_long = false;
1309
1310                         if (c != -1){
1311                                 bool scanning = true;
1312                                 do {
1313                                         switch (c){
1314                                         case 'U': case 'u':
1315                                                 if (is_unsigned)
1316                                                         scanning = false;
1317                                                 is_unsigned = true;
1318                                                 get_char ();
1319                                                 break;
1320
1321                                         case 'l':
1322                                                 if (!is_unsigned){
1323                                                         //
1324                                                         // if we have not seen anything in between
1325                                                         // report this error
1326                                                         //
1327                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1328                                                 }
1329
1330                                                 goto case 'L';
1331
1332                                         case 'L':
1333                                                 if (is_long)
1334                                                         scanning = false;
1335                                                 is_long = true;
1336                                                 get_char ();
1337                                                 break;
1338
1339                                         default:
1340                                                 scanning = false;
1341                                                 break;
1342                                         }
1343                                         c = peek_char ();
1344                                 } while (scanning);
1345                         }
1346
1347                         if (is_long && is_unsigned){
1348                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1349                         }
1350
1351                         if (is_unsigned){
1352                                 // uint if possible, or ulong else.
1353
1354                                 if ((ul & 0xffffffff00000000) == 0)
1355                                         return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc);
1356                                 else
1357                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1358                         } else if (is_long){
1359                                 // long if possible, ulong otherwise
1360                                 if ((ul & 0x8000000000000000) != 0)
1361                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1362                                 else
1363                                         return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1364                         } else {
1365                                 // int, uint, long or ulong in that order
1366                                 if ((ul & 0xffffffff00000000) == 0){
1367                                         uint ui = (uint) ul;
1368
1369                                         if ((ui & 0x80000000) != 0)
1370                                                 return new UIntLiteral (context.BuiltinTypes, ui, loc);
1371                                         else
1372                                                 return new IntLiteral (context.BuiltinTypes, (int) ui, loc);
1373                                 } else {
1374                                         if ((ul & 0x8000000000000000) != 0)
1375                                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1376                                         else
1377                                                 return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1378                                 }
1379                         }
1380                 }
1381
1382                 //
1383                 // given `c' as the next char in the input decide whether
1384                 // we need to convert to a special type, and then choose
1385                 // the best representation for the integer
1386                 //
1387                 ILiteralConstant adjust_int (int c, Location loc)
1388                 {
1389                         try {
1390                                 if (number_pos > 9){
1391                                         ulong ul = (uint) (number_builder [0] - '0');
1392
1393                                         for (int i = 1; i < number_pos; i++){
1394                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1395                                         }
1396
1397                                         return integer_type_suffix (ul, c, loc);
1398                                 } else {
1399                                         uint ui = (uint) (number_builder [0] - '0');
1400
1401                                         for (int i = 1; i < number_pos; i++){
1402                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1403                                         }
1404
1405                                         return integer_type_suffix (ui, c, loc);
1406                                 }
1407                         } catch (OverflowException) {
1408                                 Error_NumericConstantTooLong ();
1409                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1410                         }
1411                         catch (FormatException) {
1412                                 Report.Error (1013, Location, "Invalid number");
1413                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1414                         }
1415                 }
1416
1417                 ILiteralConstant adjust_real (TypeCode t, Location loc)
1418                 {
1419                         string s = new string (number_builder, 0, number_pos);
1420                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1421
1422                         switch (t){
1423                         case TypeCode.Decimal:
1424                                 try {
1425                                         return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc);
1426                                 } catch (OverflowException) {
1427                                         Report.Error (594, Location, error_details, "decimal");
1428                                         return new DecimalLiteral (context.BuiltinTypes, 0, loc);
1429                                 }
1430                         case TypeCode.Single:
1431                                 try {
1432                                         return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc);
1433                                 } catch (OverflowException) {
1434                                         Report.Error (594, Location, error_details, "float");
1435                                         return new FloatLiteral (context.BuiltinTypes, 0, loc);
1436                                 }
1437                         default:
1438                                 try {
1439                                         return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc);
1440                                 } catch (OverflowException) {
1441                                         Report.Error (594, loc, error_details, "double");
1442                                         return new DoubleLiteral (context.BuiltinTypes, 0, loc);
1443                                 }
1444                         }
1445                 }
1446
1447                 ILiteralConstant handle_hex (Location loc)
1448                 {
1449                         int d;
1450                         ulong ul;
1451
1452                         get_char ();
1453                         while ((d = peek_char ()) != -1){
1454                                 if (is_hex (d)){
1455                                         number_builder [number_pos++] = (char) d;
1456                                         get_char ();
1457                                 } else
1458                                         break;
1459                         }
1460
1461                         string s = new String (number_builder, 0, number_pos);
1462
1463                         try {
1464                                 if (number_pos <= 8)
1465                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1466                                 else
1467                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1468
1469                                 return integer_type_suffix (ul, peek_char (), loc);
1470                         } catch (OverflowException){
1471                                 Error_NumericConstantTooLong ();
1472                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1473                         }
1474                         catch (FormatException) {
1475                                 Report.Error (1013, Location, "Invalid number");
1476                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1477                         }
1478                 }
1479
1480                 //
1481                 // Invoked if we know we have .digits or digits
1482                 //
1483                 int is_number (int c)
1484                 {
1485                         ILiteralConstant res;
1486
1487 #if FULL_AST
1488                         int read_start = reader.Position - 1;
1489 #endif
1490                         number_pos = 0;
1491                         var loc = Location;
1492
1493                         if (c >= '0' && c <= '9'){
1494                                 if (c == '0'){
1495                                         int peek = peek_char ();
1496
1497                                         if (peek == 'x' || peek == 'X') {
1498                                                 val = res = handle_hex (loc);
1499 #if FULL_AST
1500                                                 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1501 #endif
1502
1503                                                 return Token.LITERAL;
1504                                         }
1505                                 }
1506                                 decimal_digits (c);
1507                                 c = get_char ();
1508                         }
1509
1510                         //
1511                         // We need to handle the case of
1512                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1513                         //
1514                         bool is_real = false;
1515                         if (c == '.'){
1516                                 if (decimal_digits ('.')){
1517                                         is_real = true;
1518                                         c = get_char ();
1519                                 } else {
1520                                         putback ('.');
1521                                         number_pos--;
1522                                         val = res = adjust_int (-1, loc);
1523
1524 #if FULL_AST
1525                                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1526 #endif
1527                                         return Token.LITERAL;
1528                                 }
1529                         }
1530
1531                         if (c == 'e' || c == 'E'){
1532                                 is_real = true;
1533                                 if (number_pos == max_number_size)
1534                                         Error_NumericConstantTooLong ();
1535                                 number_builder [number_pos++] = (char) c;
1536                                 c = get_char ();
1537
1538                                 if (c == '+'){
1539                                         if (number_pos == max_number_size)
1540                                                 Error_NumericConstantTooLong ();
1541                                         number_builder [number_pos++] = '+';
1542                                         c = -1;
1543                                 } else if (c == '-') {
1544                                         if (number_pos == max_number_size)
1545                                                 Error_NumericConstantTooLong ();
1546                                         number_builder [number_pos++] = '-';
1547                                         c = -1;
1548                                 } else {
1549                                         if (number_pos == max_number_size)
1550                                                 Error_NumericConstantTooLong ();
1551                                         number_builder [number_pos++] = '+';
1552                                 }
1553
1554                                 decimal_digits (c);
1555                                 c = get_char ();
1556                         }
1557
1558                         var type = real_type_suffix (c);
1559                         if (type == TypeCode.Empty && !is_real) {
1560                                 putback (c);
1561                                 res = adjust_int (c, loc);
1562                         } else {
1563                                 is_real = true;
1564
1565                                 if (type == TypeCode.Empty) {
1566                                         putback (c);
1567                                 }
1568
1569                                 res = adjust_real (type, loc);
1570                         }
1571
1572                         val = res;
1573
1574 #if FULL_AST
1575                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty ? 1 : 0));
1576 #endif
1577
1578                         return Token.LITERAL;
1579                 }
1580
1581                 //
1582                 // Accepts exactly count (4 or 8) hex, no more no less
1583                 //
1584                 int getHex (int count, out int surrogate, out bool error)
1585                 {
1586                         int i;
1587                         int total = 0;
1588                         int c;
1589                         int top = count != -1 ? count : 4;
1590
1591                         get_char ();
1592                         error = false;
1593                         surrogate = 0;
1594                         for (i = 0; i < top; i++){
1595                                 c = get_char ();
1596
1597                                 if (c >= '0' && c <= '9')
1598                                         c = (int) c - (int) '0';
1599                                 else if (c >= 'A' && c <= 'F')
1600                                         c = (int) c - (int) 'A' + 10;
1601                                 else if (c >= 'a' && c <= 'f')
1602                                         c = (int) c - (int) 'a' + 10;
1603                                 else {
1604                                         error = true;
1605                                         return 0;
1606                                 }
1607
1608                                 total = (total * 16) + c;
1609                                 if (count == -1){
1610                                         int p = peek_char ();
1611                                         if (p == -1)
1612                                                 break;
1613                                         if (!is_hex ((char)p))
1614                                                 break;
1615                                 }
1616                         }
1617
1618                         if (top == 8) {
1619                                 if (total > 0x0010FFFF) {
1620                                         error = true;
1621                                         return 0;
1622                                 }
1623
1624                                 if (total >= 0x00010000) {
1625                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1626                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1627                                 }
1628                         }
1629
1630                         return total;
1631                 }
1632
1633                 int escape (int c, out int surrogate)
1634                 {
1635                         bool error;
1636                         int d;
1637                         int v;
1638
1639                         d = peek_char ();
1640                         if (c != '\\') {
1641                                 surrogate = 0;
1642                                 return c;
1643                         }
1644
1645                         switch (d){
1646                         case 'a':
1647                                 v = '\a'; break;
1648                         case 'b':
1649                                 v = '\b'; break;
1650                         case 'n':
1651                                 v = '\n'; break;
1652                         case 't':
1653                                 v = '\t'; break;
1654                         case 'v':
1655                                 v = '\v'; break;
1656                         case 'r':
1657                                 v = '\r'; break;
1658                         case '\\':
1659                                 v = '\\'; break;
1660                         case 'f':
1661                                 v = '\f'; break;
1662                         case '0':
1663                                 v = 0; break;
1664                         case '"':
1665                                 v = '"'; break;
1666                         case '\'':
1667                                 v = '\''; break;
1668                         case 'x':
1669                                 v = getHex (-1, out surrogate, out error);
1670                                 if (error)
1671                                         goto default;
1672                                 return v;
1673                         case 'u':
1674                         case 'U':
1675                                 return EscapeUnicode (d, out surrogate);
1676                         default:
1677                                 surrogate = 0;
1678                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1679                                 return d;
1680                         }
1681
1682                         get_char ();
1683                         surrogate = 0;
1684                         return v;
1685                 }
1686
1687                 int EscapeUnicode (int ch, out int surrogate)
1688                 {
1689                         bool error;
1690                         if (ch == 'U') {
1691                                 ch = getHex (8, out surrogate, out error);
1692                         } else {
1693                                 ch = getHex (4, out surrogate, out error);
1694                         }
1695
1696                         if (error)
1697                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1698
1699                         return ch;
1700                 }
1701
1702                 int get_char ()
1703                 {
1704                         int x;
1705                         if (putback_char != -1) {
1706                                 x = putback_char;
1707                                 putback_char = -1;
1708                         } else {
1709                                 x = reader.Read ();
1710                         }
1711
1712                         if (x == '\r') {
1713                                 if (peek_char () == '\n') {
1714                                         putback_char = -1;
1715                                 }
1716
1717                                 x = '\n';
1718                                 advance_line ();
1719                         } else if (x == '\n') {
1720                                 advance_line ();
1721                         } else {
1722                                 col++;
1723                         }
1724                         return x;
1725                 }
1726
1727                 void advance_line ()
1728                 {
1729                         line++;
1730                         ref_line++;
1731                         previous_col = col;
1732                         col = 0;
1733                 }
1734
1735                 int peek_char ()
1736                 {
1737                         if (putback_char == -1)
1738                                 putback_char = reader.Read ();
1739                         return putback_char;
1740                 }
1741
1742                 int peek_char2 ()
1743                 {
1744                         if (putback_char != -1)
1745                                 return putback_char;
1746                         return reader.Peek ();
1747                 }
1748
1749                 void putback (int c)
1750                 {
1751                         if (putback_char != -1){
1752                                 Console.WriteLine ("Col: " + col);
1753                                 Console.WriteLine ("Row: " + line);
1754                                 Console.WriteLine ("Name: " + ref_name.Name);
1755                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1756                                 throw new Exception ("This should not happen putback on putback");
1757                         }
1758                         if (c == '\n' || col == 0) {
1759                                 // It won't happen though.
1760                                 line--;
1761                                 ref_line--;
1762                                 col = previous_col;
1763                         }
1764                         else
1765                                 col--;
1766                         putback_char = c;
1767                 }
1768
1769                 public bool advance ()
1770                 {
1771                         return peek_char () != -1 || CompleteOnEOF;
1772                 }
1773
1774                 public Object Value {
1775                         get {
1776                                 return val;
1777                         }
1778                 }
1779
1780                 public Object value ()
1781                 {
1782                         return val;
1783                 }
1784
1785                 public int token ()
1786                 {
1787                         current_token = xtoken ();
1788                         return current_token;
1789                 }
1790
1791                 int TokenizePreprocessorIdentifier (out int c)
1792                 {
1793                         // skip over white space
1794                         do {
1795                                 c = get_char ();
1796                         } while (c == ' ' || c == '\t');
1797
1798
1799                         int pos = 0;
1800                         while (c != -1 && c >= 'a' && c <= 'z') {
1801                                 id_builder[pos++] = (char) c;
1802                                 c = get_char ();
1803                                 if (c == '\\') {
1804                                         int peek = peek_char ();
1805                                         if (peek == 'U' || peek == 'u') {
1806                                                 int surrogate;
1807                                                 c = EscapeUnicode (c, out surrogate);
1808                                                 if (surrogate != 0) {
1809                                                         if (is_identifier_part_character ((char) c)) {
1810                                                                 id_builder[pos++] = (char) c;
1811                                                         }
1812                                                         c = surrogate;
1813                                                 }
1814                                         }
1815                                 }
1816                         }
1817
1818                         return pos;
1819                 }
1820
1821                 PreprocessorDirective get_cmd_arg (out string arg)
1822                 {
1823                         int c;
1824
1825                         tokens_seen = false;
1826                         arg = "";
1827
1828                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorIdentifier (out c));
1829
1830                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
1831                                 return cmd;
1832
1833                         // skip over white space
1834                         while (c == ' ' || c == '\t')
1835                                 c = get_char ();
1836
1837                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
1838                         int pos = 0;
1839
1840                         while (c != -1 && c != '\n') {
1841                                 if (c == '\\' && has_identifier_argument >= 0) {
1842                                         if (has_identifier_argument != 0) {
1843                                                 has_identifier_argument = 1;
1844
1845                                                 int peek = peek_char ();
1846                                                 if (peek == 'U' || peek == 'u') {
1847                                                         int surrogate;
1848                                                         c = EscapeUnicode (c, out surrogate);
1849                                                         if (surrogate != 0) {
1850                                                                 if (is_identifier_part_character ((char) c)) {
1851                                                                         if (pos == value_builder.Length)
1852                                                                                 Array.Resize (ref value_builder, pos * 2);
1853
1854                                                                         value_builder[pos++] = (char) c;
1855                                                                 }
1856                                                                 c = surrogate;
1857                                                         }
1858                                                 }
1859                                         } else {
1860                                                 has_identifier_argument = -1;
1861                                         }
1862                                 } else if (c == '/' && peek_char () == '/') {
1863                                         //
1864                                         // Eat single-line comments
1865                                         //
1866                                         get_char ();
1867                                         do {
1868                                                 c = get_char ();
1869                                         } while (c != -1 && c != '\n');
1870
1871                                         break;
1872                                 }
1873
1874                                 if (pos == value_builder.Length)
1875                                         Array.Resize (ref value_builder, pos * 2);
1876
1877                                 value_builder[pos++] = (char) c;
1878                                 c = get_char ();
1879                         }
1880
1881                         if (pos != 0) {
1882                                 if (pos > max_id_size)
1883                                         arg = new string (value_builder, 0, pos);
1884                                 else
1885                                         arg = InternIdentifier (value_builder, pos);
1886
1887                                 // Eat any trailing whitespaces
1888                                 arg = arg.Trim (simple_whitespaces);
1889                         }
1890
1891                         return cmd;
1892                 }
1893
1894                 //
1895                 // Handles the #line directive
1896                 //
1897                 bool PreProcessLine (string arg)
1898                 {
1899                         if (arg.Length == 0)
1900                                 return false;
1901
1902                         if (arg == "default"){
1903                                 ref_line = line;
1904                                 ref_name = file_name;
1905                                 hidden = false;
1906                                 Location.Push (file_name, ref_name);
1907                                 return true;
1908                         } else if (arg == "hidden"){
1909                                 hidden = true;
1910                                 return true;
1911                         }
1912
1913                         try {
1914                                 int pos;
1915
1916                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1917                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1918                                         pos++;
1919
1920                                         char [] quotes = { '\"' };
1921
1922                                         string name = arg.Substring (pos). Trim (quotes);
1923                                         ref_name = context.LookupFile (file_name, name);
1924                                         file_name.AddIncludeFile (ref_name);
1925                                         hidden = false;
1926                                         Location.Push (file_name, ref_name);
1927                                 } else {
1928                                         ref_line = System.Int32.Parse (arg);
1929                                         hidden = false;
1930                                 }
1931                         } catch {
1932                                 return false;
1933                         }
1934
1935                         return true;
1936                 }
1937
1938                 //
1939                 // Handles #define and #undef
1940                 //
1941                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
1942                 {
1943                         if (ident.Length == 0 || ident == "true" || ident == "false"){
1944                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
1945                                 return;
1946                         }
1947
1948                         if (ident.IndexOfAny (simple_whitespaces) != -1){
1949                                 Error_EndLineExpected ();
1950                                 return;
1951                         }
1952
1953                         if (!is_identifier_start_character (ident [0]))
1954                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
1955
1956                         foreach (char c in ident.Substring (1)){
1957                                 if (!is_identifier_part_character (c)){
1958                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
1959                                         return;
1960                                 }
1961                         }
1962
1963                         if (!caller_is_taking)
1964                                 return;
1965
1966                         if (is_define) {
1967                                 //
1968                                 // #define ident
1969                                 //
1970                                 if (context.Settings.IsConditionalSymbolDefined (ident))
1971                                         return;
1972
1973                                 file_name.AddDefine (ident);
1974                         } else {
1975                                 //
1976                                 // #undef ident
1977                                 //
1978                                 file_name.AddUndefine (ident);
1979                         }
1980                 }
1981
1982                 byte read_hex (out bool error)
1983                 {
1984                         int total;
1985                         int c = get_char ();
1986
1987                         if ((c >= '0') && (c <= '9'))
1988                                 total = (int) c - (int) '0';
1989                         else if ((c >= 'A') && (c <= 'F'))
1990                                 total = (int) c - (int) 'A' + 10;
1991                         else if ((c >= 'a') && (c <= 'f'))
1992                                 total = (int) c - (int) 'a' + 10;
1993                         else {
1994                                 error = true;
1995                                 return 0;
1996                         }
1997
1998                         total *= 16;
1999                         c = get_char ();
2000
2001                         if ((c >= '0') && (c <= '9'))
2002                                 total += (int) c - (int) '0';
2003                         else if ((c >= 'A') && (c <= 'F'))
2004                                 total += (int) c - (int) 'A' + 10;
2005                         else if ((c >= 'a') && (c <= 'f'))
2006                                 total += (int) c - (int) 'a' + 10;
2007                         else {
2008                                 error = true;
2009                                 return 0;
2010                         }
2011
2012                         error = false;
2013                         return (byte) total;
2014                 }
2015
2016                 //
2017                 // Parses #pragma checksum
2018                 //
2019                 bool ParsePragmaChecksum ()
2020                 {
2021                         //
2022                         // The syntax is ` "foo.txt" "{guid}" "hash"'
2023                         //
2024                         int c = get_char ();
2025
2026                         if (c != '"')
2027                                 return false;
2028
2029                         string_builder.Length = 0;
2030                         while (c != -1 && c != '\n') {
2031                                 c = get_char ();
2032                                 if (c == '"') {
2033                                         c = get_char ();
2034                                         break;
2035                                 }
2036
2037                                 string_builder.Append ((char) c);
2038                         }
2039
2040                         if (string_builder.Length == 0) {
2041                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
2042                         }
2043
2044                         // TODO: Any white-spaces count
2045                         if (c != ' ')
2046                                 return false;
2047
2048                         SourceFile file = context.LookupFile (file_name, string_builder.ToString ());
2049
2050                         if (get_char () != '"' || get_char () != '{')
2051                                 return false;
2052
2053                         bool error;
2054                         byte[] guid_bytes = new byte [16];
2055                         int i = 0;
2056
2057                         for (; i < 4; i++) {
2058                                 guid_bytes [i] = read_hex (out error);
2059                                 if (error)
2060                                         return false;
2061                         }
2062
2063                         if (get_char () != '-')
2064                                 return false;
2065
2066                         for (; i < 10; i++) {
2067                                 guid_bytes [i] = read_hex (out error);
2068                                 if (error)
2069                                         return false;
2070
2071                                 guid_bytes [i++] = read_hex (out error);
2072                                 if (error)
2073                                         return false;
2074
2075                                 if (get_char () != '-')
2076                                         return false;
2077                         }
2078
2079                         for (; i < 16; i++) {
2080                                 guid_bytes [i] = read_hex (out error);
2081                                 if (error)
2082                                         return false;
2083                         }
2084
2085                         if (get_char () != '}' || get_char () != '"')
2086                                 return false;
2087
2088                         // TODO: Any white-spaces count
2089                         c = get_char ();
2090                         if (c != ' ')
2091                                 return false;
2092
2093                         if (get_char () != '"')
2094                                 return false;
2095
2096                         // Any length of checksum
2097                         List<byte> checksum_bytes = new List<byte> (16);
2098
2099                         c = peek_char ();
2100                         while (c != '"' && c != -1) {
2101                                 checksum_bytes.Add (read_hex (out error));
2102                                 if (error)
2103                                         return false;
2104
2105                                 c = peek_char ();
2106                         }
2107
2108                         if (c == '/') {
2109                                 ReadSingleLineComment ();
2110                         } else if (get_char () != '"') {
2111                                 return false;
2112                         }
2113
2114                         file.SetChecksum (guid_bytes, checksum_bytes.ToArray ());
2115                         ref_name.AutoGenerated = true;
2116                         return true;
2117                 }
2118
2119                 bool IsTokenIdentifierEqual (char[] identifier)
2120                 {
2121                         for (int i = 0; i < identifier.Length; ++i) {
2122                                 if (identifier[i] != id_builder[i])
2123                                         return false;
2124                         }
2125
2126                         return true;
2127                 }
2128
2129                 int TokenizePragmaNumber (ref int c)
2130                 {
2131                         number_pos = 0;
2132
2133                         int number;
2134
2135                         if (c >= '0' && c <= '9') {
2136                                 decimal_digits (c);
2137                                 uint ui = (uint) (number_builder[0] - '0');
2138
2139                                 try {
2140                                         for (int i = 1; i < number_pos; i++) {
2141                                                 ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2142                                         }
2143
2144                                         number = (int) ui;
2145                                 } catch (OverflowException) {
2146                                         Error_NumericConstantTooLong ();
2147                                         number = -1;
2148                                 }
2149
2150
2151                                 c = get_char ();
2152
2153                                 // skip over white space
2154                                 while (c == ' ' || c == '\t')
2155                                         c = get_char ();
2156
2157                                 if (c == ',') {
2158                                         c = get_char ();
2159                                 }
2160
2161                                 // skip over white space
2162                                 while (c == ' ' || c == '\t')
2163                                         c = get_char ();
2164                         } else {
2165                                 number = -1;
2166                                 if (c == '/') {
2167                                         ReadSingleLineComment ();
2168                                 } else {
2169                                         Report.Warning (1692, 1, Location, "Invalid number");
2170
2171                                         // Read everything till the end of the line or file
2172                                         do {
2173                                                 c = get_char ();
2174                                         } while (c != -1 && c != '\n');
2175                                 }
2176                         }
2177
2178                         return number;
2179                 }
2180
2181                 void ReadSingleLineComment ()
2182                 {
2183                         if (peek_char () != '/')
2184                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2185
2186                         // Read everything till the end of the line or file
2187                         int c;
2188                         do {
2189                                 c = get_char ();
2190                         } while (c != -1 && c != '\n');
2191                 }
2192
2193                 /// <summary>
2194                 /// Handles #pragma directive
2195                 /// </summary>
2196                 void ParsePragmaDirective (string arg)
2197                 {
2198                         int c;
2199                         int length = TokenizePreprocessorIdentifier (out c);
2200                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2201                                 length = TokenizePreprocessorIdentifier (out c);
2202
2203                                 //
2204                                 // #pragma warning disable
2205                                 // #pragma warning restore
2206                                 //
2207                                 if (length == pragma_warning_disable.Length) {
2208                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2209                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2210                                                 // skip over white space
2211                                                 while (c == ' ' || c == '\t')
2212                                                         c = get_char ();
2213
2214                                                 var loc = Location;
2215
2216                                                 if (c == '\n' || c == '/') {
2217                                                         if (c == '/')
2218                                                                 ReadSingleLineComment ();
2219
2220                                                         //
2221                                                         // Disable/Restore all warnings
2222                                                         //
2223                                                         if (disable) {
2224                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2225                                                         } else {
2226                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2227                                                         }
2228                                                 } else {
2229                                                         //
2230                                                         // Disable/Restore a warning or group of warnings
2231                                                         //
2232                                                         int code;
2233                                                         do {
2234                                                                 code = TokenizePragmaNumber (ref c);
2235                                                                 if (code > 0) {
2236                                                                         if (disable) {
2237                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, Report);
2238                                                                         } else {
2239                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, Report);
2240                                                                         }
2241                                                                 }
2242                                                         } while (code >= 0 && c != '\n' && c != -1);
2243                                                 }
2244
2245                                                 return;
2246                                         }
2247                                 }
2248
2249                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2250                                 return;
2251                         }
2252
2253                         //
2254                         // #pragma checksum
2255                         //
2256                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2257                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2258                                         Report.Warning (1695, 1, Location,
2259                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2260                                 }
2261
2262                                 return;
2263                         }
2264
2265                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2266                 }
2267
2268                 bool eval_val (string s)
2269                 {
2270                         if (s == "true")
2271                                 return true;
2272                         if (s == "false")
2273                                 return false;
2274
2275                         return file_name.IsConditionalDefined (context, s);
2276                 }
2277
2278                 bool pp_primary (ref string s)
2279                 {
2280                         s = s.Trim ();
2281                         int len = s.Length;
2282
2283                         if (len > 0){
2284                                 char c = s [0];
2285
2286                                 if (c == '('){
2287                                         s = s.Substring (1);
2288                                         bool val = pp_expr (ref s, false);
2289                                         if (s.Length > 0 && s [0] == ')'){
2290                                                 s = s.Substring (1);
2291                                                 return val;
2292                                         }
2293                                         Error_InvalidDirective ();
2294                                         return false;
2295                                 }
2296
2297                                 if (is_identifier_start_character (c)){
2298                                         int j = 1;
2299
2300                                         while (j < len){
2301                                                 c = s [j];
2302
2303                                                 if (is_identifier_part_character (c)){
2304                                                         j++;
2305                                                         continue;
2306                                                 }
2307                                                 bool v = eval_val (s.Substring (0, j));
2308                                                 s = s.Substring (j);
2309                                                 return v;
2310                                         }
2311                                         bool vv = eval_val (s);
2312                                         s = "";
2313                                         return vv;
2314                                 }
2315                         }
2316                         Error_InvalidDirective ();
2317                         return false;
2318                 }
2319
2320                 bool pp_unary (ref string s)
2321                 {
2322                         s = s.Trim ();
2323                         int len = s.Length;
2324
2325                         if (len > 0){
2326                                 if (s [0] == '!'){
2327                                         if (len > 1 && s [1] == '='){
2328                                                 Error_InvalidDirective ();
2329                                                 return false;
2330                                         }
2331                                         s = s.Substring (1);
2332                                         return ! pp_primary (ref s);
2333                                 } else
2334                                         return pp_primary (ref s);
2335                         } else {
2336                                 Error_InvalidDirective ();
2337                                 return false;
2338                         }
2339                 }
2340
2341                 bool pp_eq (ref string s)
2342                 {
2343                         bool va = pp_unary (ref s);
2344
2345                         s = s.Trim ();
2346                         int len = s.Length;
2347                         if (len > 0){
2348                                 if (s [0] == '='){
2349                                         if (len > 2 && s [1] == '='){
2350                                                 s = s.Substring (2);
2351                                                 return va == pp_unary (ref s);
2352                                         } else {
2353                                                 Error_InvalidDirective ();
2354                                                 return false;
2355                                         }
2356                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2357                                         s = s.Substring (2);
2358
2359                                         return va != pp_unary (ref s);
2360
2361                                 }
2362                         }
2363
2364                         return va;
2365
2366                 }
2367
2368                 bool pp_and (ref string s)
2369                 {
2370                         bool va = pp_eq (ref s);
2371
2372                         s = s.Trim ();
2373                         int len = s.Length;
2374                         if (len > 0){
2375                                 if (s [0] == '&'){
2376                                         if (len > 2 && s [1] == '&'){
2377                                                 s = s.Substring (2);
2378                                                 return (va & pp_and (ref s));
2379                                         } else {
2380                                                 Error_InvalidDirective ();
2381                                                 return false;
2382                                         }
2383                                 }
2384                         }
2385                         return va;
2386                 }
2387
2388                 //
2389                 // Evaluates an expression for `#if' or `#elif'
2390                 //
2391                 bool pp_expr (ref string s, bool isTerm)
2392                 {
2393                         bool va = pp_and (ref s);
2394                         s = s.Trim ();
2395                         int len = s.Length;
2396                         if (len > 0){
2397                                 char c = s [0];
2398
2399                                 if (c == '|'){
2400                                         if (len > 2 && s [1] == '|'){
2401                                                 s = s.Substring (2);
2402                                                 return va | pp_expr (ref s, isTerm);
2403                                         } else {
2404                                                 Error_InvalidDirective ();
2405                                                 return false;
2406                                         }
2407                                 }
2408                                 if (isTerm) {
2409                                         Error_EndLineExpected ();
2410                                         return false;
2411                                 }
2412                         }
2413
2414                         return va;
2415                 }
2416
2417                 bool eval (string s)
2418                 {
2419                         bool v = pp_expr (ref s, true);
2420                         s = s.Trim ();
2421                         if (s.Length != 0){
2422                                 return false;
2423                         }
2424
2425                         return v;
2426                 }
2427
2428                 void Error_NumericConstantTooLong ()
2429                 {
2430                         Report.Error (1021, Location, "Integral constant is too large");
2431                 }
2432
2433                 void Error_InvalidDirective ()
2434                 {
2435                         Report.Error (1517, Location, "Invalid preprocessor directive");
2436                 }
2437
2438                 void Error_UnexpectedDirective (string extra)
2439                 {
2440                         Report.Error (
2441                                 1028, Location,
2442                                 "Unexpected processor directive ({0})", extra);
2443                 }
2444
2445                 void Error_TokensSeen ()
2446                 {
2447                         Report.Error (1032, Location,
2448                                 "Cannot define or undefine preprocessor symbols after first token in file");
2449                 }
2450
2451                 void Eror_WrongPreprocessorLocation ()
2452                 {
2453                         Report.Error (1040, Location,
2454                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2455                 }
2456
2457                 void Error_EndLineExpected ()
2458                 {
2459                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2460                 }
2461
2462                 //
2463                 // Raises a warning when tokenizer found documentation comment
2464                 // on unexpected place
2465                 //
2466                 void WarningMisplacedComment (Location loc)
2467                 {
2468                         if (doc_state != XmlCommentState.Error) {
2469                                 doc_state = XmlCommentState.Error;
2470                                 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element");
2471                         }
2472                 }
2473
2474                 //
2475                 // if true, then the code continues processing the code
2476                 // if false, the code stays in a loop until another directive is
2477                 // reached.
2478                 // When caller_is_taking is false we ignore all directives except the ones
2479                 // which can help us to identify where the #if block ends
2480                 bool ParsePreprocessingDirective (bool caller_is_taking)
2481                 {
2482                         string arg;
2483                         bool region_directive = false;
2484
2485                         var directive = get_cmd_arg (out arg);
2486
2487                         //
2488                         // The first group of pre-processing instructions is always processed
2489                         //
2490                         switch (directive) {
2491                         case PreprocessorDirective.Region:
2492                                 region_directive = true;
2493                                 arg = "true";
2494                                 goto case PreprocessorDirective.If;
2495
2496                         case PreprocessorDirective.Endregion:
2497                                 if (ifstack == null || ifstack.Count == 0){
2498                                         Error_UnexpectedDirective ("no #region for this #endregion");
2499                                         return true;
2500                                 }
2501                                 int pop = ifstack.Pop ();
2502
2503                                 if ((pop & REGION) == 0)
2504                                         Report.Error (1027, Location, "Expected `#endif' directive");
2505
2506                                 return caller_is_taking;
2507
2508                         case PreprocessorDirective.If:
2509                                 if (ifstack == null)
2510                                         ifstack = new Stack<int> (2);
2511
2512                                 int flags = region_directive ? REGION : 0;
2513                                 if (ifstack.Count == 0){
2514                                         flags |= PARENT_TAKING;
2515                                 } else {
2516                                         int state = ifstack.Peek ();
2517                                         if ((state & TAKING) != 0) {
2518                                                 flags |= PARENT_TAKING;
2519                                         }
2520                                 }
2521
2522                                 if (eval (arg) && caller_is_taking) {
2523                                         ifstack.Push (flags | TAKING);
2524                                         return true;
2525                                 }
2526                                 ifstack.Push (flags);
2527                                 return false;
2528
2529                         case PreprocessorDirective.Endif:
2530                                 if (ifstack == null || ifstack.Count == 0){
2531                                         Error_UnexpectedDirective ("no #if for this #endif");
2532                                         return true;
2533                                 } else {
2534                                         pop = ifstack.Pop ();
2535
2536                                         if ((pop & REGION) != 0)
2537                                                 Report.Error (1038, Location, "#endregion directive expected");
2538
2539                                         if (arg.Length != 0) {
2540                                                 Error_EndLineExpected ();
2541                                         }
2542
2543                                         if (ifstack.Count == 0)
2544                                                 return true;
2545
2546                                         int state = ifstack.Peek ();
2547                                         return (state & TAKING) != 0;
2548                                 }
2549
2550                         case PreprocessorDirective.Elif:
2551                                 if (ifstack == null || ifstack.Count == 0){
2552                                         Error_UnexpectedDirective ("no #if for this #elif");
2553                                         return true;
2554                                 } else {
2555                                         int state = ifstack.Pop ();
2556
2557                                         if ((state & REGION) != 0) {
2558                                                 Report.Error (1038, Location, "#endregion directive expected");
2559                                                 return true;
2560                                         }
2561
2562                                         if ((state & ELSE_SEEN) != 0){
2563                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2564                                                 return true;
2565                                         }
2566
2567                                         if ((state & TAKING) != 0) {
2568                                                 ifstack.Push (0);
2569                                                 return false;
2570                                         }
2571
2572                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2573                                                 ifstack.Push (state | TAKING);
2574                                                 return true;
2575                                         }
2576
2577                                         ifstack.Push (state);
2578                                         return false;
2579                                 }
2580
2581                         case PreprocessorDirective.Else:
2582                                 if (ifstack == null || ifstack.Count == 0){
2583                                         Error_UnexpectedDirective ("no #if for this #else");
2584                                         return true;
2585                                 } else {
2586                                         int state = ifstack.Peek ();
2587
2588                                         if ((state & REGION) != 0) {
2589                                                 Report.Error (1038, Location, "#endregion directive expected");
2590                                                 return true;
2591                                         }
2592
2593                                         if ((state & ELSE_SEEN) != 0){
2594                                                 Error_UnexpectedDirective ("#else within #else");
2595                                                 return true;
2596                                         }
2597
2598                                         ifstack.Pop ();
2599
2600                                         if (arg.Length != 0) {
2601                                                 Error_EndLineExpected ();
2602                                                 return true;
2603                                         }
2604
2605                                         bool ret = false;
2606                                         if ((state & PARENT_TAKING) != 0) {
2607                                                 ret = (state & TAKING) == 0;
2608
2609                                                 if (ret)
2610                                                         state |= TAKING;
2611                                                 else
2612                                                         state &= ~TAKING;
2613                                         }
2614
2615                                         ifstack.Push (state | ELSE_SEEN);
2616
2617                                         return ret;
2618                                 }
2619                         case PreprocessorDirective.Define:
2620                                 if (any_token_seen){
2621                                         Error_TokensSeen ();
2622                                         return caller_is_taking;
2623                                 }
2624                                 PreProcessDefinition (true, arg, caller_is_taking);
2625                                 return caller_is_taking;
2626
2627                         case PreprocessorDirective.Undef:
2628                                 if (any_token_seen){
2629                                         Error_TokensSeen ();
2630                                         return caller_is_taking;
2631                                 }
2632                                 PreProcessDefinition (false, arg, caller_is_taking);
2633                                 return caller_is_taking;
2634
2635                         case PreprocessorDirective.Invalid:
2636                                 Report.Error (1024, Location, "Wrong preprocessor directive");
2637                                 return true;
2638                         }
2639
2640                         //
2641                         // These are only processed if we are in a `taking' block
2642                         //
2643                         if (!caller_is_taking)
2644                                 return false;
2645
2646                         switch (directive){
2647                         case PreprocessorDirective.Error:
2648                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2649                                 return true;
2650
2651                         case PreprocessorDirective.Warning:
2652                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2653                                 return true;
2654
2655                         case PreprocessorDirective.Pragma:
2656                                 if (context.Settings.Version == LanguageVersion.ISO_1) {
2657                                         Report.FeatureIsNotAvailable (context, Location, "#pragma");
2658                                 }
2659
2660                                 ParsePragmaDirective (arg);
2661                                 return true;
2662
2663                         case PreprocessorDirective.Line:
2664                                 if (!PreProcessLine (arg))
2665                                         Report.Error (
2666                                                 1576, Location,
2667                                                 "The line number specified for #line directive is missing or invalid");
2668                                 return caller_is_taking;
2669                         }
2670
2671                         throw new NotImplementedException (directive.ToString ());
2672                 }
2673
2674                 private int consume_string (bool quoted)
2675                 {
2676                         int c;
2677                         int pos = 0;
2678                         Location start_location = Location;
2679                         if (quoted)
2680                                 start_location = start_location - 1;
2681
2682 #if FULL_AST
2683                         int reader_pos = reader.Position;
2684 #endif
2685
2686                         while (true){
2687                                 c = get_char ();
2688                                 if (c == '"') {
2689                                         if (quoted && peek_char () == '"') {
2690                                                 if (pos == value_builder.Length)
2691                                                         Array.Resize (ref value_builder, pos * 2);
2692
2693                                                 value_builder[pos++] = (char) c;
2694                                                 get_char ();
2695                                                 continue;
2696                                         }
2697
2698                                         string s;
2699                                         if (pos == 0)
2700                                                 s = string.Empty;
2701                                         else if (pos <= 4)
2702                                                 s = InternIdentifier (value_builder, pos);
2703                                         else
2704                                                 s = new string (value_builder, 0, pos);
2705
2706                                         ILiteralConstant res = new StringLiteral (context.BuiltinTypes, s, start_location);
2707                                         val = res;
2708 #if FULL_AST
2709                                         res.ParsedValue = quoted ?
2710                                                 reader.ReadChars (reader_pos - 2, reader.Position - 1) :
2711                                                 reader.ReadChars (reader_pos - 1, reader.Position);
2712 #endif
2713
2714                                         return Token.LITERAL;
2715                                 }
2716
2717                                 if (c == '\n') {
2718                                         if (!quoted)
2719                                                 Report.Error (1010, Location, "Newline in constant");
2720                                 } else if (c == '\\' && !quoted) {
2721                                         int surrogate;
2722                                         c = escape (c, out surrogate);
2723                                         if (c == -1)
2724                                                 return Token.ERROR;
2725                                         if (surrogate != 0) {
2726                                                 if (pos == value_builder.Length)
2727                                                         Array.Resize (ref value_builder, pos * 2);
2728
2729                                                 value_builder[pos++] = (char) c;
2730                                                 c = surrogate;
2731                                         }
2732                                 } else if (c == -1) {
2733                                         Report.Error (1039, Location, "Unterminated string literal");
2734                                         return Token.EOF;
2735                                 }
2736
2737                                 if (pos == value_builder.Length)
2738                                         Array.Resize (ref value_builder, pos * 2);
2739
2740                                 value_builder[pos++] = (char) c;
2741                         }
2742                 }
2743
2744                 private int consume_identifier (int s)
2745                 {
2746                         int res = consume_identifier (s, false);
2747
2748                         if (doc_state == XmlCommentState.Allowed)
2749                                 doc_state = XmlCommentState.NotAllowed;
2750
2751                         return res;
2752                 }
2753
2754                 int consume_identifier (int c, bool quoted)
2755                 {
2756                         //
2757                         // This method is very performance sensitive. It accounts
2758                         // for approximately 25% of all parser time
2759                         //
2760
2761                         int pos = 0;
2762                         int column = col;
2763                         if (quoted)
2764                                 --column;
2765
2766                         if (c == '\\') {
2767                                 int surrogate;
2768                                 c = escape (c, out surrogate);
2769                                 if (surrogate != 0) {
2770                                         id_builder [pos++] = (char) c;
2771                                         c = surrogate;
2772                                 }
2773                         }
2774
2775                         id_builder [pos++] = (char) c;
2776
2777                         try {
2778                                 while (true) {
2779                                         c = reader.Read ();
2780
2781                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
2782                                                 id_builder [pos++] = (char) c;
2783                                                 continue;
2784                                         }
2785
2786                                         if (c < 0x80) {
2787                                                 if (c == '\\') {
2788                                                         int surrogate;
2789                                                         c = escape (c, out surrogate);
2790                                                         if (is_identifier_part_character ((char) c))
2791                                                                 id_builder[pos++] = (char) c;
2792
2793                                                         if (surrogate != 0) {
2794                                                                 c = surrogate;
2795                                                         }
2796
2797                                                         continue;
2798                                                 }
2799                                         } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
2800                                                 id_builder [pos++] = (char) c;
2801                                                 continue;
2802                                         }
2803
2804                                         putback_char = c;
2805                                         break;
2806                                 }
2807                         } catch (IndexOutOfRangeException) {
2808                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2809                                 --pos;
2810                                 col += pos;
2811                         }
2812
2813                         col += pos - 1;
2814
2815                         //
2816                         // Optimization: avoids doing the keyword lookup
2817                         // on uppercase letters
2818                         //
2819                         if (id_builder [0] >= '_' && !quoted) {
2820                                 int keyword = GetKeyword (id_builder, pos);
2821                                 if (keyword != -1) {
2822                                         val = LocatedToken.Create (null, ref_line, column);
2823                                         return keyword;
2824                                 }
2825                         }
2826
2827                         string s = InternIdentifier (id_builder, pos);
2828                         val = LocatedToken.Create (s, ref_line, column);
2829                         if (quoted && parsing_attribute_section)
2830                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2831
2832                         return Token.IDENTIFIER;
2833                 }
2834
2835                 static string InternIdentifier (char[] charBuffer, int length)
2836                 {
2837                         //
2838                         // Keep identifiers in an array of hashtables to avoid needless
2839                         // allocations
2840                         //
2841                         var identifiers_group = identifiers[length];
2842                         string s;
2843                         if (identifiers_group != null) {
2844                                 if (identifiers_group.TryGetValue (charBuffer, out s)) {
2845                                         return s;
2846                                 }
2847                         } else {
2848                                 // TODO: this should be number of files dependant
2849                                 // corlib compilation peaks at 1000 and System.Core at 150
2850                                 int capacity = length > 20 ? 10 : 100;
2851                                 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length));
2852                                 identifiers[length] = identifiers_group;
2853                         }
2854
2855                         char[] chars = new char[length];
2856                         Array.Copy (charBuffer, chars, length);
2857
2858                         s = new string (charBuffer, 0, length);
2859                         identifiers_group.Add (chars, s);
2860                         return s;
2861                 }
2862
2863                 public int xtoken ()
2864                 {
2865                         int d, c;
2866
2867                         // Whether we have seen comments on the current line
2868                         bool comments_seen = false;
2869                         while ((c = get_char ()) != -1) {
2870                                 switch (c) {
2871                                 case '\t':
2872                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
2873                                         continue;
2874
2875                                 case ' ':
2876                                 case '\f':
2877                                 case '\v':
2878                                 case 0xa0:
2879                                 case 0:
2880                                 case 0xFEFF:    // Ignore BOM anywhere in the file
2881                                         continue;
2882
2883 /*                              This is required for compatibility with .NET
2884                                 case 0xEF:
2885                                         if (peek_char () == 0xBB) {
2886                                                 PushPosition ();
2887                                                 get_char ();
2888                                                 if (get_char () == 0xBF)
2889                                                         continue;
2890                                                 PopPosition ();
2891                                         }
2892                                         break;
2893 */
2894                                 case '\\':
2895                                         tokens_seen = true;
2896                                         return consume_identifier (c);
2897
2898                                 case '{':
2899                                         val = LocatedToken.Create (ref_line, col);
2900                                         return Token.OPEN_BRACE;
2901                                 case '}':
2902                                         val = LocatedToken.Create (ref_line, col);
2903                                         return Token.CLOSE_BRACE;
2904                                 case '[':
2905                                         // To block doccomment inside attribute declaration.
2906                                         if (doc_state == XmlCommentState.Allowed)
2907                                                 doc_state = XmlCommentState.NotAllowed;
2908
2909                                         val = LocatedToken.Create (ref_line, col);
2910
2911                                         if (parsing_block == 0 || lambda_arguments_parsing)
2912                                                 return Token.OPEN_BRACKET;
2913
2914                                         int next = peek_char ();
2915                                         switch (next) {
2916                                         case ']':
2917                                         case ',':
2918                                                 return Token.OPEN_BRACKET;
2919
2920                                         case ' ':
2921                                         case '\f':
2922                                         case '\v':
2923                                         case '\r':
2924                                         case '\n':
2925                                         case '/':
2926                                                 next = peek_token ();
2927                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
2928                                                         return Token.OPEN_BRACKET;
2929
2930                                                 return Token.OPEN_BRACKET_EXPR;
2931                                         default:
2932                                                 return Token.OPEN_BRACKET_EXPR;
2933                                         }
2934                                 case ']':
2935                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2936                                         return Token.CLOSE_BRACKET;
2937                                 case '(':
2938                                         val = LocatedToken.Create (ref_line, col);
2939                                         //
2940                                         // An expression versions of parens can appear in block context only
2941                                         //
2942                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
2943
2944                                                 //
2945                                                 // Optmize most common case where we know that parens
2946                                                 // is not special
2947                                                 //
2948                                                 switch (current_token) {
2949                                                 case Token.IDENTIFIER:
2950                                                 case Token.IF:
2951                                                 case Token.FOR:
2952                                                 case Token.FOREACH:
2953                                                 case Token.TYPEOF:
2954                                                 case Token.WHILE:
2955                                                 case Token.USING:
2956                                                 case Token.DEFAULT:
2957                                                 case Token.DELEGATE:
2958                                                 case Token.OP_GENERICS_GT:
2959                                                         return Token.OPEN_PARENS;
2960                                                 }
2961
2962                                                 // Optimize using peek
2963                                                 int xx = peek_char ();
2964                                                 switch (xx) {
2965                                                 case '(':
2966                                                 case '\'':
2967                                                 case '"':
2968                                                 case '0':
2969                                                 case '1':
2970                                                         return Token.OPEN_PARENS;
2971                                                 }
2972
2973                                                 lambda_arguments_parsing = true;
2974                                                 PushPosition ();
2975                                                 d = TokenizeOpenParens ();
2976                                                 PopPosition ();
2977                                                 lambda_arguments_parsing = false;
2978                                                 return d;
2979                                         }
2980
2981                                         return Token.OPEN_PARENS;
2982                                 case ')':
2983                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2984                                         return Token.CLOSE_PARENS;
2985                                 case ',':
2986                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2987                                         return Token.COMMA;
2988                                 case ';':
2989                                         LocatedToken.CreateOptional (ref_line, col, ref val);
2990                                         return Token.SEMICOLON;
2991                                 case '~':
2992                                         val = LocatedToken.Create (ref_line, col);
2993                                         return Token.TILDE;
2994                                 case '?':
2995                                         val = LocatedToken.Create (ref_line, col);
2996                                         return TokenizePossibleNullableType ();
2997                                 case '<':
2998                                         val = LocatedToken.Create (ref_line, col);
2999                                         if (parsing_generic_less_than++ > 0)
3000                                                 return Token.OP_GENERICS_LT;
3001
3002                                         return TokenizeLessThan ();
3003
3004                                 case '>':
3005                                         val = LocatedToken.Create (ref_line, col);
3006                                         d = peek_char ();
3007
3008                                         if (d == '='){
3009                                                 get_char ();
3010                                                 return Token.OP_GE;
3011                                         }
3012
3013                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
3014                                                 parsing_generic_less_than--;
3015                                                 return Token.OP_GENERICS_GT;
3016                                         }
3017
3018                                         if (d == '>') {
3019                                                 get_char ();
3020                                                 d = peek_char ();
3021
3022                                                 if (d == '=') {
3023                                                         get_char ();
3024                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
3025                                                 }
3026                                                 return Token.OP_SHIFT_RIGHT;
3027                                         }
3028
3029                                         return Token.OP_GT;
3030
3031                                 case '+':
3032                                         val = LocatedToken.Create (ref_line, col);
3033                                         d = peek_char ();
3034                                         if (d == '+') {
3035                                                 d = Token.OP_INC;
3036                                         } else if (d == '=') {
3037                                                 d = Token.OP_ADD_ASSIGN;
3038                                         } else {
3039                                                 return Token.PLUS;
3040                                         }
3041                                         get_char ();
3042                                         return d;
3043
3044                                 case '-':
3045                                         val = LocatedToken.Create (ref_line, col);
3046                                         d = peek_char ();
3047                                         if (d == '-') {
3048                                                 d = Token.OP_DEC;
3049                                         } else if (d == '=')
3050                                                 d = Token.OP_SUB_ASSIGN;
3051                                         else if (d == '>')
3052                                                 d = Token.OP_PTR;
3053                                         else {
3054                                                 return Token.MINUS;
3055                                         }
3056                                         get_char ();
3057                                         return d;
3058
3059                                 case '!':
3060                                         val = LocatedToken.Create (ref_line, col);
3061                                         if (peek_char () == '='){
3062                                                 get_char ();
3063                                                 return Token.OP_NE;
3064                                         }
3065                                         return Token.BANG;
3066
3067                                 case '=':
3068                                         val = LocatedToken.Create (ref_line, col);
3069                                         d = peek_char ();
3070                                         if (d == '='){
3071                                                 get_char ();
3072                                                 return Token.OP_EQ;
3073                                         }
3074                                         if (d == '>'){
3075                                                 get_char ();
3076                                                 return Token.ARROW;
3077                                         }
3078
3079                                         return Token.ASSIGN;
3080
3081                                 case '&':
3082                                         val = LocatedToken.Create (ref_line, col);
3083                                         d = peek_char ();
3084                                         if (d == '&'){
3085                                                 get_char ();
3086                                                 return Token.OP_AND;
3087                                         }
3088                                         if (d == '='){
3089                                                 get_char ();
3090                                                 return Token.OP_AND_ASSIGN;
3091                                         }
3092                                         return Token.BITWISE_AND;
3093
3094                                 case '|':
3095                                         val = LocatedToken.Create (ref_line, col);
3096                                         d = peek_char ();
3097                                         if (d == '|'){
3098                                                 get_char ();
3099                                                 return Token.OP_OR;
3100                                         }
3101                                         if (d == '='){
3102                                                 get_char ();
3103                                                 return Token.OP_OR_ASSIGN;
3104                                         }
3105                                         return Token.BITWISE_OR;
3106
3107                                 case '*':
3108                                         val = LocatedToken.Create (ref_line, col);
3109                                         if (peek_char () == '='){
3110                                                 get_char ();
3111                                                 return Token.OP_MULT_ASSIGN;
3112                                         }
3113                                         return Token.STAR;
3114
3115                                 case '/':
3116                                         d = peek_char ();
3117                                         if (d == '='){
3118                                                 val = LocatedToken.Create (ref_line, col);
3119                                                 get_char ();
3120                                                 return Token.OP_DIV_ASSIGN;
3121                                         }
3122
3123                                         // Handle double-slash comments.
3124                                         if (d == '/'){
3125                                                 get_char ();
3126                                                 if (doc_processing) {
3127                                                         if (peek_char () == '/') {
3128                                                                 get_char ();
3129                                                                 // Don't allow ////.
3130                                                                 if ((d = peek_char ()) != '/') {
3131                                                                         if (doc_state == XmlCommentState.Allowed)
3132                                                                                 handle_one_line_xml_comment ();
3133                                                                         else if (doc_state == XmlCommentState.NotAllowed)
3134                                                                                 WarningMisplacedComment (Location - 3);
3135                                                                 }
3136                                                         } else {
3137                                                                 if (xml_comment_buffer.Length > 0)
3138                                                                         doc_state = XmlCommentState.NotAllowed;
3139                                                         }
3140                                                 }
3141
3142                                                 while ((d = get_char ()) != -1 && d != '\n');
3143
3144                                                 any_token_seen |= tokens_seen;
3145                                                 tokens_seen = false;
3146                                                 comments_seen = false;
3147                                                 continue;
3148                                         } else if (d == '*'){
3149                                                 get_char ();
3150                                                 bool docAppend = false;
3151                                                 if (doc_processing && peek_char () == '*') {
3152                                                         get_char ();
3153                                                         // But when it is /**/, just do nothing.
3154                                                         if (peek_char () == '/') {
3155                                                                 get_char ();
3156                                                                 continue;
3157                                                         }
3158                                                         if (doc_state == XmlCommentState.Allowed)
3159                                                                 docAppend = true;
3160                                                         else if (doc_state == XmlCommentState.NotAllowed) {
3161                                                                 WarningMisplacedComment (Location - 2);
3162                                                         }
3163                                                 }
3164
3165                                                 int current_comment_start = 0;
3166                                                 if (docAppend) {
3167                                                         current_comment_start = xml_comment_buffer.Length;
3168                                                         xml_comment_buffer.Append (Environment.NewLine);
3169                                                 }
3170
3171                                                 while ((d = get_char ()) != -1){
3172                                                         if (d == '*' && peek_char () == '/'){
3173                                                                 get_char ();
3174                                                                 comments_seen = true;
3175                                                                 break;
3176                                                         }
3177                                                         if (docAppend)
3178                                                                 xml_comment_buffer.Append ((char) d);
3179
3180                                                         if (d == '\n'){
3181                                                                 any_token_seen |= tokens_seen;
3182                                                                 tokens_seen = false;
3183                                                                 //
3184                                                                 // Reset 'comments_seen' just to be consistent.
3185                                                                 // It doesn't matter either way, here.
3186                                                                 //
3187                                                                 comments_seen = false;
3188                                                         }
3189                                                 }
3190                                                 if (!comments_seen)
3191                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3192
3193                                                 if (docAppend)
3194                                                         update_formatted_doc_comment (current_comment_start);
3195                                                 continue;
3196                                         }
3197                                         val = LocatedToken.Create (ref_line, col);
3198                                         return Token.DIV;
3199
3200                                 case '%':
3201                                         val = LocatedToken.Create (ref_line, col);
3202                                         if (peek_char () == '='){
3203                                                 get_char ();
3204                                                 return Token.OP_MOD_ASSIGN;
3205                                         }
3206                                         return Token.PERCENT;
3207
3208                                 case '^':
3209                                         val = LocatedToken.Create (ref_line, col);
3210                                         if (peek_char () == '='){
3211                                                 get_char ();
3212                                                 return Token.OP_XOR_ASSIGN;
3213                                         }
3214                                         return Token.CARRET;
3215
3216                                 case ':':
3217                                         val = LocatedToken.Create (ref_line, col);
3218                                         if (peek_char () == ':') {
3219                                                 get_char ();
3220                                                 return Token.DOUBLE_COLON;
3221                                         }
3222                                         return Token.COLON;
3223
3224                                 case '0': case '1': case '2': case '3': case '4':
3225                                 case '5': case '6': case '7': case '8': case '9':
3226                                         tokens_seen = true;
3227                                         return is_number (c);
3228
3229                                 case '\n': // white space
3230                                         any_token_seen |= tokens_seen;
3231                                         tokens_seen = false;
3232                                         comments_seen = false;
3233                                         continue;
3234
3235                                 case '.':
3236                                         tokens_seen = true;
3237                                         d = peek_char ();
3238                                         if (d >= '0' && d <= '9')
3239                                                 return is_number (c);
3240
3241                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3242                                         return Token.DOT;
3243
3244                                 case '#':
3245                                         if (tokens_seen || comments_seen) {
3246                                                 Eror_WrongPreprocessorLocation ();
3247                                                 return Token.ERROR;
3248                                         }
3249
3250                                         if (ParsePreprocessingDirective (true))
3251                                                 continue;
3252
3253                                         bool directive_expected = false;
3254                                         while ((c = get_char ()) != -1) {
3255                                                 if (col == 1) {
3256                                                         directive_expected = true;
3257                                                 } else if (!directive_expected) {
3258                                                         // TODO: Implement comment support for disabled code and uncomment this code
3259 //                                                      if (c == '#') {
3260 //                                                              Eror_WrongPreprocessorLocation ();
3261 //                                                              return Token.ERROR;
3262 //                                                      }
3263                                                         continue;
3264                                                 }
3265
3266                                                 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' )
3267                                                         continue;
3268
3269                                                 if (c == '#') {
3270                                                         if (ParsePreprocessingDirective (false))
3271                                                                 break;
3272                                                 }
3273                                                 directive_expected = false;
3274                                         }
3275
3276                                         if (c != -1) {
3277                                                 tokens_seen = false;
3278                                                 continue;
3279                                         }
3280
3281                                         return Token.EOF;
3282
3283                                 case '"':
3284                                         return consume_string (false);
3285
3286                                 case '\'':
3287                                         return TokenizeBackslash ();
3288
3289                                 case '@':
3290                                         c = get_char ();
3291                                         if (c == '"') {
3292                                                 tokens_seen = true;
3293                                                 return consume_string (true);
3294                                         }
3295
3296                                         if (is_identifier_start_character (c)){
3297                                                 return consume_identifier (c, true);
3298                                         }
3299
3300                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3301                                         return Token.ERROR;
3302
3303                                 case EvalStatementParserCharacter:
3304                                         return Token.EVAL_STATEMENT_PARSER;
3305                                 case EvalCompilationUnitParserCharacter:
3306                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3307                                 case EvalUsingDeclarationsParserCharacter:
3308                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3309                                 case DocumentationXref:
3310                                         return Token.DOC_SEE;
3311                                 }
3312
3313                                 if (is_identifier_start_character (c)) {
3314                                         tokens_seen = true;
3315                                         return consume_identifier (c);
3316                                 }
3317
3318                                 if (char.IsWhiteSpace ((char) c))
3319                                         continue;
3320
3321                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3322                         }
3323
3324                         if (CompleteOnEOF){
3325                                 if (generated)
3326                                         return Token.COMPLETE_COMPLETION;
3327
3328                                 generated = true;
3329                                 return Token.GENERATE_COMPLETION;
3330                         }
3331
3332
3333                         return Token.EOF;
3334                 }
3335
3336                 int TokenizeBackslash ()
3337                 {
3338                         int c = get_char ();
3339                         tokens_seen = true;
3340                         if (c == '\'') {
3341                                 val = new CharLiteral (context.BuiltinTypes, (char) c, Location);
3342                                 Report.Error (1011, Location, "Empty character literal");
3343                                 return Token.LITERAL;
3344                         }
3345
3346                         if (c == '\n') {
3347                                 Report.Error (1010, Location, "Newline in constant");
3348                                 return Token.ERROR;
3349                         }
3350
3351                         int d;
3352                         c = escape (c, out d);
3353                         if (c == -1)
3354                                 return Token.ERROR;
3355                         if (d != 0)
3356                                 throw new NotImplementedException ();
3357
3358                         val = new CharLiteral (context.BuiltinTypes, (char) c, Location);
3359                         c = get_char ();
3360
3361                         if (c != '\'') {
3362                                 Report.Error (1012, Location, "Too many characters in character literal");
3363
3364                                 // Try to recover, read until newline or next "'"
3365                                 while ((c = get_char ()) != -1) {
3366                                         if (c == '\n' || c == '\'')
3367                                                 break;
3368                                 }
3369                         }
3370
3371                         return Token.LITERAL;
3372                 }
3373
3374                 int TokenizeLessThan ()
3375                 {
3376                         int d;
3377                         if (handle_typeof) {
3378                                 PushPosition ();
3379                                 if (parse_generic_dimension (out d)) {
3380                                         val = d;
3381                                         DiscardPosition ();
3382                                         return Token.GENERIC_DIMENSION;
3383                                 }
3384                                 PopPosition ();
3385                         }
3386
3387                         // Save current position and parse next token.
3388                         PushPosition ();
3389                         if (parse_less_than ()) {
3390                                 if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) {
3391                                         d = Token.OP_GENERICS_LT_DECL;
3392                                 } else {
3393                                         d = Token.OP_GENERICS_LT;
3394                                 }
3395                                 PopPosition ();
3396                                 return d;
3397                         }
3398
3399                         PopPosition ();
3400                         parsing_generic_less_than = 0;
3401
3402                         d = peek_char ();
3403                         if (d == '<') {
3404                                 get_char ();
3405                                 d = peek_char ();
3406
3407                                 if (d == '=') {
3408                                         get_char ();
3409                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3410                                 }
3411                                 return Token.OP_SHIFT_LEFT;
3412                         }
3413
3414                         if (d == '=') {
3415                                 get_char ();
3416                                 return Token.OP_LE;
3417                         }
3418                         return Token.OP_LT;
3419                 }
3420
3421                 //
3422                 // Handles one line xml comment
3423                 //
3424                 private void handle_one_line_xml_comment ()
3425                 {
3426                         int c;
3427                         while ((c = peek_char ()) == ' ')
3428                                 get_char (); // skip heading whitespaces.
3429                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3430                                 xml_comment_buffer.Append ((char) get_char ());
3431                         }
3432                         if (c == '\r' || c == '\n')
3433                                 xml_comment_buffer.Append (Environment.NewLine);
3434                 }
3435
3436                 //
3437                 // Remove heading "*" in Javadoc-like xml documentation.
3438                 //
3439                 private void update_formatted_doc_comment (int current_comment_start)
3440                 {
3441                         int length = xml_comment_buffer.Length - current_comment_start;
3442                         string [] lines = xml_comment_buffer.ToString (
3443                                 current_comment_start,
3444                                 length).Replace ("\r", "").Split ('\n');
3445
3446                         // The first line starts with /**, thus it is not target
3447                         // for the format check.
3448                         for (int i = 1; i < lines.Length; i++) {
3449                                 string s = lines [i];
3450                                 int idx = s.IndexOf ('*');
3451                                 string head = null;
3452                                 if (idx < 0) {
3453                                         if (i < lines.Length - 1)
3454                                                 return;
3455                                         head = s;
3456                                 } else
3457                                         head = s.Substring (0, idx);
3458                                 foreach (char c in head)
3459                                         if (c != ' ')
3460                                                 return;
3461                                 lines [i] = s.Substring (idx + 1);
3462                         }
3463                         xml_comment_buffer.Remove (current_comment_start, length);
3464                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3465                 }
3466
3467                 //
3468                 // Checks if there was incorrect doc comments and raise
3469                 // warnings.
3470                 //
3471                 public void check_incorrect_doc_comment ()
3472                 {
3473                         if (xml_comment_buffer.Length > 0)
3474                                 WarningMisplacedComment (Location);
3475                 }
3476
3477                 //
3478                 // Consumes the saved xml comment lines (if any)
3479                 // as for current target member or type.
3480                 //
3481                 public string consume_doc_comment ()
3482                 {
3483                         if (xml_comment_buffer.Length > 0) {
3484                                 string ret = xml_comment_buffer.ToString ();
3485                                 reset_doc_comment ();
3486                                 return ret;
3487                         }
3488                         return null;
3489                 }
3490
3491                 Report Report {
3492                         get { return context.Report; }
3493                 }
3494
3495                 void reset_doc_comment ()
3496                 {
3497                         xml_comment_buffer.Length = 0;
3498                 }
3499
3500                 public void cleanup ()
3501                 {
3502                         if (ifstack != null && ifstack.Count >= 1) {
3503                                 int state = ifstack.Pop ();
3504                                 if ((state & REGION) != 0)
3505                                         Report.Error (1038, Location, "#endregion directive expected");
3506                                 else
3507                                         Report.Error (1027, Location, "Expected `#endif' directive");
3508                         }
3509                 }
3510         }
3511
3512         //
3513         // Indicates whether it accepts XML documentation or not.
3514         //
3515         public enum XmlCommentState {
3516                 // comment is allowed in this state.
3517                 Allowed,
3518                 // comment is not allowed in this state.
3519                 NotAllowed,
3520                 // once comments appeared when it is NotAllowed, then the
3521                 // state is changed to it, until the state is changed to
3522                 // .Allowed.
3523                 Error
3524         }
3525 }
3526