mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 //
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections;
  18 using System.IO;
  19 using System.Globalization;
  20 using System.Reflection;
  21
  22 namespace Mono.CSharp
  23 {
  24         /// <summary>
  25         ///    Tokenizer for C# source code.
  26         /// </summary>
  27
  28         public class Tokenizer : yyParser.yyInput
  29         {
  30                 class KeywordEntry
  31                 {
  32                         public readonly int Token;
  33                         public KeywordEntry Next;
  34                         public readonly char[] Value;
  35
  36                         public KeywordEntry (string value, int token)
  37                         {
  38                                 this.Value = value.ToCharArray ();
  39                                 this.Token = token;
  40                         }
  41                 }
  42
  43                 SeekableStreamReader reader;
  44                 SourceFile ref_name;
  45                 CompilationUnit file_name;
  46                 bool hidden = false;
  47                 int ref_line = 1;
  48                 int line = 1;
  49                 int col = 0;
  50                 int previous_col;
  51                 int current_token;
  52                 bool handle_get_set = false;
  53                 bool handle_remove_add = false;
  54                 bool handle_where = false;
  55                 bool handle_typeof = false;
  56                 bool lambda_arguments_parsing;
  57                 Location current_comment_location = Location.Null;
  58                 ArrayList escaped_identifiers;
  59                 int parsing_generic_less_than;
  60
  61                 //
  62                 // Used mainly for parser optimizations. Some expressions for instance
  63                 // can appear only in block (including initializer, base initializer)
  64                 // scope only
  65                 //
  66                 public int parsing_block;
  67                 internal bool query_parsing;
  68
  69                 //
  70                 // When parsing type only, useful for ambiguous nullable types
  71                 //
  72                 public int parsing_type;
  73
  74                 //
  75                 // Set when parsing generic declaration (type or method header)
  76                 //
  77                 public bool parsing_generic_declaration;
  78
  79                 //
  80                 // The value indicates that we have not reach any declaration or
  81                 // namespace yet
  82                 //
  83                 public int parsing_declaration;
  84
  85                 //
  86                 // The special character to inject on streams to trigger the EXPRESSION_PARSE
  87                 // token to be returned.   It just happens to be a Unicode character that
  88                 // would never be part of a program (can not be an identifier).
  89                 //
  90                 // This character is only tested just before the tokenizer is about to report
  91                 // an error;   So on the regular operation mode, this addition will have no
  92                 // impact on the tokenizer's performance.
  93                 //
  94
  95                 public const int EvalStatementParserCharacter = 0x2190;   // Unicode Left Arrow
  96                 public const int EvalCompilationUnitParserCharacter = 0x2191;  // Unicode Arrow
  97                 public const int EvalUsingDeclarationsParserCharacter = 0x2192;  // Unicode Arrow
  98
  99                 //
 100                 // XML documentation buffer. The save point is used to divide
 101                 // comments on types and comments on members.
 102                 //
 103                 StringBuilder xml_comment_buffer;
 104
 105                 //
 106                 // See comment on XmlCommentState enumeration.
 107                 //
 108                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 109
 110                 //
 111                 // Whether tokens have been seen on this line
 112                 //
 113                 bool tokens_seen = false;
 114
 115                 //
 116                 // Set to true once the GENERATE_COMPLETION token has bee
 117                 // returned.   This helps produce one GENERATE_COMPLETION,
 118                 // as many COMPLETE_COMPLETION as necessary to complete the
 119                 // AST tree and one final EOF.
 120                 //
 121                 bool generated;
 122
 123                 //
 124                 // Whether a token has been seen on the file
 125                 // This is needed because `define' is not allowed to be used
 126                 // after a token has been seen.
 127                 //
 128                 bool any_token_seen = false;
 129
 130                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 131
 132                 public bool PropertyParsing {
 133                         get { return handle_get_set; }
 134                         set { handle_get_set = value; }
 135                 }
 136
 137                 public bool EventParsing {
 138                         get { return handle_remove_add; }
 139                         set { handle_remove_add = value; }
 140                 }
 141
 142                 public bool ConstraintsParsing {
 143                         get { return handle_where; }
 144                         set { handle_where = value; }
 145                 }
 146
 147                 public bool TypeOfParsing {
 148                         get { return handle_typeof; }
 149                         set { handle_typeof = value; }
 150                 }
 151
 152                 public XmlCommentState doc_state {
 153                         get { return xml_doc_state; }
 154                         set {
 155                                 if (value == XmlCommentState.Allowed) {
 156                                         check_incorrect_doc_comment ();
 157                                         reset_doc_comment ();
 158                                 }
 159                                 xml_doc_state = value;
 160                         }
 161                 }
 162
 163                 //
 164                 // This is used to trigger completion generation on the parser
 165                 public bool CompleteOnEOF;
 166
 167                 void AddEscapedIdentifier (LocatedToken lt)
 168                 {
 169                         if (escaped_identifiers == null)
 170                                 escaped_identifiers = new ArrayList ();
 171
 172                         escaped_identifiers.Add (lt);
 173                 }
 174
 175                 public bool IsEscapedIdentifier (Location loc)
 176                 {
 177                         if (escaped_identifiers != null) {
 178                                 foreach (LocatedToken lt in escaped_identifiers)
 179                                         if (lt.Location.Equals (loc))
 180                                                 return true;
 181                         }
 182
 183                         return false;
 184                 }
 185
 186                 //
 187                 // Class variables
 188                 //
 189                 static KeywordEntry[][] keywords;
 190                 static Hashtable keyword_strings;
 191                 static NumberStyles styles;
 192                 static NumberFormatInfo csharp_format_info;
 193
 194                 //
 195                 // Values for the associated token returned
 196                 //
 197                 internal int putback_char;      // Used by repl only
 198                 Object val;
 199
 200                 //
 201                 // Pre-processor
 202                 //
 203                 const int TAKING        = 1;
 204                 const int ELSE_SEEN     = 4;
 205                 const int PARENT_TAKING = 8;
 206                 const int REGION        = 16;
 207
 208                 //
 209                 // pre-processor if stack state:
 210                 //
 211                 Stack ifstack;
 212
 213                 static System.Text.StringBuilder string_builder;
 214
 215                 const int max_id_size = 512;
 216                 static char [] id_builder = new char [max_id_size];
 217
 218                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 219
 220                 const int max_number_size = 512;
 221                 static char [] number_builder = new char [max_number_size];
 222                 static int number_pos;
 223
 224                 //
 225                 // Details about the error encoutered by the tokenizer
 226                 //
 227                 string error_details;
 228
 229                 public string error {
 230                         get {
 231                                 return error_details;
 232                         }
 233                 }
 234
 235                 public int Line {
 236                         get {
 237                                 return ref_line;
 238                         }
 239                 }
 240
 241                 //
 242                 // This is used when the tokenizer needs to save
 243                 // the current position as it needs to do some parsing
 244                 // on its own to deamiguate a token in behalf of the
 245                 // parser.
 246                 //
 247                 Stack position_stack = new Stack (2);
 248                 class Position {
 249                         public int position;
 250                         public int line;
 251                         public int ref_line;
 252                         public int col;
 253                         public bool hidden;
 254                         public int putback_char;
 255                         public int previous_col;
 256                         public Stack ifstack;
 257                         public int parsing_generic_less_than;
 258                         public int current_token;
 259
 260                         public Position (Tokenizer t)
 261                         {
 262                                 position = t.reader.Position;
 263                                 line = t.line;
 264                                 ref_line = t.ref_line;
 265                                 col = t.col;
 266                                 hidden = t.hidden;
 267                                 putback_char = t.putback_char;
 268                                 previous_col = t.previous_col;
 269                                 if (t.ifstack != null && t.ifstack.Count != 0)
 270                                         ifstack = (Stack)t.ifstack.Clone ();
 271                                 parsing_generic_less_than = t.parsing_generic_less_than;
 272                                 current_token = t.current_token;
 273                         }
 274                 }
 275
 276                 public void PushPosition ()
 277                 {
 278                         position_stack.Push (new Position (this));
 279                 }
 280
 281                 public void PopPosition ()
 282                 {
 283                         Position p = (Position) position_stack.Pop ();
 284
 285                         reader.Position = p.position;
 286                         ref_line = p.ref_line;
 287                         line = p.line;
 288                         col = p.col;
 289                         hidden = p.hidden;
 290                         putback_char = p.putback_char;
 291                         previous_col = p.previous_col;
 292                         ifstack = p.ifstack;
 293                         parsing_generic_less_than = p.parsing_generic_less_than;
 294                         current_token = p.current_token;
 295                 }
 296
 297                 // Do not reset the position, ignore it.
 298                 public void DiscardPosition ()
 299                 {
 300                         position_stack.Pop ();
 301                 }
 302
 303                 static void AddKeyword (string kw, int token)
 304                 {
 305                         keyword_strings.Add (kw, kw);
 306
 307                         int length = kw.Length;
 308                         if (keywords [length] == null) {
 309                                 keywords [length] = new KeywordEntry ['z' - '_' + 1];
 310                         }
 311
 312                         int char_index = kw [0] - '_';
 313                         KeywordEntry kwe = keywords [length] [char_index];
 314                         if (kwe == null) {
 315                                 keywords [length] [char_index] = new KeywordEntry (kw, token);
 316                                 return;
 317                         }
 318
 319                         while (kwe.Next != null) {
 320                                 kwe = kwe.Next;
 321                         }
 322
 323                         kwe.Next = new KeywordEntry (kw, token);
 324                 }
 325
 326                 static void InitTokens ()
 327                 {
 328                         keyword_strings = new Hashtable ();
 329
 330                         // 11 is the length of the longest keyword for now
 331                         keywords = new KeywordEntry [11] [];
 332
 333                         AddKeyword ("__arglist", Token.ARGLIST);
 334                         AddKeyword ("abstract", Token.ABSTRACT);
 335                         AddKeyword ("as", Token.AS);
 336                         AddKeyword ("add", Token.ADD);
 337                         AddKeyword ("base", Token.BASE);
 338                         AddKeyword ("bool", Token.BOOL);
 339                         AddKeyword ("break", Token.BREAK);
 340                         AddKeyword ("byte", Token.BYTE);
 341                         AddKeyword ("case", Token.CASE);
 342                         AddKeyword ("catch", Token.CATCH);
 343                         AddKeyword ("char", Token.CHAR);
 344                         AddKeyword ("checked", Token.CHECKED);
 345                         AddKeyword ("class", Token.CLASS);
 346                         AddKeyword ("const", Token.CONST);
 347                         AddKeyword ("continue", Token.CONTINUE);
 348                         AddKeyword ("decimal", Token.DECIMAL);
 349                         AddKeyword ("default", Token.DEFAULT);
 350                         AddKeyword ("delegate", Token.DELEGATE);
 351                         AddKeyword ("do", Token.DO);
 352                         AddKeyword ("double", Token.DOUBLE);
 353                         AddKeyword ("else", Token.ELSE);
 354                         AddKeyword ("enum", Token.ENUM);
 355                         AddKeyword ("event", Token.EVENT);
 356                         AddKeyword ("explicit", Token.EXPLICIT);
 357                         AddKeyword ("extern", Token.EXTERN);
 358                         AddKeyword ("false", Token.FALSE);
 359                         AddKeyword ("finally", Token.FINALLY);
 360                         AddKeyword ("fixed", Token.FIXED);
 361                         AddKeyword ("float", Token.FLOAT);
 362                         AddKeyword ("for", Token.FOR);
 363                         AddKeyword ("foreach", Token.FOREACH);
 364                         AddKeyword ("goto", Token.GOTO);
 365                         AddKeyword ("get", Token.GET);
 366                         AddKeyword ("if", Token.IF);
 367                         AddKeyword ("implicit", Token.IMPLICIT);
 368                         AddKeyword ("in", Token.IN);
 369                         AddKeyword ("int", Token.INT);
 370                         AddKeyword ("interface", Token.INTERFACE);
 371                         AddKeyword ("internal", Token.INTERNAL);
 372                         AddKeyword ("is", Token.IS);
 373                         AddKeyword ("lock", Token.LOCK);
 374                         AddKeyword ("long", Token.LONG);
 375                         AddKeyword ("namespace", Token.NAMESPACE);
 376                         AddKeyword ("new", Token.NEW);
 377                         AddKeyword ("null", Token.NULL);
 378                         AddKeyword ("object", Token.OBJECT);
 379                         AddKeyword ("operator", Token.OPERATOR);
 380                         AddKeyword ("out", Token.OUT);
 381                         AddKeyword ("override", Token.OVERRIDE);
 382                         AddKeyword ("params", Token.PARAMS);
 383                         AddKeyword ("private", Token.PRIVATE);
 384                         AddKeyword ("protected", Token.PROTECTED);
 385                         AddKeyword ("public", Token.PUBLIC);
 386                         AddKeyword ("readonly", Token.READONLY);
 387                         AddKeyword ("ref", Token.REF);
 388                         AddKeyword ("remove", Token.REMOVE);
 389                         AddKeyword ("return", Token.RETURN);
 390                         AddKeyword ("sbyte", Token.SBYTE);
 391                         AddKeyword ("sealed", Token.SEALED);
 392                         AddKeyword ("set", Token.SET);
 393                         AddKeyword ("short", Token.SHORT);
 394                         AddKeyword ("sizeof", Token.SIZEOF);
 395                         AddKeyword ("stackalloc", Token.STACKALLOC);
 396                         AddKeyword ("static", Token.STATIC);
 397                         AddKeyword ("string", Token.STRING);
 398                         AddKeyword ("struct", Token.STRUCT);
 399                         AddKeyword ("switch", Token.SWITCH);
 400                         AddKeyword ("this", Token.THIS);
 401                         AddKeyword ("throw", Token.THROW);
 402                         AddKeyword ("true", Token.TRUE);
 403                         AddKeyword ("try", Token.TRY);
 404                         AddKeyword ("typeof", Token.TYPEOF);
 405                         AddKeyword ("uint", Token.UINT);
 406                         AddKeyword ("ulong", Token.ULONG);
 407                         AddKeyword ("unchecked", Token.UNCHECKED);
 408                         AddKeyword ("unsafe", Token.UNSAFE);
 409                         AddKeyword ("ushort", Token.USHORT);
 410                         AddKeyword ("using", Token.USING);
 411                         AddKeyword ("virtual", Token.VIRTUAL);
 412                         AddKeyword ("void", Token.VOID);
 413                         AddKeyword ("volatile", Token.VOLATILE);
 414                         AddKeyword ("while", Token.WHILE);
 415                         AddKeyword ("partial", Token.PARTIAL);
 416                         AddKeyword ("where", Token.WHERE);
 417
 418                         // LINQ keywords
 419                         AddKeyword ("from", Token.FROM);
 420                         AddKeyword ("join", Token.JOIN);
 421                         AddKeyword ("on", Token.ON);
 422                         AddKeyword ("equals", Token.EQUALS);
 423                         AddKeyword ("select", Token.SELECT);
 424                         AddKeyword ("group", Token.GROUP);
 425                         AddKeyword ("by", Token.BY);
 426                         AddKeyword ("let", Token.LET);
 427                         AddKeyword ("orderby", Token.ORDERBY);
 428                         AddKeyword ("ascending", Token.ASCENDING);
 429                         AddKeyword ("descending", Token.DESCENDING);
 430                         AddKeyword ("into", Token.INTO);
 431                 }
 432
 433                 //
 434                 // Class initializer
 435                 //
 436                 static Tokenizer ()
 437                 {
 438                         InitTokens ();
 439                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 440                         styles = NumberStyles.Float;
 441
 442                         string_builder = new System.Text.StringBuilder ();
 443                 }
 444
 445                 int GetKeyword (char[] id, int id_len)
 446                 {
 447                         //
 448                         // Keywords are stored in an array of arrays grouped by their
 449                         // length and then by the first character
 450                         //
 451                         if (id_len >= keywords.Length || keywords [id_len] == null)
 452                                 return -1;
 453
 454                         int first_index = id [0] - '_';
 455                         if (first_index > 'z')
 456                                 return -1;
 457
 458                         KeywordEntry kwe = keywords [id_len] [first_index];
 459                         if (kwe == null)
 460                                 return -1;
 461
 462                         int res;
 463                         do {
 464                                 res = kwe.Token;
 465                                 for (int i = 1; i < id_len; ++i) {
 466                                         if (id [i] != kwe.Value [i]) {
 467                                                 res = 0;
 468                                                 break;
 469                                         }
 470                                 }
 471                                 kwe = kwe.Next;
 472                         } while (kwe != null && res == 0);
 473
 474                         if (res == 0)
 475                                 return -1;
 476
 477                         int next_token;
 478                         switch (res) {
 479                         case Token.GET:
 480                         case Token.SET:
 481                                 if (!handle_get_set)
 482                                         res = -1;
 483                                 break;
 484                         case Token.REMOVE:
 485                         case Token.ADD:
 486                                 if (!handle_remove_add)
 487                                         res = -1;
 488                                 break;
 489                         case Token.EXTERN:
 490                                 if (parsing_declaration == 0)
 491                                         res = Token.EXTERN_ALIAS;
 492                                 break;
 493                         case Token.DEFAULT:
 494                                 if (peek_token () == Token.COLON) {
 495                                         token ();
 496                                         res = Token.DEFAULT_COLON;
 497                                 }
 498                                 break;
 499                         case Token.WHERE:
 500                                 if (!handle_where && !query_parsing)
 501                                         res = -1;
 502                                 break;
 503                         case Token.FROM:
 504                                 //
 505                                 // A query expression is any expression that starts with `from identifier'
 506                                 // followed by any token except ; , =
 507                                 //
 508                                 if (!query_parsing) {
 509                                         if (lambda_arguments_parsing) {
 510                                                 res = -1;
 511                                                 break;
 512                                         }
 513
 514                                         PushPosition ();
 515                                         // HACK: to disable generics micro-parser, because PushPosition does not
 516                                         // store identifiers array
 517                                         parsing_generic_less_than = 1;
 518                                         switch (xtoken ()) {
 519                                         case Token.IDENTIFIER:
 520                                         case Token.INT:
 521                                         case Token.BOOL:
 522                                         case Token.BYTE:
 523                                         case Token.CHAR:
 524                                         case Token.DECIMAL:
 525                                         case Token.FLOAT:
 526                                         case Token.LONG:
 527                                         case Token.OBJECT:
 528                                         case Token.STRING:
 529                                         case Token.UINT:
 530                                         case Token.ULONG:
 531                                                 next_token = xtoken ();
 532                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 533                                                         goto default;
 534
 535                                                 res = Token.FROM_FIRST;
 536                                                 query_parsing = true;
 537                                                 if (RootContext.Version <= LanguageVersion.ISO_2)
 538                                                         Report.FeatureIsNotAvailable (Location, "query expressions");
 539                                                 break;
 540                                         case Token.VOID:
 541                                                 Expression.Error_VoidInvalidInTheContext (Location);
 542                                                 break;
 543                                         default:
 544                                                 PopPosition ();
 545                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 546                                                 // which has been overwritten before we grabbed the identifier
 547                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 548                                                 return -1;
 549                                         }
 550                                         PopPosition ();
 551                                 }
 552                                 break;
 553                         case Token.JOIN:
 554                         case Token.ON:
 555                         case Token.EQUALS:
 556                         case Token.SELECT:
 557                         case Token.GROUP:
 558                         case Token.BY:
 559                         case Token.LET:
 560                         case Token.ORDERBY:
 561                         case Token.ASCENDING:
 562                         case Token.DESCENDING:
 563                         case Token.INTO:
 564                                 if (!query_parsing)
 565                                         res = -1;
 566                                 break;
 567
 568                         case Token.USING:
 569                         case Token.NAMESPACE:
 570                                 // TODO: some explanation needed
 571                                 check_incorrect_doc_comment ();
 572                                 break;
 573
 574                         case Token.PARTIAL:
 575                                 if (parsing_block > 0) {
 576                                         res = -1;
 577                                         break;
 578                                 }
 579
 580                                 // Save current position and parse next token.
 581                                 PushPosition ();
 582
 583                                 next_token = token ();
 584                                 bool ok = (next_token == Token.CLASS) ||
 585                                         (next_token == Token.STRUCT) ||
 586                                         (next_token == Token.INTERFACE) ||
 587                                         (next_token == Token.VOID);
 588
 589                                 PopPosition ();
 590
 591                                 if (ok) {
 592                                         if (next_token == Token.VOID) {
 593                                                 if (RootContext.Version == LanguageVersion.ISO_1 ||
 594                                                     RootContext.Version == LanguageVersion.ISO_2)
 595                                                         Report.FeatureIsNotAvailable (Location, "partial methods");
 596                                         } else if (RootContext.Version == LanguageVersion.ISO_1)
 597                                                 Report.FeatureIsNotAvailable (Location, "partial types");
 598
 599                                         return res;
 600                                 }
 601
 602                                 if (next_token < Token.LAST_KEYWORD) {
 603                                         Report.Error (267, Location,
 604                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 605                                         return token ();
 606                                 }
 607
 608                                 res = -1;
 609                                 break;
 610                         }
 611
 612                         return res;
 613                 }
 614
 615                 public Location Location {
 616                         get {
 617                                 return new Location (ref_line, hidden ? -1 : col);
 618                         }
 619                 }
 620
 621                 public Tokenizer (SeekableStreamReader input, CompilationUnit file)
 622                 {
 623                         this.ref_name = file;
 624                         this.file_name = file;
 625                         reader = input;
 626
 627                         putback_char = -1;
 628
 629                         xml_comment_buffer = new StringBuilder ();
 630
 631                         //
 632                         // FIXME: This could be `Location.Push' but we have to
 633                         // find out why the MS compiler allows this
 634                         //
 635                         Mono.CSharp.Location.Push (file, file);
 636                 }
 637
 638                 static bool is_identifier_start_character (int c)
 639                 {
 640                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 641                 }
 642
 643                 static bool is_identifier_part_character (char c)
 644                 {
 645                         if (c >= 'a' && c <= 'z')
 646                                 return true;
 647
 648                         if (c >= 'A' && c <= 'Z')
 649                                 return true;
 650
 651                         if (c == '_' || (c >= '0' && c <= '9'))
 652                                 return true;
 653
 654                         if (c < 0x80)
 655                                 return false;
 656
 657                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 658                 }
 659
 660                 public static bool IsKeyword (string s)
 661                 {
 662                         return keyword_strings [s] != null;
 663                 }
 664
 665                 //
 666                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 667                 //
 668
 669                 int TokenizeOpenParens ()
 670                 {
 671                         int ptoken;
 672                         current_token = -1;
 673
 674                         int bracket_level = 0;
 675                         bool is_type = false;
 676                         bool can_be_type = false;
 677
 678                         while (true) {
 679                                 ptoken = current_token;
 680                                 token ();
 681
 682                                 switch (current_token) {
 683                                 case Token.CLOSE_PARENS:
 684                                         token ();
 685
 686                                         //
 687                                         // Expression inside parens is lambda, (int i) =>
 688                                         //
 689                                         if (current_token == Token.ARROW) {
 690                                                 if (RootContext.Version <= LanguageVersion.ISO_2)
 691                                                         Report.FeatureIsNotAvailable (Location, "lambda expressions");
 692
 693                                                 return Token.OPEN_PARENS_LAMBDA;
 694                                         }
 695
 696                                         //
 697                                         // Expression inside parens is single type, (int[])
 698                                         //
 699                                         if (is_type)
 700                                                 return Token.OPEN_PARENS_CAST;
 701
 702                                         //
 703                                         // Expression is possible cast, look at next token, (T)null
 704                                         //
 705                                         if (can_be_type) {
 706                                                 switch (current_token) {
 707                                                 case Token.OPEN_PARENS:
 708                                                 case Token.BANG:
 709                                                 case Token.TILDE:
 710                                                 case Token.IDENTIFIER:
 711                                                 case Token.LITERAL_INTEGER:
 712                                                 case Token.LITERAL_FLOAT:
 713                                                 case Token.LITERAL_DOUBLE:
 714                                                 case Token.LITERAL_DECIMAL:
 715                                                 case Token.LITERAL_CHARACTER:
 716                                                 case Token.LITERAL_STRING:
 717                                                 case Token.BASE:
 718                                                 case Token.CHECKED:
 719                                                 case Token.DELEGATE:
 720                                                 case Token.FALSE:
 721                                                 case Token.FIXED:
 722                                                 case Token.NEW:
 723                                                 case Token.NULL:
 724                                                 case Token.SIZEOF:
 725                                                 case Token.THIS:
 726                                                 case Token.THROW:
 727                                                 case Token.TRUE:
 728                                                 case Token.TYPEOF:
 729                                                 case Token.UNCHECKED:
 730                                                 case Token.UNSAFE:
 731                                                 case Token.DEFAULT:
 732
 733                                                 //
 734                                                 // These can be part of a member access
 735                                                 //
 736                                                 case Token.INT:
 737                                                 case Token.UINT:
 738                                                 case Token.SHORT:
 739                                                 case Token.USHORT:
 740                                                 case Token.LONG:
 741                                                 case Token.ULONG:
 742                                                 case Token.DOUBLE:
 743                                                 case Token.FLOAT:
 744                                                 case Token.CHAR:
 745                                                 case Token.BYTE:
 746                                                 case Token.DECIMAL:
 747                                                 case Token.BOOL:
 748                                                         return Token.OPEN_PARENS_CAST;
 749                                                 }
 750                                         }
 751                                         return Token.OPEN_PARENS;
 752
 753                                 case Token.DOT:
 754                                 case Token.DOUBLE_COLON:
 755                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
 756                                                 goto default;
 757
 758                                         continue;
 759
 760                                 case Token.IDENTIFIER:
 761                                         switch (ptoken) {
 762                                         case Token.DOT:
 763                                         case Token.OP_GENERICS_LT:
 764                                         case Token.COMMA:
 765                                         case Token.DOUBLE_COLON:
 766                                         case -1:
 767                                                 if (bracket_level == 0)
 768                                                         can_be_type = true;
 769                                                 continue;
 770                                         default:
 771                                                 can_be_type = is_type = false;
 772                                                 continue;
 773                                         }
 774
 775                                 case Token.OBJECT:
 776                                 case Token.STRING:
 777                                 case Token.BOOL:
 778                                 case Token.DECIMAL:
 779                                 case Token.FLOAT:
 780                                 case Token.DOUBLE:
 781                                 case Token.SBYTE:
 782                                 case Token.BYTE:
 783                                 case Token.SHORT:
 784                                 case Token.USHORT:
 785                                 case Token.INT:
 786                                 case Token.UINT:
 787                                 case Token.LONG:
 788                                 case Token.ULONG:
 789                                 case Token.CHAR:
 790                                 case Token.VOID:
 791                                         if (bracket_level == 0)
 792                                                 is_type = true;
 793                                         continue;
 794
 795                                 case Token.COMMA:
 796                                         if (bracket_level == 0) {
 797                                                 bracket_level = 100;
 798                                                 can_be_type = is_type = false;
 799                                         }
 800                                         continue;
 801
 802                                 case Token.OP_GENERICS_LT:
 803                                 case Token.OPEN_BRACKET:
 804                                         if (bracket_level++ == 0)
 805                                                 is_type = true;
 806                                         continue;
 807
 808                                 case Token.OP_GENERICS_GT:
 809                                 case Token.CLOSE_BRACKET:
 810                                         --bracket_level;
 811                                         continue;
 812
 813                                 case Token.INTERR_NULLABLE:
 814                                 case Token.STAR:
 815                                         if (bracket_level == 0)
 816                                                 is_type = true;
 817                                         continue;
 818
 819                                 case Token.REF:
 820                                 case Token.OUT:
 821                                         can_be_type = is_type = false;
 822                                         continue;
 823
 824                                 default:
 825                                         return Token.OPEN_PARENS;
 826                                 }
 827                         }
 828                 }
 829
 830                 public static bool IsValidIdentifier (string s)
 831                 {
 832                         if (s == null || s.Length == 0)
 833                                 return false;
 834
 835                         if (!is_identifier_start_character (s [0]))
 836                                 return false;
 837
 838                         for (int i = 1; i < s.Length; i ++)
 839                                 if (! is_identifier_part_character (s [i]))
 840                                         return false;
 841
 842                         return true;
 843                 }
 844
 845                 bool parse_less_than ()
 846                 {
 847                 start:
 848                         int the_token = token ();
 849                         if (the_token == Token.OPEN_BRACKET) {
 850                                 do {
 851                                         the_token = token ();
 852                                 } while (the_token != Token.CLOSE_BRACKET);
 853                                 the_token = token ();
 854                         } else if (the_token == Token.IN || the_token == Token.OUT) {
 855                                 the_token = token ();
 856                         }
 857                         switch (the_token) {
 858                         case Token.IDENTIFIER:
 859                         case Token.OBJECT:
 860                         case Token.STRING:
 861                         case Token.BOOL:
 862                         case Token.DECIMAL:
 863                         case Token.FLOAT:
 864                         case Token.DOUBLE:
 865                         case Token.SBYTE:
 866                         case Token.BYTE:
 867                         case Token.SHORT:
 868                         case Token.USHORT:
 869                         case Token.INT:
 870                         case Token.UINT:
 871                         case Token.LONG:
 872                         case Token.ULONG:
 873                         case Token.CHAR:
 874                         case Token.VOID:
 875                                 break;
 876                         case Token.OP_GENERICS_GT:
 877                                 return true;
 878
 879                         default:
 880                                 return false;
 881                         }
 882                 again:
 883                         the_token = token ();
 884
 885                         if (the_token == Token.OP_GENERICS_GT)
 886                                 return true;
 887                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
 888                                 goto start;
 889                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
 890                                 goto again;
 891                         else if (the_token == Token.OP_GENERICS_LT) {
 892                                 if (!parse_less_than ())
 893                                         return false;
 894                                 goto again;
 895                         } else if (the_token == Token.OPEN_BRACKET) {
 896                         rank_specifiers:
 897                                 the_token = token ();
 898                                 if (the_token == Token.CLOSE_BRACKET)
 899                                         goto again;
 900                                 else if (the_token == Token.COMMA)
 901                                         goto rank_specifiers;
 902                                 return false;
 903                         }
 904
 905                         return false;
 906                 }
 907
 908                 bool parse_generic_dimension (out int dimension)
 909                 {
 910                         dimension = 1;
 911
 912                 again:
 913                         int the_token = token ();
 914                         if (the_token == Token.OP_GENERICS_GT)
 915                                 return true;
 916                         else if (the_token == Token.COMMA) {
 917                                 dimension++;
 918                                 goto again;
 919                         }
 920
 921                         return false;
 922                 }
 923
 924                 public int peek_token ()
 925                 {
 926                         int the_token;
 927
 928                         PushPosition ();
 929                         the_token = token ();
 930                         PopPosition ();
 931
 932                         return the_token;
 933                 }
 934
 935                 //
 936                 // Tonizes `?' using custom disambiguous rules to return one
 937                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
 938                 //
 939                 // Tricky expression look like:
 940                 //
 941                 // Foo ? a = x ? b : c;
 942                 //
 943                 int TokenizePossibleNullableType ()
 944                 {
 945                         if (parsing_block == 0 || parsing_type > 0)
 946                                 return Token.INTERR_NULLABLE;
 947
 948                         int d = peek_char ();
 949                         if (d == '?') {
 950                                 get_char ();
 951                                 return Token.OP_COALESCING;
 952                         }
 953
 954                         switch (current_token) {
 955                         case Token.CLOSE_PARENS:
 956                         case Token.TRUE:
 957                         case Token.FALSE:
 958                         case Token.NULL:
 959                         case Token.LITERAL_INTEGER:
 960                         case Token.LITERAL_STRING:
 961                                 return Token.INTERR;
 962                         }
 963
 964                         if (d != ' ') {
 965                                 if (d == ',' || d == ';' || d == '>')
 966                                         return Token.INTERR_NULLABLE;
 967                                 if (d == '*' || (d >= '0' && d <= '9'))
 968                                         return Token.INTERR;
 969                         }
 970
 971                         PushPosition ();
 972                         current_token = Token.NONE;
 973                         int next_token;
 974                         switch (xtoken ()) {
 975                         case Token.LITERAL_INTEGER:
 976                         case Token.LITERAL_STRING:
 977                         case Token.LITERAL_CHARACTER:
 978                         case Token.LITERAL_DECIMAL:
 979                         case Token.LITERAL_DOUBLE:
 980                         case Token.LITERAL_FLOAT:
 981                         case Token.TRUE:
 982                         case Token.FALSE:
 983                         case Token.NULL:
 984                         case Token.THIS:
 985                         case Token.NEW:
 986                                 next_token = Token.INTERR;
 987                                 break;
 988
 989                         case Token.SEMICOLON:
 990                         case Token.COMMA:
 991                         case Token.CLOSE_PARENS:
 992                         case Token.OPEN_BRACKET:
 993                         case Token.OP_GENERICS_GT:
 994                                 next_token = Token.INTERR_NULLABLE;
 995                                 break;
 996
 997                         default:
 998                                 next_token = -1;
 999                                 break;
1000                         }
1001
1002                         if (next_token == -1) {
1003                                 switch (xtoken ()) {
1004                                 case Token.COMMA:
1005                                 case Token.SEMICOLON:
1006                                 case Token.OPEN_BRACE:
1007                                 case Token.CLOSE_PARENS:
1008                                 case Token.IN:
1009                                         next_token = Token.INTERR_NULLABLE;
1010                                         break;
1011
1012                                 case Token.COLON:
1013                                         next_token = Token.INTERR;
1014                                         break;
1015
1016                                 default:
1017                                         int ntoken;
1018                                         int interrs = 1;
1019                                         int colons = 0;
1020                                         //
1021                                         // All shorcuts failed, do it hard way
1022                                         //
1023                                         while ((ntoken = xtoken ()) != Token.EOF) {
1024                                                 if (ntoken == Token.SEMICOLON)
1025                                                         break;
1026
1027                                                 if (ntoken == Token.COLON) {
1028                                                         if (++colons == interrs)
1029                                                                 break;
1030                                                         continue;
1031                                                 }
1032
1033                                                 if (ntoken == Token.INTERR) {
1034                                                         ++interrs;
1035                                                         continue;
1036                                                 }
1037                                         }
1038
1039                                         next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1040                                         break;
1041                                 }
1042                         }
1043
1044                         PopPosition ();
1045                         return next_token;
1046                 }
1047
1048                 bool decimal_digits (int c)
1049                 {
1050                         int d;
1051                         bool seen_digits = false;
1052
1053                         if (c != -1){
1054                                 if (number_pos == max_number_size)
1055                                         Error_NumericConstantTooLong ();
1056                                 number_builder [number_pos++] = (char) c;
1057                         }
1058
1059                         //
1060                         // We use peek_char2, because decimal_digits needs to do a
1061                         // 2-character look-ahead (5.ToString for example).
1062                         //
1063                         while ((d = peek_char2 ()) != -1){
1064                                 if (d >= '0' && d <= '9'){
1065                                         if (number_pos == max_number_size)
1066                                                 Error_NumericConstantTooLong ();
1067                                         number_builder [number_pos++] = (char) d;
1068                                         get_char ();
1069                                         seen_digits = true;
1070                                 } else
1071                                         break;
1072                         }
1073
1074                         return seen_digits;
1075                 }
1076
1077                 static bool is_hex (int e)
1078                 {
1079                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1080                 }
1081
1082                 static int real_type_suffix (int c)
1083                 {
1084                         int t;
1085
1086                         switch (c){
1087                         case 'F': case 'f':
1088                                 t =  Token.LITERAL_FLOAT;
1089                                 break;
1090                         case 'D': case 'd':
1091                                 t = Token.LITERAL_DOUBLE;
1092                                 break;
1093                         case 'M': case 'm':
1094                                  t= Token.LITERAL_DECIMAL;
1095                                 break;
1096                         default:
1097                                 return Token.NONE;
1098                         }
1099                         return t;
1100                 }
1101
1102                 int integer_type_suffix (ulong ul, int c)
1103                 {
1104                         bool is_unsigned = false;
1105                         bool is_long = false;
1106
1107                         if (c != -1){
1108                                 bool scanning = true;
1109                                 do {
1110                                         switch (c){
1111                                         case 'U': case 'u':
1112                                                 if (is_unsigned)
1113                                                         scanning = false;
1114                                                 is_unsigned = true;
1115                                                 get_char ();
1116                                                 break;
1117
1118                                         case 'l':
1119                                                 if (!is_unsigned){
1120                                                         //
1121                                                         // if we have not seen anything in between
1122                                                         // report this error
1123                                                         //
1124                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1125                                                 }
1126                                                 //
1127                                                 // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
1128                                                 // work around that.
1129                                                 //
1130                                                 //goto case 'L';
1131                                                 if (is_long)
1132                                                         scanning = false;
1133                                                 is_long = true;
1134                                                 get_char ();
1135                                                 break;
1136
1137                                         case 'L':
1138                                                 if (is_long)
1139                                                         scanning = false;
1140                                                 is_long = true;
1141                                                 get_char ();
1142                                                 break;
1143
1144                                         default:
1145                                                 scanning = false;
1146                                                 break;
1147                                         }
1148                                         c = peek_char ();
1149                                 } while (scanning);
1150                         }
1151
1152                         if (is_long && is_unsigned){
1153                                 val = ul;
1154                                 return Token.LITERAL_INTEGER;
1155                         } else if (is_unsigned){
1156                                 // uint if possible, or ulong else.
1157
1158                                 if ((ul & 0xffffffff00000000) == 0)
1159                                         val = (uint) ul;
1160                                 else
1161                                         val = ul;
1162                         } else if (is_long){
1163                                 // long if possible, ulong otherwise
1164                                 if ((ul & 0x8000000000000000) != 0)
1165                                         val = ul;
1166                                 else
1167                                         val = (long) ul;
1168                         } else {
1169                                 // int, uint, long or ulong in that order
1170                                 if ((ul & 0xffffffff00000000) == 0){
1171                                         uint ui = (uint) ul;
1172
1173                                         if ((ui & 0x80000000) != 0)
1174                                                 val = ui;
1175                                         else
1176                                                 val = (int) ui;
1177                                 } else {
1178                                         if ((ul & 0x8000000000000000) != 0)
1179                                                 val = ul;
1180                                         else
1181                                                 val = (long) ul;
1182                                 }
1183                         }
1184                         return Token.LITERAL_INTEGER;
1185                 }
1186
1187                 //
1188                 // given `c' as the next char in the input decide whether
1189                 // we need to convert to a special type, and then choose
1190                 // the best representation for the integer
1191                 //
1192                 int adjust_int (int c)
1193                 {
1194                         try {
1195                                 if (number_pos > 9){
1196                                         ulong ul = (uint) (number_builder [0] - '0');
1197
1198                                         for (int i = 1; i < number_pos; i++){
1199                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1200                                         }
1201                                         return integer_type_suffix (ul, c);
1202                                 } else {
1203                                         uint ui = (uint) (number_builder [0] - '0');
1204
1205                                         for (int i = 1; i < number_pos; i++){
1206                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1207                                         }
1208                                         return integer_type_suffix (ui, c);
1209                                 }
1210                         } catch (OverflowException) {
1211                                 error_details = "Integral constant is too large";
1212                                 Report.Error (1021, Location, error_details);
1213                                 val = 0ul;
1214                                 return Token.LITERAL_INTEGER;
1215                         }
1216                         catch (FormatException) {
1217                                 Report.Error (1013, Location, "Invalid number");
1218                                 val = 0ul;
1219                                 return Token.LITERAL_INTEGER;
1220                         }
1221                 }
1222
1223                 int adjust_real (int t)
1224                 {
1225                         string s = new String (number_builder, 0, number_pos);
1226                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1227
1228                         switch (t){
1229                         case Token.LITERAL_DECIMAL:
1230                                 try {
1231                                         val = System.Decimal.Parse (s, styles, csharp_format_info);
1232                                 } catch (OverflowException) {
1233                                         val = 0m;
1234                                         Report.Error (594, Location, error_details, "decimal");
1235                                 }
1236                                 break;
1237                         case Token.LITERAL_FLOAT:
1238                                 try {
1239                                         val = float.Parse (s, styles, csharp_format_info);
1240                                 } catch (OverflowException) {
1241                                         val = 0.0f;
1242                                         Report.Error (594, Location, error_details, "float");
1243                                 }
1244                                 break;
1245
1246                         case Token.LITERAL_DOUBLE:
1247                         case Token.NONE:
1248                                 t = Token.LITERAL_DOUBLE;
1249                                 try {
1250                                         val = System.Double.Parse (s, styles, csharp_format_info);
1251                                 } catch (OverflowException) {
1252                                         val = 0.0;
1253                                         Report.Error (594, Location, error_details, "double");
1254                                 }
1255                                 break;
1256                         }
1257                         return t;
1258                 }
1259
1260                 int handle_hex ()
1261                 {
1262                         int d;
1263                         ulong ul;
1264
1265                         get_char ();
1266                         while ((d = peek_char ()) != -1){
1267                                 if (is_hex (d)){
1268                                         number_builder [number_pos++] = (char) d;
1269                                         get_char ();
1270                                 } else
1271                                         break;
1272                         }
1273
1274                         string s = new String (number_builder, 0, number_pos);
1275                         try {
1276                                 if (number_pos <= 8)
1277                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1278                                 else
1279                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1280                         } catch (OverflowException){
1281                                 error_details = "Integral constant is too large";
1282                                 Report.Error (1021, Location, error_details);
1283                                 val = 0ul;
1284                                 return Token.LITERAL_INTEGER;
1285                         }
1286                         catch (FormatException) {
1287                                 Report.Error (1013, Location, "Invalid number");
1288                                 val = 0ul;
1289                                 return Token.LITERAL_INTEGER;
1290                         }
1291
1292                         return integer_type_suffix (ul, peek_char ());
1293                 }
1294
1295                 //
1296                 // Invoked if we know we have .digits or digits
1297                 //
1298                 int is_number (int c)
1299                 {
1300                         bool is_real = false;
1301                         int type;
1302
1303                         number_pos = 0;
1304
1305                         if (c >= '0' && c <= '9'){
1306                                 if (c == '0'){
1307                                         int peek = peek_char ();
1308
1309                                         if (peek == 'x' || peek == 'X')
1310                                                 return handle_hex ();
1311                                 }
1312                                 decimal_digits (c);
1313                                 c = get_char ();
1314                         }
1315
1316                         //
1317                         // We need to handle the case of
1318                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1319                         //
1320                         if (c == '.'){
1321                                 if (decimal_digits ('.')){
1322                                         is_real = true;
1323                                         c = get_char ();
1324                                 } else {
1325                                         putback ('.');
1326                                         number_pos--;
1327                                         return adjust_int (-1);
1328                                 }
1329                         }
1330
1331                         if (c == 'e' || c == 'E'){
1332                                 is_real = true;
1333                                 if (number_pos == max_number_size)
1334                                         Error_NumericConstantTooLong ();
1335                                 number_builder [number_pos++] = 'e';
1336                                 c = get_char ();
1337
1338                                 if (c == '+'){
1339                                         if (number_pos == max_number_size)
1340                                                 Error_NumericConstantTooLong ();
1341                                         number_builder [number_pos++] = '+';
1342                                         c = -1;
1343                                 } else if (c == '-') {
1344                                         if (number_pos == max_number_size)
1345                                                 Error_NumericConstantTooLong ();
1346                                         number_builder [number_pos++] = '-';
1347                                         c = -1;
1348                                 } else {
1349                                         if (number_pos == max_number_size)
1350                                                 Error_NumericConstantTooLong ();
1351                                         number_builder [number_pos++] = '+';
1352                                 }
1353
1354                                 decimal_digits (c);
1355                                 c = get_char ();
1356                         }
1357
1358                         type = real_type_suffix (c);
1359                         if (type == Token.NONE && !is_real){
1360                                 putback (c);
1361                                 return adjust_int (c);
1362                         } else
1363                                 is_real = true;
1364
1365                         if (type == Token.NONE){
1366                                 putback (c);
1367                         }
1368
1369                         if (is_real)
1370                                 return adjust_real (type);
1371
1372                         Console.WriteLine ("This should not be reached");
1373                         throw new Exception ("Is Number should never reach this point");
1374                 }
1375
1376                 //
1377                 // Accepts exactly count (4 or 8) hex, no more no less
1378                 //
1379                 int getHex (int count, out int surrogate, out bool error)
1380                 {
1381                         int i;
1382                         int total = 0;
1383                         int c;
1384                         int top = count != -1 ? count : 4;
1385
1386                         get_char ();
1387                         error = false;
1388                         surrogate = 0;
1389                         for (i = 0; i < top; i++){
1390                                 c = get_char ();
1391
1392                                 if (c >= '0' && c <= '9')
1393                                         c = (int) c - (int) '0';
1394                                 else if (c >= 'A' && c <= 'F')
1395                                         c = (int) c - (int) 'A' + 10;
1396                                 else if (c >= 'a' && c <= 'f')
1397                                         c = (int) c - (int) 'a' + 10;
1398                                 else {
1399                                         error = true;
1400                                         return 0;
1401                                 }
1402
1403                                 total = (total * 16) + c;
1404                                 if (count == -1){
1405                                         int p = peek_char ();
1406                                         if (p == -1)
1407                                                 break;
1408                                         if (!is_hex ((char)p))
1409                                                 break;
1410                                 }
1411                         }
1412
1413                         if (top == 8) {
1414                                 if (total > 0x0010FFFF) {
1415                                         error = true;
1416                                         return 0;
1417                                 }
1418
1419                                 if (total >= 0x00010000) {
1420                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1421                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1422                                 }
1423                         }
1424
1425                         return total;
1426                 }
1427
1428                 int escape (int c, out int surrogate)
1429                 {
1430                         bool error;
1431                         int d;
1432                         int v;
1433
1434                         d = peek_char ();
1435                         if (c != '\\') {
1436                                 surrogate = 0;
1437                                 return c;
1438                         }
1439
1440                         switch (d){
1441                         case 'a':
1442                                 v = '\a'; break;
1443                         case 'b':
1444                                 v = '\b'; break;
1445                         case 'n':
1446                                 v = '\n'; break;
1447                         case 't':
1448                                 v = '\t'; break;
1449                         case 'v':
1450                                 v = '\v'; break;
1451                         case 'r':
1452                                 v = '\r'; break;
1453                         case '\\':
1454                                 v = '\\'; break;
1455                         case 'f':
1456                                 v = '\f'; break;
1457                         case '0':
1458                                 v = 0; break;
1459                         case '"':
1460                                 v = '"'; break;
1461                         case '\'':
1462                                 v = '\''; break;
1463                         case 'x':
1464                                 v = getHex (-1, out surrogate, out error);
1465                                 if (error)
1466                                         goto default;
1467                                 return v;
1468                         case 'u':
1469                         case 'U':
1470                                 return EscapeUnicode (d, out surrogate);
1471                         default:
1472                                 surrogate = 0;
1473                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1474                                 return d;
1475                         }
1476
1477                         get_char ();
1478                         surrogate = 0;
1479                         return v;
1480                 }
1481
1482                 int EscapeUnicode (int ch, out int surrogate)
1483                 {
1484                         bool error;
1485                         if (ch == 'U') {
1486                                 ch = getHex (8, out surrogate, out error);
1487                         } else {
1488                                 ch = getHex (4, out surrogate, out error);
1489                         }
1490
1491                         if (error)
1492                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1493
1494                         return ch;
1495                 }
1496
1497                 int get_char ()
1498                 {
1499                         int x;
1500                         if (putback_char != -1) {
1501                                 x = putback_char;
1502                                 putback_char = -1;
1503                         } else
1504                                 x = reader.Read ();
1505                         if (x == '\n') {
1506                                 advance_line ();
1507                         } else {
1508                                 col++;
1509                         }
1510                         return x;
1511                 }
1512
1513                 void advance_line ()
1514                 {
1515                         line++;
1516                         ref_line++;
1517                         previous_col = col;
1518                         col = 0;
1519                 }
1520
1521                 int peek_char ()
1522                 {
1523                         if (putback_char == -1)
1524                                 putback_char = reader.Read ();
1525                         return putback_char;
1526                 }
1527
1528                 int peek_char2 ()
1529                 {
1530                         if (putback_char != -1)
1531                                 return putback_char;
1532                         return reader.Peek ();
1533                 }
1534
1535                 void putback (int c)
1536                 {
1537                         if (putback_char != -1){
1538                                 Console.WriteLine ("Col: " + col);
1539                                 Console.WriteLine ("Row: " + line);
1540                                 Console.WriteLine ("Name: " + ref_name.Name);
1541                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1542                                 throw new Exception ("This should not happen putback on putback");
1543                         }
1544                         if (c == '\n' || col == 0) {
1545                                 // It won't happen though.
1546                                 line--;
1547                                 ref_line--;
1548                                 col = previous_col;
1549                         }
1550                         else
1551                                 col--;
1552                         putback_char = c;
1553                 }
1554
1555                 public bool advance ()
1556                 {
1557                         return peek_char () != -1 || CompleteOnEOF;
1558                 }
1559
1560                 public Object Value {
1561                         get {
1562                                 return val;
1563                         }
1564                 }
1565
1566                 public Object value ()
1567                 {
1568                         return val;
1569                 }
1570
1571                 public int token ()
1572                 {
1573                         current_token = xtoken ();
1574                         return current_token;
1575                 }
1576
1577                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1578
1579                 void get_cmd_arg (out string cmd, out string arg)
1580                 {
1581                         int c;
1582
1583                         tokens_seen = false;
1584                         arg = "";
1585
1586                         // skip over white space
1587                         do {
1588                                 c = get_char ();
1589                         } while (c == '\r' || c == ' ' || c == '\t');
1590
1591                         static_cmd_arg.Length = 0;
1592                         while (c != -1 && is_identifier_part_character ((char)c)) {
1593                                 static_cmd_arg.Append ((char)c);
1594                                 c = get_char ();
1595                                 if (c == '\\') {
1596                                         int peek = peek_char ();
1597                                         if (peek == 'U' || peek == 'u') {
1598                                                 int surrogate;
1599                                                 c = EscapeUnicode (c, out surrogate);
1600                                                 if (surrogate != 0) {
1601                                                         if (is_identifier_part_character ((char) c))
1602                                                                 static_cmd_arg.Append ((char) c);
1603                                                         c = surrogate;
1604                                                 }
1605                                         }
1606                                 }
1607                         }
1608
1609                         cmd = static_cmd_arg.ToString ();
1610
1611                         // skip over white space
1612                         while (c == '\r' || c == ' ' || c == '\t')
1613                                 c = get_char ();
1614
1615                         static_cmd_arg.Length = 0;
1616                         int has_identifier_argument = 0;
1617
1618                         while (c != -1 && c != '\n' && c != '\r') {
1619                                 if (c == '\\' && has_identifier_argument >= 0) {
1620                                         if (has_identifier_argument != 0 || (cmd == "define" || cmd == "if" || cmd == "elif" || cmd == "undef")) {
1621                                                 has_identifier_argument = 1;
1622
1623                                                 int peek = peek_char ();
1624                                                 if (peek == 'U' || peek == 'u') {
1625                                                         int surrogate;
1626                                                         c = EscapeUnicode (c, out surrogate);
1627                                                         if (surrogate != 0) {
1628                                                                 if (is_identifier_part_character ((char) c))
1629                                                                         static_cmd_arg.Append ((char) c);
1630                                                                 c = surrogate;
1631                                                         }
1632                                                 }
1633                                         } else {
1634                                                 has_identifier_argument = -1;
1635                                         }
1636                                 }
1637                                 static_cmd_arg.Append ((char) c);
1638                                 c = get_char ();
1639                         }
1640
1641                         if (static_cmd_arg.Length != 0)
1642                                 arg = static_cmd_arg.ToString ();
1643                 }
1644
1645                 //
1646                 // Handles the #line directive
1647                 //
1648                 bool PreProcessLine (string arg)
1649                 {
1650                         if (arg.Length == 0)
1651                                 return false;
1652
1653                         if (arg == "default"){
1654                                 ref_line = line;
1655                                 ref_name = file_name;
1656                                 hidden = false;
1657                                 Location.Push (file_name, ref_name);
1658                                 return true;
1659                         } else if (arg == "hidden"){
1660                                 hidden = true;
1661                                 return true;
1662                         }
1663
1664                         try {
1665                                 int pos;
1666
1667                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1668                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1669                                         pos++;
1670
1671                                         char [] quotes = { '\"' };
1672
1673                                         string name = arg.Substring (pos). Trim (quotes);
1674                                         ref_name = Location.LookupFile (file_name, name);
1675                                         file_name.AddFile (ref_name);
1676                                         hidden = false;
1677                                         Location.Push (file_name, ref_name);
1678                                 } else {
1679                                         ref_line = System.Int32.Parse (arg);
1680                                         hidden = false;
1681                                 }
1682                         } catch {
1683                                 return false;
1684                         }
1685
1686                         return true;
1687                 }
1688
1689                 //
1690                 // Handles #define and #undef
1691                 //
1692                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
1693                 {
1694                         if (ident.Length == 0 || ident == "true" || ident == "false"){
1695                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
1696                                 return;
1697                         }
1698
1699                         if (ident.IndexOfAny (simple_whitespaces) != -1){
1700                                 Error_EndLineExpected ();
1701                                 return;
1702                         }
1703
1704                         if (!is_identifier_start_character (ident [0]))
1705                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
1706
1707                         foreach (char c in ident.Substring (1)){
1708                                 if (!is_identifier_part_character (c)){
1709                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
1710                                         return;
1711                                 }
1712                         }
1713
1714                         if (!caller_is_taking)
1715                                 return;
1716
1717                         if (is_define) {
1718                                 //
1719                                 // #define ident
1720                                 //
1721                                 if (RootContext.IsConditionalDefined (ident))
1722                                         return;
1723
1724                                 file_name.AddDefine (ident);
1725                         } else {
1726                                 //
1727                                 // #undef ident
1728                                 //
1729                                 file_name.AddUndefine (ident);
1730                         }
1731                 }
1732
1733                 static byte read_hex (string arg, int pos, out bool error)
1734                 {
1735                         error = false;
1736
1737                         int total;
1738                         char c = arg [pos];
1739
1740                         if ((c >= '0') && (c <= '9'))
1741                                 total = (int) c - (int) '0';
1742                         else if ((c >= 'A') && (c <= 'F'))
1743                                 total = (int) c - (int) 'A' + 10;
1744                         else if ((c >= 'a') && (c <= 'f'))
1745                                 total = (int) c - (int) 'a' + 10;
1746                         else {
1747                                 error = true;
1748                                 return 0;
1749                         }
1750
1751                         total *= 16;
1752                         c = arg [pos+1];
1753
1754                         if ((c >= '0') && (c <= '9'))
1755                                 total += (int) c - (int) '0';
1756                         else if ((c >= 'A') && (c <= 'F'))
1757                                 total += (int) c - (int) 'A' + 10;
1758                         else if ((c >= 'a') && (c <= 'f'))
1759                                 total += (int) c - (int) 'a' + 10;
1760                         else {
1761                                 error = true;
1762                                 return 0;
1763                         }
1764
1765                         return (byte) total;
1766                 }
1767
1768                 /// <summary>
1769                 /// Handles #pragma checksum
1770                 /// </summary>
1771                 bool PreProcessPragmaChecksum (string arg)
1772                 {
1773                         if ((arg [0] != ' ') && (arg [0] != '\t'))
1774                                 return false;
1775
1776                         arg = arg.Trim (simple_whitespaces);
1777                         if ((arg.Length < 2) || (arg [0] != '"'))
1778                                 return false;
1779
1780                         StringBuilder file_sb = new StringBuilder ();
1781
1782                         int pos = 1;
1783                         char ch;
1784                         while ((ch = arg [pos++]) != '"') {
1785                                 if (pos >= arg.Length)
1786                                         return false;
1787
1788                                 if (ch == '\\') {
1789                                         if (pos+1 >= arg.Length)
1790                                                 return false;
1791                                         ch = arg [pos++];
1792                                 }
1793
1794                                 file_sb.Append (ch);
1795                         }
1796
1797                         if ((pos+2 >= arg.Length) || ((arg [pos] != ' ') && (arg [pos] != '\t')))
1798                                 return false;
1799
1800                         arg = arg.Substring (pos).Trim (simple_whitespaces);
1801                         if ((arg.Length < 42) || (arg [0] != '"') || (arg [1] != '{') ||
1802                             (arg [10] != '-') || (arg [15] != '-') || (arg [20] != '-') ||
1803                             (arg [25] != '-') || (arg [38] != '}') || (arg [39] != '"'))
1804                                 return false;
1805
1806                         bool error;
1807                         byte[] guid_bytes = new byte [16];
1808
1809                         for (int i = 0; i < 4; i++) {
1810                                 guid_bytes [i] = read_hex (arg, 2+2*i, out error);
1811                                 if (error)
1812                                         return false;
1813                         }
1814                         for (int i = 0; i < 2; i++) {
1815                                 guid_bytes [i+4] = read_hex (arg, 11+2*i, out error);
1816                                 if (error)
1817                                         return false;
1818                                 guid_bytes [i+6] = read_hex (arg, 16+2*i, out error);
1819                                 if (error)
1820                                         return false;
1821                                 guid_bytes [i+8] = read_hex (arg, 21+2*i, out error);
1822                                 if (error)
1823                                         return false;
1824                         }
1825
1826                         for (int i = 0; i < 6; i++) {
1827                                 guid_bytes [i+10] = read_hex (arg, 26+2*i, out error);
1828                                 if (error)
1829                                         return false;
1830                         }
1831
1832                         arg = arg.Substring (40).Trim (simple_whitespaces);
1833                         if ((arg.Length < 34) || (arg [0] != '"') || (arg [33] != '"'))
1834                                 return false;
1835
1836                         byte[] checksum_bytes = new byte [16];
1837                         for (int i = 0; i < 16; i++) {
1838                                 checksum_bytes [i] = read_hex (arg, 1+2*i, out error);
1839                                 if (error)
1840                                         return false;
1841                         }
1842
1843                         arg = arg.Substring (34).Trim (simple_whitespaces);
1844                         if (arg.Length > 0)
1845                                 return false;
1846
1847                         SourceFile file = Location.LookupFile (file_name, file_sb.ToString ());
1848                         file.SetChecksum (guid_bytes, checksum_bytes);
1849                         ref_name.AutoGenerated = true;
1850                         return true;
1851                 }
1852
1853                 /// <summary>
1854                 /// Handles #pragma directive
1855                 /// </summary>
1856                 void PreProcessPragma (string arg)
1857                 {
1858                         const string warning = "warning";
1859                         const string w_disable = "warning disable";
1860                         const string w_restore = "warning restore";
1861                         const string checksum = "checksum";
1862
1863                         if (arg == w_disable) {
1864                                 Report.RegisterWarningRegion (Location).WarningDisable (Location.Row);
1865                                 return;
1866                         }
1867
1868                         if (arg == w_restore) {
1869                                 Report.RegisterWarningRegion (Location).WarningEnable (Location.Row);
1870                                 return;
1871                         }
1872
1873                         if (arg.StartsWith (w_disable)) {
1874                                 int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
1875                                 foreach (int code in codes) {
1876                                         if (code != 0)
1877                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1878                                 }
1879                                 return;
1880                         }
1881
1882                         if (arg.StartsWith (w_restore)) {
1883                                 int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
1884                                 Hashtable w_table = Report.warning_ignore_table;
1885                                 foreach (int code in codes) {
1886                                         if (w_table != null && w_table.Contains (code))
1887                                                 Report.Warning (1635, 1, Location, String.Format ("Cannot restore warning `CS{0:0000}' because it was disabled globally", code));
1888                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1889                                 }
1890                                 return;
1891                         }
1892
1893                         if (arg.StartsWith (warning)) {
1894                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
1895                                 return;
1896                         }
1897
1898                         if (arg.StartsWith (checksum)) {
1899                                 if (!PreProcessPragmaChecksum (arg.Substring (checksum.Length)))
1900                                         Warning_InvalidPragmaChecksum ();
1901                                 return;
1902                         }
1903
1904                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
1905                 }
1906
1907                 int[] ParseNumbers (string text)
1908                 {
1909                         string[] string_array = text.Split (',');
1910                         int[] values = new int [string_array.Length];
1911                         int index = 0;
1912                         foreach (string string_code in string_array) {
1913                                 try {
1914                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1915                                 }
1916                                 catch (FormatException) {
1917                                         Report.Warning (1692, 1, Location, "Invalid number");
1918                                 }
1919                         }
1920                         return values;
1921                 }
1922
1923                 bool eval_val (string s)
1924                 {
1925                         if (s == "true")
1926                                 return true;
1927                         if (s == "false")
1928                                 return false;
1929
1930                         return file_name.IsConditionalDefined (s);
1931                 }
1932
1933                 bool pp_primary (ref string s)
1934                 {
1935                         s = s.Trim ();
1936                         int len = s.Length;
1937
1938                         if (len > 0){
1939                                 char c = s [0];
1940
1941                                 if (c == '('){
1942                                         s = s.Substring (1);
1943                                         bool val = pp_expr (ref s, false);
1944                                         if (s.Length > 0 && s [0] == ')'){
1945                                                 s = s.Substring (1);
1946                                                 return val;
1947                                         }
1948                                         Error_InvalidDirective ();
1949                                         return false;
1950                                 }
1951
1952                                 if (is_identifier_start_character (c)){
1953                                         int j = 1;
1954
1955                                         while (j < len){
1956                                                 c = s [j];
1957
1958                                                 if (is_identifier_part_character (c)){
1959                                                         j++;
1960                                                         continue;
1961                                                 }
1962                                                 bool v = eval_val (s.Substring (0, j));
1963                                                 s = s.Substring (j);
1964                                                 return v;
1965                                         }
1966                                         bool vv = eval_val (s);
1967                                         s = "";
1968                                         return vv;
1969                                 }
1970                         }
1971                         Error_InvalidDirective ();
1972                         return false;
1973                 }
1974
1975                 bool pp_unary (ref string s)
1976                 {
1977                         s = s.Trim ();
1978                         int len = s.Length;
1979
1980                         if (len > 0){
1981                                 if (s [0] == '!'){
1982                                         if (len > 1 && s [1] == '='){
1983                                                 Error_InvalidDirective ();
1984                                                 return false;
1985                                         }
1986                                         s = s.Substring (1);
1987                                         return ! pp_primary (ref s);
1988                                 } else
1989                                         return pp_primary (ref s);
1990                         } else {
1991                                 Error_InvalidDirective ();
1992                                 return false;
1993                         }
1994                 }
1995
1996                 bool pp_eq (ref string s)
1997                 {
1998                         bool va = pp_unary (ref s);
1999
2000                         s = s.Trim ();
2001                         int len = s.Length;
2002                         if (len > 0){
2003                                 if (s [0] == '='){
2004                                         if (len > 2 && s [1] == '='){
2005                                                 s = s.Substring (2);
2006                                                 return va == pp_unary (ref s);
2007                                         } else {
2008                                                 Error_InvalidDirective ();
2009                                                 return false;
2010                                         }
2011                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2012                                         s = s.Substring (2);
2013
2014                                         return va != pp_unary (ref s);
2015
2016                                 }
2017                         }
2018
2019                         return va;
2020
2021                 }
2022
2023                 bool pp_and (ref string s)
2024                 {
2025                         bool va = pp_eq (ref s);
2026
2027                         s = s.Trim ();
2028                         int len = s.Length;
2029                         if (len > 0){
2030                                 if (s [0] == '&'){
2031                                         if (len > 2 && s [1] == '&'){
2032                                                 s = s.Substring (2);
2033                                                 return (va & pp_and (ref s));
2034                                         } else {
2035                                                 Error_InvalidDirective ();
2036                                                 return false;
2037                                         }
2038                                 }
2039                         }
2040                         return va;
2041                 }
2042
2043                 //
2044                 // Evaluates an expression for `#if' or `#elif'
2045                 //
2046                 bool pp_expr (ref string s, bool isTerm)
2047                 {
2048                         bool va = pp_and (ref s);
2049                         s = s.Trim ();
2050                         int len = s.Length;
2051                         if (len > 0){
2052                                 char c = s [0];
2053
2054                                 if (c == '|'){
2055                                         if (len > 2 && s [1] == '|'){
2056                                                 s = s.Substring (2);
2057                                                 return va | pp_expr (ref s, isTerm);
2058                                         } else {
2059                                                 Error_InvalidDirective ();
2060                                                 return false;
2061                                         }
2062                                 }
2063                                 if (isTerm) {
2064                                         Error_EndLineExpected ();
2065                                         return false;
2066                                 }
2067                         }
2068
2069                         return va;
2070                 }
2071
2072                 bool eval (string s)
2073                 {
2074                         bool v = pp_expr (ref s, true);
2075                         s = s.Trim ();
2076                         if (s.Length != 0){
2077                                 return false;
2078                         }
2079
2080                         return v;
2081                 }
2082
2083                 void Error_NumericConstantTooLong ()
2084                 {
2085                         Report.Error (1021, Location, "Numeric constant too long");
2086                 }
2087
2088                 void Error_InvalidDirective ()
2089                 {
2090                         Report.Error (1517, Location, "Invalid preprocessor directive");
2091                 }
2092
2093                 void Error_UnexpectedDirective (string extra)
2094                 {
2095                         Report.Error (
2096                                 1028, Location,
2097                                 "Unexpected processor directive ({0})", extra);
2098                 }
2099
2100                 void Error_TokensSeen ()
2101                 {
2102                         Report.Error (1032, Location,
2103                                 "Cannot define or undefine preprocessor symbols after first token in file");
2104                 }
2105
2106                 void Eror_WrongPreprocessorLocation ()
2107                 {
2108                         Report.Error (1040, Location,
2109                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2110                 }
2111
2112                 void Error_EndLineExpected ()
2113                 {
2114                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2115                 }
2116
2117                 void Warning_InvalidPragmaChecksum ()
2118                 {
2119                         Report.Warning (1695, 1, Location,
2120                                         "Invalid #pragma checksum syntax; should be " +
2121                                         "#pragma checksum \"filename\" " +
2122                                         "\"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2123                 }
2124                 //
2125                 // if true, then the code continues processing the code
2126                 // if false, the code stays in a loop until another directive is
2127                 // reached.
2128                 // When caller_is_taking is false we ignore all directives except the ones
2129                 // which can help us to identify where the #if block ends
2130                 bool handle_preprocessing_directive (bool caller_is_taking)
2131                 {
2132                         string cmd, arg;
2133                         bool region_directive = false;
2134
2135                         get_cmd_arg (out cmd, out arg);
2136
2137                         // Eat any trailing whitespaces and single-line comments
2138                         if (arg.IndexOf ("//") != -1)
2139                                 arg = arg.Substring (0, arg.IndexOf ("//"));
2140                         arg = arg.Trim (simple_whitespaces);
2141
2142                         //
2143                         // The first group of pre-processing instructions is always processed
2144                         //
2145                         switch (cmd){
2146                         case "region":
2147                                 region_directive = true;
2148                                 arg = "true";
2149                                 goto case "if";
2150
2151                         case "endregion":
2152                                 if (ifstack == null || ifstack.Count == 0){
2153                                         Error_UnexpectedDirective ("no #region for this #endregion");
2154                                         return true;
2155                                 }
2156                                 int pop = (int) ifstack.Pop ();
2157
2158                                 if ((pop & REGION) == 0)
2159                                         Report.Error (1027, Location, "Expected `#endif' directive");
2160
2161                                 return caller_is_taking;
2162
2163                         case "if":
2164                                 if (ifstack == null)
2165                                         ifstack = new Stack (2);
2166
2167                                 int flags = region_directive ? REGION : 0;
2168                                 if (ifstack.Count == 0){
2169                                         flags |= PARENT_TAKING;
2170                                 } else {
2171                                         int state = (int) ifstack.Peek ();
2172                                         if ((state & TAKING) != 0) {
2173                                                 flags |= PARENT_TAKING;
2174                                         }
2175                                 }
2176
2177                                 if (caller_is_taking && eval (arg)) {
2178                                         ifstack.Push (flags | TAKING);
2179                                         return true;
2180                                 }
2181                                 ifstack.Push (flags);
2182                                 return false;
2183
2184                         case "endif":
2185                                 if (ifstack == null || ifstack.Count == 0){
2186                                         Error_UnexpectedDirective ("no #if for this #endif");
2187                                         return true;
2188                                 } else {
2189                                         pop = (int) ifstack.Pop ();
2190
2191                                         if ((pop & REGION) != 0)
2192                                                 Report.Error (1038, Location, "#endregion directive expected");
2193
2194                                         if (arg.Length != 0) {
2195                                                 Error_EndLineExpected ();
2196                                         }
2197
2198                                         if (ifstack.Count == 0)
2199                                                 return true;
2200
2201                                         int state = (int) ifstack.Peek ();
2202                                         return (state & TAKING) != 0;
2203                                 }
2204
2205                         case "elif":
2206                                 if (ifstack == null || ifstack.Count == 0){
2207                                         Error_UnexpectedDirective ("no #if for this #elif");
2208                                         return true;
2209                                 } else {
2210                                         int state = (int) ifstack.Pop ();
2211
2212                                         if ((state & REGION) != 0) {
2213                                                 Report.Error (1038, Location, "#endregion directive expected");
2214                                                 return true;
2215                                         }
2216
2217                                         if ((state & ELSE_SEEN) != 0){
2218                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2219                                                 return true;
2220                                         }
2221
2222                                         if ((state & TAKING) != 0) {
2223                                                 ifstack.Push (0);
2224                                                 return false;
2225                                         }
2226
2227                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2228                                                 ifstack.Push (state | TAKING);
2229                                                 return true;
2230                                         }
2231
2232                                         ifstack.Push (state);
2233                                         return false;
2234                                 }
2235
2236                         case "else":
2237                                 if (ifstack == null || ifstack.Count == 0){
2238                                         Error_UnexpectedDirective ("no #if for this #else");
2239                                         return true;
2240                                 } else {
2241                                         int state = (int) ifstack.Peek ();
2242
2243                                         if ((state & REGION) != 0) {
2244                                                 Report.Error (1038, Location, "#endregion directive expected");
2245                                                 return true;
2246                                         }
2247
2248                                         if ((state & ELSE_SEEN) != 0){
2249                                                 Error_UnexpectedDirective ("#else within #else");
2250                                                 return true;
2251                                         }
2252
2253                                         ifstack.Pop ();
2254
2255                                         if (arg.Length != 0) {
2256                                                 Error_EndLineExpected ();
2257                                                 return true;
2258                                         }
2259
2260                                         bool ret = false;
2261                                         if ((state & PARENT_TAKING) != 0) {
2262                                                 ret = (state & TAKING) == 0;
2263
2264                                                 if (ret)
2265                                                         state |= TAKING;
2266                                                 else
2267                                                         state &= ~TAKING;
2268                                         }
2269
2270                                         ifstack.Push (state | ELSE_SEEN);
2271
2272                                         return ret;
2273                                 }
2274                         case "define":
2275                                 if (any_token_seen){
2276                                         Error_TokensSeen ();
2277                                         return caller_is_taking;
2278                                 }
2279                                 PreProcessDefinition (true, arg, caller_is_taking);
2280                                 return caller_is_taking;
2281
2282                         case "undef":
2283                                 if (any_token_seen){
2284                                         Error_TokensSeen ();
2285                                         return caller_is_taking;
2286                                 }
2287                                 PreProcessDefinition (false, arg, caller_is_taking);
2288                                 return caller_is_taking;
2289                         }
2290
2291                         //
2292                         // These are only processed if we are in a `taking' block
2293                         //
2294                         if (!caller_is_taking)
2295                                 return false;
2296
2297                         switch (cmd){
2298                         case "error":
2299                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2300                                 return true;
2301
2302                         case "warning":
2303                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2304                                 return true;
2305
2306                         case "pragma":
2307                                 if (RootContext.Version == LanguageVersion.ISO_1) {
2308                                         Report.FeatureIsNotAvailable (Location, "#pragma");
2309                                         return true;
2310                                 }
2311
2312                                 PreProcessPragma (arg);
2313                                 return true;
2314
2315                         case "line":
2316                                 if (!PreProcessLine (arg))
2317                                         Report.Error (
2318                                                 1576, Location,
2319                                                 "The line number specified for #line directive is missing or invalid");
2320                                 return caller_is_taking;
2321                         }
2322
2323                         Report.Error (1024, Location, "Wrong preprocessor directive");
2324                         return true;
2325
2326                 }
2327
2328                 private int consume_string (bool quoted)
2329                 {
2330                         int c;
2331                         string_builder.Length = 0;
2332
2333                         while ((c = get_char ()) != -1){
2334                                 if (c == '"'){
2335                                         if (quoted && peek_char () == '"'){
2336                                                 string_builder.Append ((char) c);
2337                                                 get_char ();
2338                                                 continue;
2339                                         } else {
2340                                                 val = string_builder.ToString ();
2341                                                 return Token.LITERAL_STRING;
2342                                         }
2343                                 }
2344
2345                                 if (c == '\n'){
2346                                         if (!quoted)
2347                                                 Report.Error (1010, Location, "Newline in constant");
2348                                 }
2349
2350                                 if (!quoted){
2351                                         int surrogate;
2352                                         c = escape (c, out surrogate);
2353                                         if (c == -1)
2354                                                 return Token.ERROR;
2355                                         if (surrogate != 0) {
2356                                                 string_builder.Append ((char) c);
2357                                                 c = surrogate;
2358                                         }
2359                                 }
2360                                 string_builder.Append ((char) c);
2361                         }
2362
2363                         Report.Error (1039, Location, "Unterminated string literal");
2364                         return Token.EOF;
2365                 }
2366
2367                 private int consume_identifier (int s)
2368                 {
2369                         int res = consume_identifier (s, false);
2370
2371                         if (doc_state == XmlCommentState.Allowed)
2372                                 doc_state = XmlCommentState.NotAllowed;
2373
2374                         return res;
2375                 }
2376
2377                 private int consume_identifier (int c, bool quoted)
2378                 {
2379                         int pos = 0;
2380
2381                         if (c == '\\') {
2382                                 int surrogate;
2383                                 c = escape (c, out surrogate);
2384                                 if (surrogate != 0) {
2385                                         id_builder [pos++] = (char) c;
2386                                         c = surrogate;
2387                                 }
2388                         }
2389
2390                         id_builder [pos++] = (char) c;
2391                         Location loc = Location;
2392
2393                         while ((c = get_char ()) != -1) {
2394                         loop:
2395                                 if (is_identifier_part_character ((char) c)){
2396                                         if (pos == max_id_size){
2397                                                 Report.Error (645, loc, "Identifier too long (limit is 512 chars)");
2398                                                 return Token.ERROR;
2399                                         }
2400
2401                                         id_builder [pos++] = (char) c;
2402                                 } else if (c == '\\') {
2403                                         int surrogate;
2404                                         c = escape (c, out surrogate);
2405                                         if (surrogate != 0) {
2406                                                 if (is_identifier_part_character ((char) c))
2407                                                         id_builder [pos++] = (char) c;
2408                                                 c = surrogate;
2409                                         }
2410                                         goto loop;
2411                                 } else {
2412                                         putback (c);
2413                                         break;
2414                                 }
2415                         }
2416
2417                         //
2418                         // Optimization: avoids doing the keyword lookup
2419                         // on uppercase letters
2420                         //
2421                         if (id_builder [0] >= '_' && !quoted) {
2422                                 int keyword = GetKeyword (id_builder, pos);
2423                                 if (keyword != -1) {
2424                                         // TODO: No need to store location for keyword, required location cleanup
2425                                         val = loc;
2426                                         return keyword;
2427                                 }
2428                         }
2429
2430                         //
2431                         // Keep identifiers in an array of hashtables to avoid needless
2432                         // allocations
2433                         //
2434                         CharArrayHashtable identifiers_group = identifiers [pos];
2435                         if (identifiers_group != null) {
2436                                 val = identifiers_group [id_builder];
2437                                 if (val != null) {
2438                                         val = new LocatedToken (loc, (string) val);
2439                                         if (quoted)
2440                                                 AddEscapedIdentifier ((LocatedToken) val);
2441                                         return Token.IDENTIFIER;
2442                                 }
2443                         } else {
2444                                 identifiers_group = new CharArrayHashtable (pos);
2445                                 identifiers [pos] = identifiers_group;
2446                         }
2447
2448                         char [] chars = new char [pos];
2449                         Array.Copy (id_builder, chars, pos);
2450
2451                         val = new String (id_builder, 0, pos);
2452                         identifiers_group.Add (chars, val);
2453
2454                         if (RootContext.Version == LanguageVersion.ISO_1) {
2455                                 for (int i = 1; i < chars.Length; i += 3) {
2456                                         if (chars [i] == '_' && (chars [i - 1] == '_' || chars [i + 1] == '_')) {
2457                                                 Report.Error (1638, loc,
2458                                                         "`{0}': Any identifier with double underscores cannot be used when ISO language version mode is specified", val.ToString ());
2459                                         }
2460                                 }
2461                         }
2462
2463                         val = new LocatedToken (loc, (string) val);
2464                         if (quoted)
2465                                 AddEscapedIdentifier ((LocatedToken) val);
2466                         return Token.IDENTIFIER;
2467                 }
2468
2469                 public int xtoken ()
2470                 {
2471                         int d, c;
2472
2473                         // Whether we have seen comments on the current line
2474                         bool comments_seen = false;
2475                         while ((c = get_char ()) != -1) {
2476                                 switch (c) {
2477                                 case '\t':
2478                                         col = ((col + 8) / 8) * 8;
2479                                         continue;
2480
2481                                 case ' ':
2482                                 case '\f':
2483                                 case '\v':
2484                                 case 0xa0:
2485                                 case 0:
2486                                 case 0xFEFF:    // Ignore BOM anywhere in the file
2487                                         continue;
2488
2489 /*                              This is required for compatibility with .NET
2490                                 case 0xEF:
2491                                         if (peek_char () == 0xBB) {
2492                                                 PushPosition ();
2493                                                 get_char ();
2494                                                 if (get_char () == 0xBF)
2495                                                         continue;
2496                                                 PopPosition ();
2497                                         }
2498                                         break;
2499 */
2500                                 case '\r':
2501                                         if (peek_char () != '\n')
2502                                                 advance_line ();
2503                                         else
2504                                                 get_char ();
2505
2506                                         any_token_seen |= tokens_seen;
2507                                         tokens_seen = false;
2508                                         comments_seen = false;
2509                                         continue;
2510
2511                                 case '\\':
2512                                         tokens_seen = true;
2513                                         return consume_identifier (c);
2514
2515                                 case '{':
2516                                         val = Location;
2517                                         return Token.OPEN_BRACE;
2518                                 case '}':
2519                                         val = Location;
2520                                         return Token.CLOSE_BRACE;
2521                                 case '[':
2522                                         // To block doccomment inside attribute declaration.
2523                                         if (doc_state == XmlCommentState.Allowed)
2524                                                 doc_state = XmlCommentState.NotAllowed;
2525                                         return Token.OPEN_BRACKET;
2526                                 case ']':
2527                                         return Token.CLOSE_BRACKET;
2528                                 case '(':
2529                                         val = Location;
2530                                         //
2531                                         // An expression versions of parens can appear in block context only
2532                                         //
2533                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
2534
2535                                                 //
2536                                                 // Optmize most common case where we know that parens
2537                                                 // is not special
2538                                                 //
2539                                                 switch (current_token) {
2540                                                 case Token.IDENTIFIER:
2541                                                 case Token.IF:
2542                                                 case Token.FOR:
2543                                                 case Token.FOREACH:
2544                                                 case Token.TYPEOF:
2545                                                 case Token.WHILE:
2546                                                 case Token.USING:
2547                                                 case Token.DEFAULT:
2548                                                 case Token.DELEGATE:
2549                                                 case Token.OP_GENERICS_GT:
2550                                                         return Token.OPEN_PARENS;
2551                                                 }
2552
2553                                                 // Optimize using peek
2554                                                 int xx = peek_char ();
2555                                                 switch (xx) {
2556                                                 case '(':
2557                                                 case '\'':
2558                                                 case '"':
2559                                                 case '0':
2560                                                 case '1':
2561                                                         return Token.OPEN_PARENS;
2562                                                 }
2563
2564                                                 lambda_arguments_parsing = true;
2565                                                 PushPosition ();
2566                                                 d = TokenizeOpenParens ();
2567                                                 PopPosition ();
2568                                                 lambda_arguments_parsing = false;
2569                                                 return d;
2570                                         }
2571
2572                                         return Token.OPEN_PARENS;
2573                                 case ')':
2574                                         return Token.CLOSE_PARENS;
2575                                 case ',':
2576                                         return Token.COMMA;
2577                                 case ';':
2578                                         return Token.SEMICOLON;
2579                                 case '~':
2580                                         return Token.TILDE;
2581                                 case '?':
2582                                         return TokenizePossibleNullableType ();
2583                                 case '<':
2584                                         if (parsing_generic_less_than++ > 0)
2585                                                 return Token.OP_GENERICS_LT;
2586
2587                                         return TokenizeLessThan ();
2588
2589                                 case '>':
2590                                         d = peek_char ();
2591
2592                                         if (d == '='){
2593                                                 get_char ();
2594                                                 return Token.OP_GE;
2595                                         }
2596
2597                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
2598                                                 parsing_generic_less_than--;
2599                                                 return Token.OP_GENERICS_GT;
2600                                         }
2601
2602                                         if (d == '>') {
2603                                                 get_char ();
2604                                                 d = peek_char ();
2605
2606                                                 if (d == '=') {
2607                                                         get_char ();
2608                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
2609                                                 }
2610                                                 return Token.OP_SHIFT_RIGHT;
2611                                         }
2612
2613                                         return Token.OP_GT;
2614
2615                                 case '+':
2616                                         d = peek_char ();
2617                                         if (d == '+') {
2618                                                 d = Token.OP_INC;
2619                                         } else if (d == '=') {
2620                                                 d = Token.OP_ADD_ASSIGN;
2621                                         } else {
2622                                                 return Token.PLUS;
2623                                         }
2624                                         get_char ();
2625                                         return d;
2626
2627                                 case '-':
2628                                         d = peek_char ();
2629                                         if (d == '-') {
2630                                                 d = Token.OP_DEC;
2631                                         } else if (d == '=')
2632                                                 d = Token.OP_SUB_ASSIGN;
2633                                         else if (d == '>')
2634                                                 d = Token.OP_PTR;
2635                                         else {
2636                                                 return Token.MINUS;
2637                                         }
2638                                         get_char ();
2639                                         return d;
2640
2641                                 case '!':
2642                                         if (peek_char () == '='){
2643                                                 get_char ();
2644                                                 return Token.OP_NE;
2645                                         }
2646                                         return Token.BANG;
2647
2648                                 case '=':
2649                                         d = peek_char ();
2650                                         if (d == '='){
2651                                                 get_char ();
2652                                                 return Token.OP_EQ;
2653                                         }
2654                                         if (d == '>'){
2655                                                 get_char ();
2656                                                 return Token.ARROW;
2657                                         }
2658
2659                                         return Token.ASSIGN;
2660
2661                                 case '&':
2662                                         d = peek_char ();
2663                                         if (d == '&'){
2664                                                 get_char ();
2665                                                 return Token.OP_AND;
2666                                         }
2667                                         if (d == '='){
2668                                                 get_char ();
2669                                                 return Token.OP_AND_ASSIGN;
2670                                         }
2671                                         return Token.BITWISE_AND;
2672
2673                                 case '|':
2674                                         d = peek_char ();
2675                                         if (d == '|'){
2676                                                 get_char ();
2677                                                 return Token.OP_OR;
2678                                         }
2679                                         if (d == '='){
2680                                                 get_char ();
2681                                                 return Token.OP_OR_ASSIGN;
2682                                         }
2683                                         return Token.BITWISE_OR;
2684
2685                                 case '*':
2686                                         if (peek_char () == '='){
2687                                                 get_char ();
2688                                                 return Token.OP_MULT_ASSIGN;
2689                                         }
2690                                         val = Location;
2691                                         return Token.STAR;
2692
2693                                 case '/':
2694                                         d = peek_char ();
2695                                         if (d == '='){
2696                                                 get_char ();
2697                                                 return Token.OP_DIV_ASSIGN;
2698                                         }
2699
2700                                         // Handle double-slash comments.
2701                                         if (d == '/'){
2702                                                 get_char ();
2703                                                 if (RootContext.Documentation != null && peek_char () == '/') {
2704                                                         get_char ();
2705                                                         // Don't allow ////.
2706                                                         if ((d = peek_char ()) != '/') {
2707                                                                 update_comment_location ();
2708                                                                 if (doc_state == XmlCommentState.Allowed)
2709                                                                         handle_one_line_xml_comment ();
2710                                                                 else if (doc_state == XmlCommentState.NotAllowed)
2711                                                                         warn_incorrect_doc_comment ();
2712                                                         }
2713                                                 }
2714                                                 while ((d = get_char ()) != -1 && (d != '\n') && d != '\r');
2715
2716                                                 any_token_seen |= tokens_seen;
2717                                                 tokens_seen = false;
2718                                                 comments_seen = false;
2719                                                 continue;
2720                                         } else if (d == '*'){
2721                                                 get_char ();
2722                                                 bool docAppend = false;
2723                                                 if (RootContext.Documentation != null && peek_char () == '*') {
2724                                                         get_char ();
2725                                                         update_comment_location ();
2726                                                         // But when it is /**/, just do nothing.
2727                                                         if (peek_char () == '/') {
2728                                                                 get_char ();
2729                                                                 continue;
2730                                                         }
2731                                                         if (doc_state == XmlCommentState.Allowed)
2732                                                                 docAppend = true;
2733                                                         else if (doc_state == XmlCommentState.NotAllowed)
2734                                                                 warn_incorrect_doc_comment ();
2735                                                 }
2736
2737                                                 int current_comment_start = 0;
2738                                                 if (docAppend) {
2739                                                         current_comment_start = xml_comment_buffer.Length;
2740                                                         xml_comment_buffer.Append (Environment.NewLine);
2741                                                 }
2742
2743                                                 while ((d = get_char ()) != -1){
2744                                                         if (d == '*' && peek_char () == '/'){
2745                                                                 get_char ();
2746                                                                 comments_seen = true;
2747                                                                 break;
2748                                                         }
2749                                                         if (docAppend)
2750                                                                 xml_comment_buffer.Append ((char) d);
2751
2752                                                         if (d == '\n'){
2753                                                                 any_token_seen |= tokens_seen;
2754                                                                 tokens_seen = false;
2755                                                                 //
2756                                                                 // Reset 'comments_seen' just to be consistent.
2757                                                                 // It doesn't matter either way, here.
2758                                                                 //
2759                                                                 comments_seen = false;
2760                                                         }
2761                                                 }
2762                                                 if (!comments_seen)
2763                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
2764
2765                                                 if (docAppend)
2766                                                         update_formatted_doc_comment (current_comment_start);
2767                                                 continue;
2768                                         }
2769                                         return Token.DIV;
2770
2771                                 case '%':
2772                                         if (peek_char () == '='){
2773                                                 get_char ();
2774                                                 return Token.OP_MOD_ASSIGN;
2775                                         }
2776                                         return Token.PERCENT;
2777
2778                                 case '^':
2779                                         if (peek_char () == '='){
2780                                                 get_char ();
2781                                                 return Token.OP_XOR_ASSIGN;
2782                                         }
2783                                         return Token.CARRET;
2784
2785                                 case ':':
2786                                         if (peek_char () == ':') {
2787                                                 get_char ();
2788                                                 return Token.DOUBLE_COLON;
2789                                         }
2790                                         return Token.COLON;
2791
2792                                 case '0': case '1': case '2': case '3': case '4':
2793                                 case '5': case '6': case '7': case '8': case '9':
2794                                         tokens_seen = true;
2795                                         return is_number (c);
2796
2797                                 case '\n': // white space
2798                                         any_token_seen |= tokens_seen;
2799                                         tokens_seen = false;
2800                                         comments_seen = false;
2801                                         continue;
2802
2803                                 case '.':
2804                                         tokens_seen = true;
2805                                         d = peek_char ();
2806                                         if (d >= '0' && d <= '9')
2807                                                 return is_number (c);
2808                                         return Token.DOT;
2809
2810                                 case '#':
2811                                         if (tokens_seen || comments_seen) {
2812                                                 Eror_WrongPreprocessorLocation ();
2813                                                 return Token.ERROR;
2814                                         }
2815
2816                                         if (handle_preprocessing_directive (true))
2817                                                 continue;
2818
2819                                         bool directive_expected = false;
2820                                         while ((c = get_char ()) != -1) {
2821                                                 if (col == 1) {
2822                                                         directive_expected = true;
2823                                                 } else if (!directive_expected) {
2824                                                         // TODO: Implement comment support for disabled code and uncomment this code
2825 //                                                      if (c == '#') {
2826 //                                                              Eror_WrongPreprocessorLocation ();
2827 //                                                              return Token.ERROR;
2828 //                                                      }
2829                                                         continue;
2830                                                 }
2831
2832                                                 if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
2833                                                         continue;
2834
2835                                                 if (c == '#') {
2836                                                         if (handle_preprocessing_directive (false))
2837                                                                 break;
2838                                                 }
2839                                                 directive_expected = false;
2840                                         }
2841
2842                                         if (c != -1) {
2843                                                 tokens_seen = false;
2844                                                 continue;
2845                                         }
2846
2847                                         return Token.EOF;
2848
2849                                 case '"':
2850                                         return consume_string (false);
2851
2852                                 case '\'':
2853                                         return TokenizeBackslash ();
2854
2855                                 case '@':
2856                                         c = get_char ();
2857                                         if (c == '"') {
2858                                                 tokens_seen = true;
2859                                                 return consume_string (true);
2860                                         }
2861
2862                                         if (is_identifier_start_character (c)){
2863                                                 return consume_identifier (c, true);
2864                                         }
2865
2866                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
2867                                         return Token.ERROR;
2868
2869                                 case EvalStatementParserCharacter:
2870                                         return Token.EVAL_STATEMENT_PARSER;
2871                                 case EvalCompilationUnitParserCharacter:
2872                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
2873                                 case EvalUsingDeclarationsParserCharacter:
2874                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
2875                                 }
2876
2877                                 if (is_identifier_start_character (c)) {
2878                                         tokens_seen = true;
2879                                         return consume_identifier (c);
2880                                 }
2881
2882                                 error_details = ((char)c).ToString ();
2883                                 return Token.ERROR;
2884                         }
2885
2886                         if (CompleteOnEOF){
2887                                 if (generated)
2888                                         return Token.COMPLETE_COMPLETION;
2889
2890                                 generated = true;
2891                                 return Token.GENERATE_COMPLETION;
2892                         }
2893
2894
2895                         return Token.EOF;
2896                 }
2897
2898                 int TokenizeBackslash ()
2899                 {
2900                         int c = get_char ();
2901                         tokens_seen = true;
2902                         if (c == '\'') {
2903                                 error_details = "Empty character literal";
2904                                 Report.Error (1011, Location, error_details);
2905                                 return Token.ERROR;
2906                         }
2907                         if (c == '\r' || c == '\n') {
2908                                 Report.Error (1010, Location, "Newline in constant");
2909                                 return Token.ERROR;
2910                         }
2911
2912                         int d;
2913                         c = escape (c, out d);
2914                         if (c == -1)
2915                                 return Token.ERROR;
2916                         if (d != 0)
2917                                 throw new NotImplementedException ();
2918
2919                         val = (char) c;
2920                         c = get_char ();
2921
2922                         if (c != '\'') {
2923                                 Report.Error (1012, Location, "Too many characters in character literal");
2924
2925                                 // Try to recover, read until newline or next "'"
2926                                 while ((c = get_char ()) != -1) {
2927                                         if (c == '\n' || c == '\'')
2928                                                 break;
2929                                 }
2930                                 return Token.ERROR;
2931                         }
2932
2933                         return Token.LITERAL_CHARACTER;
2934                 }
2935
2936                 int TokenizeLessThan ()
2937                 {
2938                         int d;
2939                         if (handle_typeof) {
2940                                 PushPosition ();
2941                                 if (parse_generic_dimension (out d)) {
2942                                         val = d;
2943                                         DiscardPosition ();
2944                                         return Token.GENERIC_DIMENSION;
2945                                 }
2946                                 PopPosition ();
2947                         }
2948
2949                         // Save current position and parse next token.
2950                         PushPosition ();
2951                         if (parse_less_than ()) {
2952                                 if (parsing_generic_declaration && token () != Token.DOT) {
2953                                         d = Token.OP_GENERICS_LT_DECL;
2954                                 } else {
2955                                         d = Token.OP_GENERICS_LT;
2956                                 }
2957                                 PopPosition ();
2958                                 return d;
2959                         }
2960
2961                         PopPosition ();
2962                         parsing_generic_less_than = 0;
2963
2964                         d = peek_char ();
2965                         if (d == '<') {
2966                                 get_char ();
2967                                 d = peek_char ();
2968
2969                                 if (d == '=') {
2970                                         get_char ();
2971                                         return Token.OP_SHIFT_LEFT_ASSIGN;
2972                                 }
2973                                 return Token.OP_SHIFT_LEFT;
2974                         }
2975
2976                         if (d == '=') {
2977                                 get_char ();
2978                                 return Token.OP_LE;
2979                         }
2980                         return Token.OP_LT;
2981                 }
2982
2983                 //
2984                 // Handles one line xml comment
2985                 //
2986                 private void handle_one_line_xml_comment ()
2987                 {
2988                         int c;
2989                         while ((c = peek_char ()) == ' ')
2990                                 get_char (); // skip heading whitespaces.
2991                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
2992                                 xml_comment_buffer.Append ((char) get_char ());
2993                         }
2994                         if (c == '\r' || c == '\n')
2995                                 xml_comment_buffer.Append (Environment.NewLine);
2996                 }
2997
2998                 //
2999                 // Remove heading "*" in Javadoc-like xml documentation.
3000                 //
3001                 private void update_formatted_doc_comment (int current_comment_start)
3002                 {
3003                         int length = xml_comment_buffer.Length - current_comment_start;
3004                         string [] lines = xml_comment_buffer.ToString (
3005                                 current_comment_start,
3006                                 length).Replace ("\r", "").Split ('\n');
3007
3008                         // The first line starts with /**, thus it is not target
3009                         // for the format check.
3010                         for (int i = 1; i < lines.Length; i++) {
3011                                 string s = lines [i];
3012                                 int idx = s.IndexOf ('*');
3013                                 string head = null;
3014                                 if (idx < 0) {
3015                                         if (i < lines.Length - 1)
3016                                                 return;
3017                                         head = s;
3018                                 } else
3019                                         head = s.Substring (0, idx);
3020                                 foreach (char c in head)
3021                                         if (c != ' ')
3022                                                 return;
3023                                 lines [i] = s.Substring (idx + 1);
3024                         }
3025                         xml_comment_buffer.Remove (current_comment_start, length);
3026                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3027                 }
3028
3029                 //
3030                 // Updates current comment location.
3031                 //
3032                 private void update_comment_location ()
3033                 {
3034                         if (current_comment_location.IsNull) {
3035                                 // "-2" is for heading "//" or "/*"
3036                                 current_comment_location =
3037                                         new Location (ref_line, hidden ? -1 : col - 2);
3038                         }
3039                 }
3040
3041                 //
3042                 // Checks if there was incorrect doc comments and raise
3043                 // warnings.
3044                 //
3045                 public void check_incorrect_doc_comment ()
3046                 {
3047                         if (xml_comment_buffer.Length > 0)
3048                                 warn_incorrect_doc_comment ();
3049                 }
3050
3051                 //
3052                 // Raises a warning when tokenizer found incorrect doccomment
3053                 // markup.
3054                 //
3055                 private void warn_incorrect_doc_comment ()
3056                 {
3057                         if (doc_state != XmlCommentState.Error) {
3058                                 doc_state = XmlCommentState.Error;
3059                                 // in csc, it is 'XML comment is not placed on
3060                                 // a valid language element'. But that does not
3061                                 // make sense.
3062                                 Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
3063                         }
3064                 }
3065
3066                 //
3067                 // Consumes the saved xml comment lines (if any)
3068                 // as for current target member or type.
3069                 //
3070                 public string consume_doc_comment ()
3071                 {
3072                         if (xml_comment_buffer.Length > 0) {
3073                                 string ret = xml_comment_buffer.ToString ();
3074                                 reset_doc_comment ();
3075                                 return ret;
3076                         }
3077                         return null;
3078                 }
3079
3080                 void reset_doc_comment ()
3081                 {
3082                         xml_comment_buffer.Length = 0;
3083                         current_comment_location = Location.Null;
3084                 }
3085
3086                 public void cleanup ()
3087                 {
3088                         if (ifstack != null && ifstack.Count >= 1) {
3089                                 int state = (int) ifstack.Pop ();
3090                                 if ((state & REGION) != 0)
3091                                         Report.Error (1038, Location, "#endregion directive expected");
3092                                 else
3093                                         Report.Error (1027, Location, "Expected `#endif' directive");
3094                         }
3095                 }
3096         }
3097
3098         //
3099         // Indicates whether it accepts XML documentation or not.
3100         //
3101         public enum XmlCommentState {
3102                 // comment is allowed in this state.
3103                 Allowed,
3104                 // comment is not allowed in this state.
3105                 NotAllowed,
3106                 // once comments appeared when it is NotAllowed, then the
3107                 // state is changed to it, until the state is changed to
3108                 // .Allowed.
3109                 Error
3110         }
3111 }
3112