mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@seznam.cz)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 //
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections;
  18 using System.IO;
  19 using System.Globalization;
  20 using System.Reflection;
  21
  22 namespace Mono.CSharp
  23 {
  24         /// <summary>
  25         ///    Tokenizer for C# source code.
  26         /// </summary>
  27
  28         public class Tokenizer : yyParser.yyInput
  29         {
  30                 class KeywordEntry
  31                 {
  32                         public readonly int Token;
  33                         public KeywordEntry Next;
  34                         public readonly char[] Value;
  35
  36                         public KeywordEntry (string value, int token)
  37                         {
  38                                 this.Value = value.ToCharArray ();
  39                                 this.Token = token;
  40                         }
  41                 }
  42
  43                 SeekableStreamReader reader;
  44                 SourceFile ref_name;
  45                 CompilationUnit file_name;
  46                 bool hidden = false;
  47                 int ref_line = 1;
  48                 int line = 1;
  49                 int col = 0;
  50                 int previous_col;
  51                 int current_token;
  52                 bool handle_get_set = false;
  53                 bool handle_remove_add = false;
  54                 bool handle_where = false;
  55                 bool handle_typeof = false;
  56                 bool lambda_arguments_parsing;
  57                 Location current_comment_location = Location.Null;
  58                 ArrayList escaped_identifiers;
  59                 int parsing_generic_less_than;
  60
  61                 //
  62                 // Used mainly for parser optimizations. Some expressions for instance
  63                 // can appear only in block (including initializer, base initializer)
  64                 // scope only
  65                 //
  66                 public int parsing_block;
  67                 internal bool query_parsing;
  68
  69                 //
  70                 // When parsing type only, useful for ambiguous nullable types
  71                 //
  72                 public int parsing_type;
  73
  74                 //
  75                 // Set when parsing generic declaration (type or method header)
  76                 //
  77                 public bool parsing_generic_declaration;
  78
  79                 //
  80                 // The value indicates that we have not reach any declaration or
  81                 // namespace yet
  82                 //
  83                 public int parsing_declaration;
  84
  85                 //
  86                 // The special character to inject on streams to trigger the EXPRESSION_PARSE
  87                 // token to be returned.   It just happens to be a Unicode character that
  88                 // would never be part of a program (can not be an identifier).
  89                 //
  90                 // This character is only tested just before the tokenizer is about to report
  91                 // an error;   So on the regular operation mode, this addition will have no
  92                 // impact on the tokenizer's performance.
  93                 //
  94
  95                 public const int EvalStatementParserCharacter = 0x2190;   // Unicode Left Arrow
  96                 public const int EvalCompilationUnitParserCharacter = 0x2191;  // Unicode Arrow
  97                 public const int EvalUsingDeclarationsParserCharacter = 0x2192;  // Unicode Arrow
  98
  99                 //
 100                 // XML documentation buffer. The save point is used to divide
 101                 // comments on types and comments on members.
 102                 //
 103                 StringBuilder xml_comment_buffer;
 104
 105                 //
 106                 // See comment on XmlCommentState enumeration.
 107                 //
 108                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 109
 110                 //
 111                 // Whether tokens have been seen on this line
 112                 //
 113                 bool tokens_seen = false;
 114
 115                 //
 116                 // Whether a token has been seen on the file
 117                 // This is needed because `define' is not allowed to be used
 118                 // after a token has been seen.
 119                 //
 120                 bool any_token_seen = false;
 121
 122                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 123
 124                 public bool PropertyParsing {
 125                         get { return handle_get_set; }
 126                         set { handle_get_set = value; }
 127                 }
 128
 129                 public bool EventParsing {
 130                         get { return handle_remove_add; }
 131                         set { handle_remove_add = value; }
 132                 }
 133
 134                 public bool ConstraintsParsing {
 135                         get { return handle_where; }
 136                         set { handle_where = value; }
 137                 }
 138
 139                 public bool TypeOfParsing {
 140                         get { return handle_typeof; }
 141                         set { handle_typeof = value; }
 142                 }
 143
 144                 public XmlCommentState doc_state {
 145                         get { return xml_doc_state; }
 146                         set {
 147                                 if (value == XmlCommentState.Allowed) {
 148                                         check_incorrect_doc_comment ();
 149                                         reset_doc_comment ();
 150                                 }
 151                                 xml_doc_state = value;
 152                         }
 153                 }
 154
 155                 void AddEscapedIdentifier (LocatedToken lt)
 156                 {
 157                         if (escaped_identifiers == null)
 158                                 escaped_identifiers = new ArrayList ();
 159
 160                         escaped_identifiers.Add (lt);
 161                 }
 162
 163                 public bool IsEscapedIdentifier (Location loc)
 164                 {
 165                         if (escaped_identifiers != null) {
 166                                 foreach (LocatedToken lt in escaped_identifiers)
 167                                         if (lt.Location.Equals (loc))
 168                                                 return true;
 169                         }
 170
 171                         return false;
 172                 }
 173
 174                 //
 175                 // Class variables
 176                 //
 177                 static KeywordEntry[][] keywords;
 178                 static Hashtable keyword_strings;
 179                 static NumberStyles styles;
 180                 static NumberFormatInfo csharp_format_info;
 181
 182                 //
 183                 // Values for the associated token returned
 184                 //
 185                 internal int putback_char;      // Used by repl only
 186                 Object val;
 187
 188                 //
 189                 // Pre-processor
 190                 //
 191                 const int TAKING        = 1;
 192                 const int ELSE_SEEN     = 4;
 193                 const int PARENT_TAKING = 8;
 194                 const int REGION        = 16;
 195
 196                 //
 197                 // pre-processor if stack state:
 198                 //
 199                 Stack ifstack;
 200
 201                 static System.Text.StringBuilder string_builder;
 202
 203                 const int max_id_size = 512;
 204                 static char [] id_builder = new char [max_id_size];
 205
 206                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 207
 208                 const int max_number_size = 512;
 209                 static char [] number_builder = new char [max_number_size];
 210                 static int number_pos;
 211
 212                 //
 213                 // Details about the error encoutered by the tokenizer
 214                 //
 215                 string error_details;
 216
 217                 public string error {
 218                         get {
 219                                 return error_details;
 220                         }
 221                 }
 222
 223                 public int Line {
 224                         get {
 225                                 return ref_line;
 226                         }
 227                 }
 228
 229                 //
 230                 // This is used when the tokenizer needs to save
 231                 // the current position as it needs to do some parsing
 232                 // on its own to deamiguate a token in behalf of the
 233                 // parser.
 234                 //
 235                 Stack position_stack = new Stack (2);
 236                 class Position {
 237                         public int position;
 238                         public int line;
 239                         public int ref_line;
 240                         public int col;
 241                         public bool hidden;
 242                         public int putback_char;
 243                         public int previous_col;
 244                         public Stack ifstack;
 245                         public int parsing_generic_less_than;
 246                         public int current_token;
 247
 248                         public Position (Tokenizer t)
 249                         {
 250                                 position = t.reader.Position;
 251                                 line = t.line;
 252                                 ref_line = t.ref_line;
 253                                 col = t.col;
 254                                 hidden = t.hidden;
 255                                 putback_char = t.putback_char;
 256                                 previous_col = t.previous_col;
 257                                 if (t.ifstack != null && t.ifstack.Count != 0)
 258                                         ifstack = (Stack)t.ifstack.Clone ();
 259                                 parsing_generic_less_than = t.parsing_generic_less_than;
 260                                 current_token = t.current_token;
 261                         }
 262                 }
 263
 264                 public void PushPosition ()
 265                 {
 266                         position_stack.Push (new Position (this));
 267                 }
 268
 269                 public void PopPosition ()
 270                 {
 271                         Position p = (Position) position_stack.Pop ();
 272
 273                         reader.Position = p.position;
 274                         ref_line = p.ref_line;
 275                         line = p.line;
 276                         col = p.col;
 277                         hidden = p.hidden;
 278                         putback_char = p.putback_char;
 279                         previous_col = p.previous_col;
 280                         ifstack = p.ifstack;
 281                         parsing_generic_less_than = p.parsing_generic_less_than;
 282                         current_token = p.current_token;
 283                 }
 284
 285                 // Do not reset the position, ignore it.
 286                 public void DiscardPosition ()
 287                 {
 288                         position_stack.Pop ();
 289                 }
 290
 291                 static void AddKeyword (string kw, int token)
 292                 {
 293                         keyword_strings.Add (kw, kw);
 294
 295                         int length = kw.Length;
 296                         if (keywords [length] == null) {
 297                                 keywords [length] = new KeywordEntry ['z' - '_' + 1];
 298                         }
 299
 300                         int char_index = kw [0] - '_';
 301                         KeywordEntry kwe = keywords [length] [char_index];
 302                         if (kwe == null) {
 303                                 keywords [length] [char_index] = new KeywordEntry (kw, token);
 304                                 return;
 305                         }
 306
 307                         while (kwe.Next != null) {
 308                                 kwe = kwe.Next;
 309                         }
 310
 311                         kwe.Next = new KeywordEntry (kw, token);
 312                 }
 313
 314                 static void InitTokens ()
 315                 {
 316                         keyword_strings = new Hashtable ();
 317
 318                         // 11 is the length of the longest keyword for now
 319                         keywords = new KeywordEntry [11] [];
 320
 321                         AddKeyword ("__arglist", Token.ARGLIST);
 322                         AddKeyword ("abstract", Token.ABSTRACT);
 323                         AddKeyword ("as", Token.AS);
 324                         AddKeyword ("add", Token.ADD);
 325                         AddKeyword ("base", Token.BASE);
 326                         AddKeyword ("bool", Token.BOOL);
 327                         AddKeyword ("break", Token.BREAK);
 328                         AddKeyword ("byte", Token.BYTE);
 329                         AddKeyword ("case", Token.CASE);
 330                         AddKeyword ("catch", Token.CATCH);
 331                         AddKeyword ("char", Token.CHAR);
 332                         AddKeyword ("checked", Token.CHECKED);
 333                         AddKeyword ("class", Token.CLASS);
 334                         AddKeyword ("const", Token.CONST);
 335                         AddKeyword ("continue", Token.CONTINUE);
 336                         AddKeyword ("decimal", Token.DECIMAL);
 337                         AddKeyword ("default", Token.DEFAULT);
 338                         AddKeyword ("delegate", Token.DELEGATE);
 339                         AddKeyword ("do", Token.DO);
 340                         AddKeyword ("double", Token.DOUBLE);
 341                         AddKeyword ("else", Token.ELSE);
 342                         AddKeyword ("enum", Token.ENUM);
 343                         AddKeyword ("event", Token.EVENT);
 344                         AddKeyword ("explicit", Token.EXPLICIT);
 345                         AddKeyword ("extern", Token.EXTERN);
 346                         AddKeyword ("false", Token.FALSE);
 347                         AddKeyword ("finally", Token.FINALLY);
 348                         AddKeyword ("fixed", Token.FIXED);
 349                         AddKeyword ("float", Token.FLOAT);
 350                         AddKeyword ("for", Token.FOR);
 351                         AddKeyword ("foreach", Token.FOREACH);
 352                         AddKeyword ("goto", Token.GOTO);
 353                         AddKeyword ("get", Token.GET);
 354                         AddKeyword ("if", Token.IF);
 355                         AddKeyword ("implicit", Token.IMPLICIT);
 356                         AddKeyword ("in", Token.IN);
 357                         AddKeyword ("int", Token.INT);
 358                         AddKeyword ("interface", Token.INTERFACE);
 359                         AddKeyword ("internal", Token.INTERNAL);
 360                         AddKeyword ("is", Token.IS);
 361                         AddKeyword ("lock", Token.LOCK);
 362                         AddKeyword ("long", Token.LONG);
 363                         AddKeyword ("namespace", Token.NAMESPACE);
 364                         AddKeyword ("new", Token.NEW);
 365                         AddKeyword ("null", Token.NULL);
 366                         AddKeyword ("object", Token.OBJECT);
 367                         AddKeyword ("operator", Token.OPERATOR);
 368                         AddKeyword ("out", Token.OUT);
 369                         AddKeyword ("override", Token.OVERRIDE);
 370                         AddKeyword ("params", Token.PARAMS);
 371                         AddKeyword ("private", Token.PRIVATE);
 372                         AddKeyword ("protected", Token.PROTECTED);
 373                         AddKeyword ("public", Token.PUBLIC);
 374                         AddKeyword ("readonly", Token.READONLY);
 375                         AddKeyword ("ref", Token.REF);
 376                         AddKeyword ("remove", Token.REMOVE);
 377                         AddKeyword ("return", Token.RETURN);
 378                         AddKeyword ("sbyte", Token.SBYTE);
 379                         AddKeyword ("sealed", Token.SEALED);
 380                         AddKeyword ("set", Token.SET);
 381                         AddKeyword ("short", Token.SHORT);
 382                         AddKeyword ("sizeof", Token.SIZEOF);
 383                         AddKeyword ("stackalloc", Token.STACKALLOC);
 384                         AddKeyword ("static", Token.STATIC);
 385                         AddKeyword ("string", Token.STRING);
 386                         AddKeyword ("struct", Token.STRUCT);
 387                         AddKeyword ("switch", Token.SWITCH);
 388                         AddKeyword ("this", Token.THIS);
 389                         AddKeyword ("throw", Token.THROW);
 390                         AddKeyword ("true", Token.TRUE);
 391                         AddKeyword ("try", Token.TRY);
 392                         AddKeyword ("typeof", Token.TYPEOF);
 393                         AddKeyword ("uint", Token.UINT);
 394                         AddKeyword ("ulong", Token.ULONG);
 395                         AddKeyword ("unchecked", Token.UNCHECKED);
 396                         AddKeyword ("unsafe", Token.UNSAFE);
 397                         AddKeyword ("ushort", Token.USHORT);
 398                         AddKeyword ("using", Token.USING);
 399                         AddKeyword ("virtual", Token.VIRTUAL);
 400                         AddKeyword ("void", Token.VOID);
 401                         AddKeyword ("volatile", Token.VOLATILE);
 402                         AddKeyword ("while", Token.WHILE);
 403                         AddKeyword ("partial", Token.PARTIAL);
 404                         AddKeyword ("where", Token.WHERE);
 405
 406                         // LINQ keywords
 407                         AddKeyword ("from", Token.FROM);
 408                         AddKeyword ("join", Token.JOIN);
 409                         AddKeyword ("on", Token.ON);
 410                         AddKeyword ("equals", Token.EQUALS);
 411                         AddKeyword ("select", Token.SELECT);
 412                         AddKeyword ("group", Token.GROUP);
 413                         AddKeyword ("by", Token.BY);
 414                         AddKeyword ("let", Token.LET);
 415                         AddKeyword ("orderby", Token.ORDERBY);
 416                         AddKeyword ("ascending", Token.ASCENDING);
 417                         AddKeyword ("descending", Token.DESCENDING);
 418                         AddKeyword ("into", Token.INTO);
 419                 }
 420
 421                 //
 422                 // Class initializer
 423                 //
 424                 static Tokenizer ()
 425                 {
 426                         InitTokens ();
 427                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 428                         styles = NumberStyles.Float;
 429
 430                         string_builder = new System.Text.StringBuilder ();
 431                 }
 432
 433                 int GetKeyword (char[] id, int id_len)
 434                 {
 435                         //
 436                         // Keywords are stored in an array of arrays grouped by their
 437                         // length and then by the first character
 438                         //
 439                         if (id_len >= keywords.Length || keywords [id_len] == null)
 440                                 return -1;
 441
 442                         int first_index = id [0] - '_';
 443                         if (first_index > 'z')
 444                                 return -1;
 445
 446                         KeywordEntry kwe = keywords [id_len] [first_index];
 447                         if (kwe == null)
 448                                 return -1;
 449
 450                         int res;
 451                         do {
 452                                 res = kwe.Token;
 453                                 for (int i = 1; i < id_len; ++i) {
 454                                         if (id [i] != kwe.Value [i]) {
 455                                                 res = 0;
 456                                                 break;
 457                                         }
 458                                 }
 459                                 kwe = kwe.Next;
 460                         } while (kwe != null && res == 0);
 461
 462                         if (res == 0)
 463                                 return -1;
 464
 465                         int next_token;
 466                         switch (res) {
 467                         case Token.GET:
 468                         case Token.SET:
 469                                 if (!handle_get_set)
 470                                         res = -1;
 471                                 break;
 472                         case Token.REMOVE:
 473                         case Token.ADD:
 474                                 if (!handle_remove_add)
 475                                         res = -1;
 476                                 break;
 477                         case Token.EXTERN:
 478                                 if (parsing_declaration == 0)
 479                                         res = Token.EXTERN_ALIAS;
 480                                 break;
 481                         case Token.DEFAULT:
 482                                 if (peek_token () == Token.COLON) {
 483                                         token ();
 484                                         res = Token.DEFAULT_COLON;
 485                                 }
 486                                 break;
 487                         case Token.WHERE:
 488                                 if (!handle_where && !query_parsing)
 489                                         res = -1;
 490                                 break;
 491                         case Token.FROM:
 492                                 //
 493                                 // A query expression is any expression that starts with `from identifier'
 494                                 // followed by any token except ; , =
 495                                 //
 496                                 if (!query_parsing) {
 497                                         if (lambda_arguments_parsing) {
 498                                                 res = -1;
 499                                                 break;
 500                                         }
 501
 502                                         PushPosition ();
 503                                         // HACK: to disable generics micro-parser, because PushPosition does not
 504                                         // store identifiers array
 505                                         parsing_generic_less_than = 1;
 506                                         switch (xtoken ()) {
 507                                         case Token.IDENTIFIER:
 508                                         case Token.INT:
 509                                         case Token.BOOL:
 510                                         case Token.BYTE:
 511                                         case Token.CHAR:
 512                                         case Token.DECIMAL:
 513                                         case Token.FLOAT:
 514                                         case Token.LONG:
 515                                         case Token.OBJECT:
 516                                         case Token.STRING:
 517                                         case Token.UINT:
 518                                         case Token.ULONG:
 519                                                 next_token = xtoken ();
 520                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 521                                                         goto default;
 522
 523                                                 res = Token.FROM_FIRST;
 524                                                 query_parsing = true;
 525                                                 if (RootContext.Version <= LanguageVersion.ISO_2)
 526                                                         Report.FeatureIsNotAvailable (Location, "query expressions");
 527                                                 break;
 528                                         case Token.VOID:
 529                                                 Expression.Error_VoidInvalidInTheContext (Location);
 530                                                 break;
 531                                         default:
 532                                                 PopPosition ();
 533                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 534                                                 // which has been overwritten before we grabbed the identifier
 535                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 536                                                 return -1;
 537                                         }
 538                                         PopPosition ();
 539                                 }
 540                                 break;
 541                         case Token.JOIN:
 542                         case Token.ON:
 543                         case Token.EQUALS:
 544                         case Token.SELECT:
 545                         case Token.GROUP:
 546                         case Token.BY:
 547                         case Token.LET:
 548                         case Token.ORDERBY:
 549                         case Token.ASCENDING:
 550                         case Token.DESCENDING:
 551                         case Token.INTO:
 552                                 if (!query_parsing)
 553                                         res = -1;
 554                                 break;
 555
 556                         case Token.USING:
 557                         case Token.NAMESPACE:
 558                                 // TODO: some explanation needed
 559                                 check_incorrect_doc_comment ();
 560                                 break;
 561
 562                         case Token.PARTIAL:
 563                                 if (parsing_block > 0) {
 564                                         res = -1;
 565                                         break;
 566                                 }
 567
 568                                 // Save current position and parse next token.
 569                                 PushPosition ();
 570
 571                                 next_token = token ();
 572                                 bool ok = (next_token == Token.CLASS) ||
 573                                         (next_token == Token.STRUCT) ||
 574                                         (next_token == Token.INTERFACE) ||
 575                                         (next_token == Token.VOID);
 576
 577                                 PopPosition ();
 578
 579                                 if (ok) {
 580                                         if (next_token == Token.VOID) {
 581                                                 if (RootContext.Version == LanguageVersion.ISO_1 ||
 582                                                     RootContext.Version == LanguageVersion.ISO_2)
 583                                                         Report.FeatureIsNotAvailable (Location, "partial methods");
 584                                         } else if (RootContext.Version == LanguageVersion.ISO_1)
 585                                                 Report.FeatureIsNotAvailable (Location, "partial types");
 586
 587                                         return res;
 588                                 }
 589
 590                                 if (next_token < Token.LAST_KEYWORD) {
 591                                         Report.Error (267, Location,
 592                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 593                                         return token ();
 594                                 }
 595
 596                                 res = -1;
 597                                 break;
 598                         }
 599
 600                         return res;
 601                 }
 602
 603                 public Location Location {
 604                         get {
 605                                 return new Location (ref_line, hidden ? -1 : col);
 606                         }
 607                 }
 608
 609                 public Tokenizer (SeekableStreamReader input, CompilationUnit file)
 610                 {
 611                         this.ref_name = file;
 612                         this.file_name = file;
 613                         reader = input;
 614
 615                         putback_char = -1;
 616
 617                         xml_comment_buffer = new StringBuilder ();
 618
 619                         //
 620                         // FIXME: This could be `Location.Push' but we have to
 621                         // find out why the MS compiler allows this
 622                         //
 623                         Mono.CSharp.Location.Push (file, file);
 624                 }
 625
 626                 static bool is_identifier_start_character (int c)
 627                 {
 628                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 629                 }
 630
 631                 static bool is_identifier_part_character (char c)
 632                 {
 633                         if (c >= 'a' && c <= 'z')
 634                                 return true;
 635
 636                         if (c >= 'A' && c <= 'Z')
 637                                 return true;
 638
 639                         if (c == '_' || (c >= '0' && c <= '9'))
 640                                 return true;
 641
 642                         if (c < 0x80)
 643                                 return false;
 644
 645                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 646                 }
 647
 648                 public static bool IsKeyword (string s)
 649                 {
 650                         return keyword_strings [s] != null;
 651                 }
 652
 653                 //
 654                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 655                 //
 656
 657                 int TokenizeOpenParens ()
 658                 {
 659                         int ptoken;
 660                         current_token = -1;
 661
 662                         int bracket_level = 0;
 663                         bool is_type = false;
 664                         bool can_be_type = false;
 665
 666                         while (true) {
 667                                 ptoken = current_token;
 668                                 token ();
 669
 670                                 switch (current_token) {
 671                                 case Token.CLOSE_PARENS:
 672                                         token ();
 673
 674                                         //
 675                                         // Expression inside parens is lambda, (int i) =>
 676                                         //
 677                                         if (current_token == Token.ARROW) {
 678                                                 if (RootContext.Version <= LanguageVersion.ISO_2)
 679                                                         Report.FeatureIsNotAvailable (Location, "lambda expressions");
 680
 681                                                 return Token.OPEN_PARENS_LAMBDA;
 682                                         }
 683
 684                                         //
 685                                         // Expression inside parens is single type, (int[])
 686                                         //
 687                                         if (is_type)
 688                                                 return Token.OPEN_PARENS_CAST;
 689
 690                                         //
 691                                         // Expression is possible cast, look at next token, (T)null
 692                                         //
 693                                         if (can_be_type) {
 694                                                 switch (current_token) {
 695                                                 case Token.OPEN_PARENS:
 696                                                 case Token.BANG:
 697                                                 case Token.TILDE:
 698                                                 case Token.IDENTIFIER:
 699                                                 case Token.LITERAL_INTEGER:
 700                                                 case Token.LITERAL_FLOAT:
 701                                                 case Token.LITERAL_DOUBLE:
 702                                                 case Token.LITERAL_DECIMAL:
 703                                                 case Token.LITERAL_CHARACTER:
 704                                                 case Token.LITERAL_STRING:
 705                                                 case Token.BASE:
 706                                                 case Token.CHECKED:
 707                                                 case Token.DELEGATE:
 708                                                 case Token.FALSE:
 709                                                 case Token.FIXED:
 710                                                 case Token.NEW:
 711                                                 case Token.NULL:
 712                                                 case Token.SIZEOF:
 713                                                 case Token.THIS:
 714                                                 case Token.THROW:
 715                                                 case Token.TRUE:
 716                                                 case Token.TYPEOF:
 717                                                 case Token.UNCHECKED:
 718                                                 case Token.UNSAFE:
 719                                                 case Token.DEFAULT:
 720
 721                                                 //
 722                                                 // These can be part of a member access
 723                                                 //
 724                                                 case Token.INT:
 725                                                 case Token.UINT:
 726                                                 case Token.SHORT:
 727                                                 case Token.USHORT:
 728                                                 case Token.LONG:
 729                                                 case Token.ULONG:
 730                                                 case Token.DOUBLE:
 731                                                 case Token.FLOAT:
 732                                                 case Token.CHAR:
 733                                                 case Token.BYTE:
 734                                                 case Token.DECIMAL:
 735                                                 case Token.BOOL:
 736                                                         return Token.OPEN_PARENS_CAST;
 737                                                 }
 738                                         }
 739                                         return Token.OPEN_PARENS;
 740
 741                                 case Token.DOT:
 742                                 case Token.DOUBLE_COLON:
 743                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
 744                                                 goto default;
 745
 746                                         continue;
 747
 748                                 case Token.IDENTIFIER:
 749                                         switch (ptoken) {
 750                                         case Token.DOT:
 751                                         case Token.OP_GENERICS_LT:
 752                                         case Token.COMMA:
 753                                         case Token.DOUBLE_COLON:
 754                                         case -1:
 755                                                 if (bracket_level == 0)
 756                                                         can_be_type = true;
 757                                                 continue;
 758                                         default:
 759                                                 can_be_type = is_type = false;
 760                                                 continue;
 761                                         }
 762
 763                                 case Token.OBJECT:
 764                                 case Token.STRING:
 765                                 case Token.BOOL:
 766                                 case Token.DECIMAL:
 767                                 case Token.FLOAT:
 768                                 case Token.DOUBLE:
 769                                 case Token.SBYTE:
 770                                 case Token.BYTE:
 771                                 case Token.SHORT:
 772                                 case Token.USHORT:
 773                                 case Token.INT:
 774                                 case Token.UINT:
 775                                 case Token.LONG:
 776                                 case Token.ULONG:
 777                                 case Token.CHAR:
 778                                 case Token.VOID:
 779                                         if (bracket_level == 0)
 780                                                 is_type = true;
 781                                         continue;
 782
 783                                 case Token.COMMA:
 784                                         if (bracket_level == 0) {
 785                                                 bracket_level = 100;
 786                                                 can_be_type = is_type = false;
 787                                         }
 788                                         continue;
 789
 790                                 case Token.OP_GENERICS_LT:
 791                                 case Token.OPEN_BRACKET:
 792                                         if (bracket_level++ == 0)
 793                                                 is_type = true;
 794                                         continue;
 795
 796                                 case Token.OP_GENERICS_GT:
 797                                 case Token.CLOSE_BRACKET:
 798                                         --bracket_level;
 799                                         continue;
 800
 801                                 case Token.INTERR_NULLABLE:
 802                                 case Token.STAR:
 803                                         if (bracket_level == 0)
 804                                                 is_type = true;
 805                                         continue;
 806
 807                                 case Token.REF:
 808                                 case Token.OUT:
 809                                         can_be_type = is_type = false;
 810                                         continue;
 811
 812                                 default:
 813                                         return Token.OPEN_PARENS;
 814                                 }
 815                         }
 816                 }
 817
 818                 public static bool IsValidIdentifier (string s)
 819                 {
 820                         if (s == null || s.Length == 0)
 821                                 return false;
 822
 823                         if (!is_identifier_start_character (s [0]))
 824                                 return false;
 825
 826                         for (int i = 1; i < s.Length; i ++)
 827                                 if (! is_identifier_part_character (s [i]))
 828                                         return false;
 829
 830                         return true;
 831                 }
 832
 833                 bool parse_less_than ()
 834                 {
 835                 start:
 836                         int the_token = token ();
 837                         if (the_token == Token.OPEN_BRACKET) {
 838                                 do {
 839                                         the_token = token ();
 840                                 } while (the_token != Token.CLOSE_BRACKET);
 841                                 the_token = token ();
 842                         } else if (the_token == Token.IN || the_token == Token.OUT) {
 843                                 the_token = token ();
 844                         }
 845                         switch (the_token) {
 846                         case Token.IDENTIFIER:
 847                         case Token.OBJECT:
 848                         case Token.STRING:
 849                         case Token.BOOL:
 850                         case Token.DECIMAL:
 851                         case Token.FLOAT:
 852                         case Token.DOUBLE:
 853                         case Token.SBYTE:
 854                         case Token.BYTE:
 855                         case Token.SHORT:
 856                         case Token.USHORT:
 857                         case Token.INT:
 858                         case Token.UINT:
 859                         case Token.LONG:
 860                         case Token.ULONG:
 861                         case Token.CHAR:
 862                         case Token.VOID:
 863                                 break;
 864                         case Token.OP_GENERICS_GT:
 865                                 return true;
 866
 867                         default:
 868                                 return false;
 869                         }
 870                 again:
 871                         the_token = token ();
 872
 873                         if (the_token == Token.OP_GENERICS_GT)
 874                                 return true;
 875                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
 876                                 goto start;
 877                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
 878                                 goto again;
 879                         else if (the_token == Token.OP_GENERICS_LT) {
 880                                 if (!parse_less_than ())
 881                                         return false;
 882                                 goto again;
 883                         } else if (the_token == Token.OPEN_BRACKET) {
 884                         rank_specifiers:
 885                                 the_token = token ();
 886                                 if (the_token == Token.CLOSE_BRACKET)
 887                                         goto again;
 888                                 else if (the_token == Token.COMMA)
 889                                         goto rank_specifiers;
 890                                 return false;
 891                         }
 892
 893                         return false;
 894                 }
 895
 896                 bool parse_generic_dimension (out int dimension)
 897                 {
 898                         dimension = 1;
 899
 900                 again:
 901                         int the_token = token ();
 902                         if (the_token == Token.OP_GENERICS_GT)
 903                                 return true;
 904                         else if (the_token == Token.COMMA) {
 905                                 dimension++;
 906                                 goto again;
 907                         }
 908
 909                         return false;
 910                 }
 911
 912                 public int peek_token ()
 913                 {
 914                         int the_token;
 915
 916                         PushPosition ();
 917                         the_token = token ();
 918                         PopPosition ();
 919
 920                         return the_token;
 921                 }
 922
 923                 //
 924                 // Tonizes `?' using custom disambiguous rules to return one
 925                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
 926                 //
 927                 // Tricky expression look like:
 928                 //
 929                 // Foo ? a = x ? b : c;
 930                 //
 931                 int TokenizePossibleNullableType ()
 932                 {
 933                         if (parsing_block == 0 || parsing_type > 0)
 934                                 return Token.INTERR_NULLABLE;
 935
 936                         int d = peek_char ();
 937                         if (d == '?') {
 938                                 get_char ();
 939                                 return Token.OP_COALESCING;
 940                         }
 941
 942                         switch (current_token) {
 943                         case Token.CLOSE_PARENS:
 944                         case Token.TRUE:
 945                         case Token.FALSE:
 946                         case Token.NULL:
 947                         case Token.LITERAL_INTEGER:
 948                         case Token.LITERAL_STRING:
 949                                 return Token.INTERR;
 950                         }
 951
 952                         if (d != ' ') {
 953                                 if (d == ',' || d == ';' || d == '>')
 954                                         return Token.INTERR_NULLABLE;
 955                                 if (d == '*' || (d >= '0' && d <= '9'))
 956                                         return Token.INTERR;
 957                         }
 958
 959                         PushPosition ();
 960                         current_token = Token.NONE;
 961                         int next_token;
 962                         switch (xtoken ()) {
 963                         case Token.LITERAL_INTEGER:
 964                         case Token.LITERAL_STRING:
 965                         case Token.LITERAL_CHARACTER:
 966                         case Token.LITERAL_DECIMAL:
 967                         case Token.LITERAL_DOUBLE:
 968                         case Token.LITERAL_FLOAT:
 969                         case Token.TRUE:
 970                         case Token.FALSE:
 971                         case Token.NULL:
 972                         case Token.THIS:
 973                         case Token.NEW:
 974                                 next_token = Token.INTERR;
 975                                 break;
 976
 977                         case Token.SEMICOLON:
 978                         case Token.COMMA:
 979                         case Token.CLOSE_PARENS:
 980                         case Token.OPEN_BRACKET:
 981                         case Token.OP_GENERICS_GT:
 982                                 next_token = Token.INTERR_NULLABLE;
 983                                 break;
 984
 985                         default:
 986                                 next_token = -1;
 987                                 break;
 988                         }
 989
 990                         if (next_token == -1) {
 991                                 switch (xtoken ()) {
 992                                 case Token.COMMA:
 993                                 case Token.SEMICOLON:
 994                                 case Token.OPEN_BRACE:
 995                                 case Token.CLOSE_PARENS:
 996                                 case Token.IN:
 997                                         next_token = Token.INTERR_NULLABLE;
 998                                         break;
 999
1000                                 case Token.COLON:
1001                                         next_token = Token.INTERR;
1002                                         break;
1003
1004                                 default:
1005                                         int ntoken;
1006                                         int interrs = 1;
1007                                         int colons = 0;
1008                                         //
1009                                         // All shorcuts failed, do it hard way
1010                                         //
1011                                         while ((ntoken = xtoken ()) != Token.EOF) {
1012                                                 if (ntoken == Token.SEMICOLON)
1013                                                         break;
1014
1015                                                 if (ntoken == Token.COLON) {
1016                                                         if (++colons == interrs)
1017                                                                 break;
1018                                                         continue;
1019                                                 }
1020
1021                                                 if (ntoken == Token.INTERR) {
1022                                                         ++interrs;
1023                                                         continue;
1024                                                 }
1025                                         }
1026
1027                                         next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1028                                         break;
1029                                 }
1030                         }
1031
1032                         PopPosition ();
1033                         return next_token;
1034                 }
1035
1036                 bool decimal_digits (int c)
1037                 {
1038                         int d;
1039                         bool seen_digits = false;
1040
1041                         if (c != -1){
1042                                 if (number_pos == max_number_size)
1043                                         Error_NumericConstantTooLong ();
1044                                 number_builder [number_pos++] = (char) c;
1045                         }
1046
1047                         //
1048                         // We use peek_char2, because decimal_digits needs to do a
1049                         // 2-character look-ahead (5.ToString for example).
1050                         //
1051                         while ((d = peek_char2 ()) != -1){
1052                                 if (d >= '0' && d <= '9'){
1053                                         if (number_pos == max_number_size)
1054                                                 Error_NumericConstantTooLong ();
1055                                         number_builder [number_pos++] = (char) d;
1056                                         get_char ();
1057                                         seen_digits = true;
1058                                 } else
1059                                         break;
1060                         }
1061
1062                         return seen_digits;
1063                 }
1064
1065                 static bool is_hex (int e)
1066                 {
1067                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1068                 }
1069
1070                 static int real_type_suffix (int c)
1071                 {
1072                         int t;
1073
1074                         switch (c){
1075                         case 'F': case 'f':
1076                                 t =  Token.LITERAL_FLOAT;
1077                                 break;
1078                         case 'D': case 'd':
1079                                 t = Token.LITERAL_DOUBLE;
1080                                 break;
1081                         case 'M': case 'm':
1082                                  t= Token.LITERAL_DECIMAL;
1083                                 break;
1084                         default:
1085                                 return Token.NONE;
1086                         }
1087                         return t;
1088                 }
1089
1090                 int integer_type_suffix (ulong ul, int c)
1091                 {
1092                         bool is_unsigned = false;
1093                         bool is_long = false;
1094
1095                         if (c != -1){
1096                                 bool scanning = true;
1097                                 do {
1098                                         switch (c){
1099                                         case 'U': case 'u':
1100                                                 if (is_unsigned)
1101                                                         scanning = false;
1102                                                 is_unsigned = true;
1103                                                 get_char ();
1104                                                 break;
1105
1106                                         case 'l':
1107                                                 if (!is_unsigned){
1108                                                         //
1109                                                         // if we have not seen anything in between
1110                                                         // report this error
1111                                                         //
1112                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1113                                                 }
1114                                                 //
1115                                                 // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
1116                                                 // work around that.
1117                                                 //
1118                                                 //goto case 'L';
1119                                                 if (is_long)
1120                                                         scanning = false;
1121                                                 is_long = true;
1122                                                 get_char ();
1123                                                 break;
1124
1125                                         case 'L':
1126                                                 if (is_long)
1127                                                         scanning = false;
1128                                                 is_long = true;
1129                                                 get_char ();
1130                                                 break;
1131
1132                                         default:
1133                                                 scanning = false;
1134                                                 break;
1135                                         }
1136                                         c = peek_char ();
1137                                 } while (scanning);
1138                         }
1139
1140                         if (is_long && is_unsigned){
1141                                 val = ul;
1142                                 return Token.LITERAL_INTEGER;
1143                         } else if (is_unsigned){
1144                                 // uint if possible, or ulong else.
1145
1146                                 if ((ul & 0xffffffff00000000) == 0)
1147                                         val = (uint) ul;
1148                                 else
1149                                         val = ul;
1150                         } else if (is_long){
1151                                 // long if possible, ulong otherwise
1152                                 if ((ul & 0x8000000000000000) != 0)
1153                                         val = ul;
1154                                 else
1155                                         val = (long) ul;
1156                         } else {
1157                                 // int, uint, long or ulong in that order
1158                                 if ((ul & 0xffffffff00000000) == 0){
1159                                         uint ui = (uint) ul;
1160
1161                                         if ((ui & 0x80000000) != 0)
1162                                                 val = ui;
1163                                         else
1164                                                 val = (int) ui;
1165                                 } else {
1166                                         if ((ul & 0x8000000000000000) != 0)
1167                                                 val = ul;
1168                                         else
1169                                                 val = (long) ul;
1170                                 }
1171                         }
1172                         return Token.LITERAL_INTEGER;
1173                 }
1174
1175                 //
1176                 // given `c' as the next char in the input decide whether
1177                 // we need to convert to a special type, and then choose
1178                 // the best representation for the integer
1179                 //
1180                 int adjust_int (int c)
1181                 {
1182                         try {
1183                                 if (number_pos > 9){
1184                                         ulong ul = (uint) (number_builder [0] - '0');
1185
1186                                         for (int i = 1; i < number_pos; i++){
1187                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1188                                         }
1189                                         return integer_type_suffix (ul, c);
1190                                 } else {
1191                                         uint ui = (uint) (number_builder [0] - '0');
1192
1193                                         for (int i = 1; i < number_pos; i++){
1194                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1195                                         }
1196                                         return integer_type_suffix (ui, c);
1197                                 }
1198                         } catch (OverflowException) {
1199                                 error_details = "Integral constant is too large";
1200                                 Report.Error (1021, Location, error_details);
1201                                 val = 0ul;
1202                                 return Token.LITERAL_INTEGER;
1203                         }
1204                         catch (FormatException) {
1205                                 Report.Error (1013, Location, "Invalid number");
1206                                 val = 0ul;
1207                                 return Token.LITERAL_INTEGER;
1208                         }
1209                 }
1210
1211                 int adjust_real (int t)
1212                 {
1213                         string s = new String (number_builder, 0, number_pos);
1214                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1215
1216                         switch (t){
1217                         case Token.LITERAL_DECIMAL:
1218                                 try {
1219                                         val = System.Decimal.Parse (s, styles, csharp_format_info);
1220                                 } catch (OverflowException) {
1221                                         val = 0m;
1222                                         Report.Error (594, Location, error_details, "decimal");
1223                                 }
1224                                 break;
1225                         case Token.LITERAL_FLOAT:
1226                                 try {
1227                                         val = float.Parse (s, styles, csharp_format_info);
1228                                 } catch (OverflowException) {
1229                                         val = 0.0f;
1230                                         Report.Error (594, Location, error_details, "float");
1231                                 }
1232                                 break;
1233
1234                         case Token.LITERAL_DOUBLE:
1235                         case Token.NONE:
1236                                 t = Token.LITERAL_DOUBLE;
1237                                 try {
1238                                         val = System.Double.Parse (s, styles, csharp_format_info);
1239                                 } catch (OverflowException) {
1240                                         val = 0.0;
1241                                         Report.Error (594, Location, error_details, "double");
1242                                 }
1243                                 break;
1244                         }
1245                         return t;
1246                 }
1247
1248                 int handle_hex ()
1249                 {
1250                         int d;
1251                         ulong ul;
1252
1253                         get_char ();
1254                         while ((d = peek_char ()) != -1){
1255                                 if (is_hex (d)){
1256                                         number_builder [number_pos++] = (char) d;
1257                                         get_char ();
1258                                 } else
1259                                         break;
1260                         }
1261
1262                         string s = new String (number_builder, 0, number_pos);
1263                         try {
1264                                 if (number_pos <= 8)
1265                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1266                                 else
1267                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1268                         } catch (OverflowException){
1269                                 error_details = "Integral constant is too large";
1270                                 Report.Error (1021, Location, error_details);
1271                                 val = 0ul;
1272                                 return Token.LITERAL_INTEGER;
1273                         }
1274                         catch (FormatException) {
1275                                 Report.Error (1013, Location, "Invalid number");
1276                                 val = 0ul;
1277                                 return Token.LITERAL_INTEGER;
1278                         }
1279
1280                         return integer_type_suffix (ul, peek_char ());
1281                 }
1282
1283                 //
1284                 // Invoked if we know we have .digits or digits
1285                 //
1286                 int is_number (int c)
1287                 {
1288                         bool is_real = false;
1289                         int type;
1290
1291                         number_pos = 0;
1292
1293                         if (c >= '0' && c <= '9'){
1294                                 if (c == '0'){
1295                                         int peek = peek_char ();
1296
1297                                         if (peek == 'x' || peek == 'X')
1298                                                 return handle_hex ();
1299                                 }
1300                                 decimal_digits (c);
1301                                 c = get_char ();
1302                         }
1303
1304                         //
1305                         // We need to handle the case of
1306                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1307                         //
1308                         if (c == '.'){
1309                                 if (decimal_digits ('.')){
1310                                         is_real = true;
1311                                         c = get_char ();
1312                                 } else {
1313                                         putback ('.');
1314                                         number_pos--;
1315                                         return adjust_int (-1);
1316                                 }
1317                         }
1318
1319                         if (c == 'e' || c == 'E'){
1320                                 is_real = true;
1321                                 if (number_pos == max_number_size)
1322                                         Error_NumericConstantTooLong ();
1323                                 number_builder [number_pos++] = 'e';
1324                                 c = get_char ();
1325
1326                                 if (c == '+'){
1327                                         if (number_pos == max_number_size)
1328                                                 Error_NumericConstantTooLong ();
1329                                         number_builder [number_pos++] = '+';
1330                                         c = -1;
1331                                 } else if (c == '-') {
1332                                         if (number_pos == max_number_size)
1333                                                 Error_NumericConstantTooLong ();
1334                                         number_builder [number_pos++] = '-';
1335                                         c = -1;
1336                                 } else {
1337                                         if (number_pos == max_number_size)
1338                                                 Error_NumericConstantTooLong ();
1339                                         number_builder [number_pos++] = '+';
1340                                 }
1341
1342                                 decimal_digits (c);
1343                                 c = get_char ();
1344                         }
1345
1346                         type = real_type_suffix (c);
1347                         if (type == Token.NONE && !is_real){
1348                                 putback (c);
1349                                 return adjust_int (c);
1350                         } else
1351                                 is_real = true;
1352
1353                         if (type == Token.NONE){
1354                                 putback (c);
1355                         }
1356
1357                         if (is_real)
1358                                 return adjust_real (type);
1359
1360                         Console.WriteLine ("This should not be reached");
1361                         throw new Exception ("Is Number should never reach this point");
1362                 }
1363
1364                 //
1365                 // Accepts exactly count (4 or 8) hex, no more no less
1366                 //
1367                 int getHex (int count, out int surrogate, out bool error)
1368                 {
1369                         int i;
1370                         int total = 0;
1371                         int c;
1372                         int top = count != -1 ? count : 4;
1373
1374                         get_char ();
1375                         error = false;
1376                         surrogate = 0;
1377                         for (i = 0; i < top; i++){
1378                                 c = get_char ();
1379
1380                                 if (c >= '0' && c <= '9')
1381                                         c = (int) c - (int) '0';
1382                                 else if (c >= 'A' && c <= 'F')
1383                                         c = (int) c - (int) 'A' + 10;
1384                                 else if (c >= 'a' && c <= 'f')
1385                                         c = (int) c - (int) 'a' + 10;
1386                                 else {
1387                                         error = true;
1388                                         return 0;
1389                                 }
1390
1391                                 total = (total * 16) + c;
1392                                 if (count == -1){
1393                                         int p = peek_char ();
1394                                         if (p == -1)
1395                                                 break;
1396                                         if (!is_hex ((char)p))
1397                                                 break;
1398                                 }
1399                         }
1400
1401                         if (top == 8) {
1402                                 if (total > 0x0010FFFF) {
1403                                         error = true;
1404                                         return 0;
1405                                 }
1406
1407                                 if (total >= 0x00010000) {
1408                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1409                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1410                                 }
1411                         }
1412
1413                         return total;
1414                 }
1415
1416                 int escape (int c, out int surrogate)
1417                 {
1418                         bool error;
1419                         int d;
1420                         int v;
1421
1422                         d = peek_char ();
1423                         if (c != '\\') {
1424                                 surrogate = 0;
1425                                 return c;
1426                         }
1427
1428                         switch (d){
1429                         case 'a':
1430                                 v = '\a'; break;
1431                         case 'b':
1432                                 v = '\b'; break;
1433                         case 'n':
1434                                 v = '\n'; break;
1435                         case 't':
1436                                 v = '\t'; break;
1437                         case 'v':
1438                                 v = '\v'; break;
1439                         case 'r':
1440                                 v = '\r'; break;
1441                         case '\\':
1442                                 v = '\\'; break;
1443                         case 'f':
1444                                 v = '\f'; break;
1445                         case '0':
1446                                 v = 0; break;
1447                         case '"':
1448                                 v = '"'; break;
1449                         case '\'':
1450                                 v = '\''; break;
1451                         case 'x':
1452                                 v = getHex (-1, out surrogate, out error);
1453                                 if (error)
1454                                         goto default;
1455                                 return v;
1456                         case 'u':
1457                         case 'U':
1458                                 return EscapeUnicode (d, out surrogate);
1459                         default:
1460                                 surrogate = 0;
1461                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1462                                 return d;
1463                         }
1464
1465                         get_char ();
1466                         surrogate = 0;
1467                         return v;
1468                 }
1469
1470                 int EscapeUnicode (int ch, out int surrogate)
1471                 {
1472                         bool error;
1473                         if (ch == 'U') {
1474                                 ch = getHex (8, out surrogate, out error);
1475                         } else {
1476                                 ch = getHex (4, out surrogate, out error);
1477                         }
1478
1479                         if (error)
1480                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1481
1482                         return ch;
1483                 }
1484
1485                 int get_char ()
1486                 {
1487                         int x;
1488                         if (putback_char != -1) {
1489                                 x = putback_char;
1490                                 putback_char = -1;
1491                         } else
1492                                 x = reader.Read ();
1493                         if (x == '\n') {
1494                                 advance_line ();
1495                         } else {
1496                                 col++;
1497                         }
1498                         return x;
1499                 }
1500
1501                 void advance_line ()
1502                 {
1503                         line++;
1504                         ref_line++;
1505                         previous_col = col;
1506                         col = 0;
1507                 }
1508
1509                 int peek_char ()
1510                 {
1511                         if (putback_char == -1)
1512                                 putback_char = reader.Read ();
1513                         return putback_char;
1514                 }
1515
1516                 int peek_char2 ()
1517                 {
1518                         if (putback_char != -1)
1519                                 return putback_char;
1520                         return reader.Peek ();
1521                 }
1522
1523                 void putback (int c)
1524                 {
1525                         if (putback_char != -1){
1526                                 Console.WriteLine ("Col: " + col);
1527                                 Console.WriteLine ("Row: " + line);
1528                                 Console.WriteLine ("Name: " + ref_name.Name);
1529                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1530                                 throw new Exception ("This should not happen putback on putback");
1531                         }
1532                         if (c == '\n' || col == 0) {
1533                                 // It won't happen though.
1534                                 line--;
1535                                 ref_line--;
1536                                 col = previous_col;
1537                         }
1538                         else
1539                                 col--;
1540                         putback_char = c;
1541                 }
1542
1543                 public bool advance ()
1544                 {
1545                         return peek_char () != -1;
1546                 }
1547
1548                 public Object Value {
1549                         get {
1550                                 return val;
1551                         }
1552                 }
1553
1554                 public Object value ()
1555                 {
1556                         return val;
1557                 }
1558
1559                 public int token ()
1560                 {
1561                         current_token = xtoken ();
1562                         return current_token;
1563                 }
1564
1565                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1566
1567                 void get_cmd_arg (out string cmd, out string arg)
1568                 {
1569                         int c;
1570
1571                         tokens_seen = false;
1572                         arg = "";
1573
1574                         // skip over white space
1575                         do {
1576                                 c = get_char ();
1577                         } while (c == '\r' || c == ' ' || c == '\t');
1578
1579                         static_cmd_arg.Length = 0;
1580                         while (c != -1 && is_identifier_part_character ((char)c)) {
1581                                 static_cmd_arg.Append ((char)c);
1582                                 c = get_char ();
1583                                 if (c == '\\') {
1584                                         int peek = peek_char ();
1585                                         if (peek == 'U' || peek == 'u') {
1586                                                 int surrogate;
1587                                                 c = EscapeUnicode (c, out surrogate);
1588                                                 if (surrogate != 0) {
1589                                                         if (is_identifier_part_character ((char) c))
1590                                                                 static_cmd_arg.Append ((char) c);
1591                                                         c = surrogate;
1592                                                 }
1593                                         }
1594                                 }
1595                         }
1596
1597                         cmd = static_cmd_arg.ToString ();
1598
1599                         // skip over white space
1600                         while (c == '\r' || c == ' ' || c == '\t')
1601                                 c = get_char ();
1602
1603                         static_cmd_arg.Length = 0;
1604                         int has_identifier_argument = 0;
1605
1606                         while (c != -1 && c != '\n' && c != '\r') {
1607                                 if (c == '\\' && has_identifier_argument >= 0) {
1608                                         if (has_identifier_argument != 0 || (cmd == "define" || cmd == "if" || cmd == "elif" || cmd == "undef")) {
1609                                                 has_identifier_argument = 1;
1610
1611                                                 int peek = peek_char ();
1612                                                 if (peek == 'U' || peek == 'u') {
1613                                                         int surrogate;
1614                                                         c = EscapeUnicode (c, out surrogate);
1615                                                         if (surrogate != 0) {
1616                                                                 if (is_identifier_part_character ((char) c))
1617                                                                         static_cmd_arg.Append ((char) c);
1618                                                                 c = surrogate;
1619                                                         }
1620                                                 }
1621                                         } else {
1622                                                 has_identifier_argument = -1;
1623                                         }
1624                                 }
1625                                 static_cmd_arg.Append ((char) c);
1626                                 c = get_char ();
1627                         }
1628
1629                         if (static_cmd_arg.Length != 0)
1630                                 arg = static_cmd_arg.ToString ();
1631                 }
1632
1633                 //
1634                 // Handles the #line directive
1635                 //
1636                 bool PreProcessLine (string arg)
1637                 {
1638                         if (arg.Length == 0)
1639                                 return false;
1640
1641                         if (arg == "default"){
1642                                 ref_line = line;
1643                                 ref_name = file_name;
1644                                 hidden = false;
1645                                 Location.Push (file_name, ref_name);
1646                                 return true;
1647                         } else if (arg == "hidden"){
1648                                 hidden = true;
1649                                 return true;
1650                         }
1651
1652                         try {
1653                                 int pos;
1654
1655                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1656                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1657                                         pos++;
1658
1659                                         char [] quotes = { '\"' };
1660
1661                                         string name = arg.Substring (pos). Trim (quotes);
1662                                         ref_name = Location.LookupFile (file_name, name);
1663                                         file_name.AddFile (ref_name);
1664                                         hidden = false;
1665                                         Location.Push (file_name, ref_name);
1666                                 } else {
1667                                         ref_line = System.Int32.Parse (arg);
1668                                         hidden = false;
1669                                 }
1670                         } catch {
1671                                 return false;
1672                         }
1673
1674                         return true;
1675                 }
1676
1677                 //
1678                 // Handles #define and #undef
1679                 //
1680                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
1681                 {
1682                         if (ident.Length == 0 || ident == "true" || ident == "false"){
1683                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
1684                                 return;
1685                         }
1686
1687                         if (ident.IndexOfAny (simple_whitespaces) != -1){
1688                                 Error_EndLineExpected ();
1689                                 return;
1690                         }
1691
1692                         if (!is_identifier_start_character (ident [0]))
1693                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
1694
1695                         foreach (char c in ident.Substring (1)){
1696                                 if (!is_identifier_part_character (c)){
1697                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
1698                                         return;
1699                                 }
1700                         }
1701
1702                         if (!caller_is_taking)
1703                                 return;
1704
1705                         if (is_define) {
1706                                 //
1707                                 // #define ident
1708                                 //
1709                                 if (RootContext.IsConditionalDefined (ident))
1710                                         return;
1711
1712                                 file_name.AddDefine (ident);
1713                         } else {
1714                                 //
1715                                 // #undef ident
1716                                 //
1717                                 file_name.AddUndefine (ident);
1718                         }
1719                 }
1720
1721                 static byte read_hex (string arg, int pos, out bool error)
1722                 {
1723                         error = false;
1724
1725                         int total;
1726                         char c = arg [pos];
1727
1728                         if ((c >= '0') && (c <= '9'))
1729                                 total = (int) c - (int) '0';
1730                         else if ((c >= 'A') && (c <= 'F'))
1731                                 total = (int) c - (int) 'A' + 10;
1732                         else if ((c >= 'a') && (c <= 'f'))
1733                                 total = (int) c - (int) 'a' + 10;
1734                         else {
1735                                 error = true;
1736                                 return 0;
1737                         }
1738
1739                         total *= 16;
1740                         c = arg [pos+1];
1741
1742                         if ((c >= '0') && (c <= '9'))
1743                                 total += (int) c - (int) '0';
1744                         else if ((c >= 'A') && (c <= 'F'))
1745                                 total += (int) c - (int) 'A' + 10;
1746                         else if ((c >= 'a') && (c <= 'f'))
1747                                 total += (int) c - (int) 'a' + 10;
1748                         else {
1749                                 error = true;
1750                                 return 0;
1751                         }
1752
1753                         return (byte) total;
1754                 }
1755
1756                 /// <summary>
1757                 /// Handles #pragma checksum
1758                 /// </summary>
1759                 bool PreProcessPragmaChecksum (string arg)
1760                 {
1761                         if ((arg [0] != ' ') && (arg [0] != '\t'))
1762                                 return false;
1763
1764                         arg = arg.Trim (simple_whitespaces);
1765                         if ((arg.Length < 2) || (arg [0] != '"'))
1766                                 return false;
1767
1768                         StringBuilder file_sb = new StringBuilder ();
1769
1770                         int pos = 1;
1771                         char ch;
1772                         while ((ch = arg [pos++]) != '"') {
1773                                 if (pos >= arg.Length)
1774                                         return false;
1775
1776                                 if (ch == '\\') {
1777                                         if (pos+1 >= arg.Length)
1778                                                 return false;
1779                                         ch = arg [pos++];
1780                                 }
1781
1782                                 file_sb.Append (ch);
1783                         }
1784
1785                         if ((pos+2 >= arg.Length) || ((arg [pos] != ' ') && (arg [pos] != '\t')))
1786                                 return false;
1787
1788                         arg = arg.Substring (pos).Trim (simple_whitespaces);
1789                         if ((arg.Length < 42) || (arg [0] != '"') || (arg [1] != '{') ||
1790                             (arg [10] != '-') || (arg [15] != '-') || (arg [20] != '-') ||
1791                             (arg [25] != '-') || (arg [38] != '}') || (arg [39] != '"'))
1792                                 return false;
1793
1794                         bool error;
1795                         byte[] guid_bytes = new byte [16];
1796
1797                         for (int i = 0; i < 4; i++) {
1798                                 guid_bytes [i] = read_hex (arg, 2+2*i, out error);
1799                                 if (error)
1800                                         return false;
1801                         }
1802                         for (int i = 0; i < 2; i++) {
1803                                 guid_bytes [i+4] = read_hex (arg, 11+2*i, out error);
1804                                 if (error)
1805                                         return false;
1806                                 guid_bytes [i+6] = read_hex (arg, 16+2*i, out error);
1807                                 if (error)
1808                                         return false;
1809                                 guid_bytes [i+8] = read_hex (arg, 21+2*i, out error);
1810                                 if (error)
1811                                         return false;
1812                         }
1813
1814                         for (int i = 0; i < 6; i++) {
1815                                 guid_bytes [i+10] = read_hex (arg, 26+2*i, out error);
1816                                 if (error)
1817                                         return false;
1818                         }
1819
1820                         arg = arg.Substring (40).Trim (simple_whitespaces);
1821                         if ((arg.Length < 34) || (arg [0] != '"') || (arg [33] != '"'))
1822                                 return false;
1823
1824                         byte[] checksum_bytes = new byte [16];
1825                         for (int i = 0; i < 16; i++) {
1826                                 checksum_bytes [i] = read_hex (arg, 1+2*i, out error);
1827                                 if (error)
1828                                         return false;
1829                         }
1830
1831                         arg = arg.Substring (34).Trim (simple_whitespaces);
1832                         if (arg.Length > 0)
1833                                 return false;
1834
1835                         SourceFile file = Location.LookupFile (file_name, file_sb.ToString ());
1836                         file.SetChecksum (guid_bytes, checksum_bytes);
1837                         ref_name.AutoGenerated = true;
1838                         return true;
1839                 }
1840
1841                 /// <summary>
1842                 /// Handles #pragma directive
1843                 /// </summary>
1844                 void PreProcessPragma (string arg)
1845                 {
1846                         const string warning = "warning";
1847                         const string w_disable = "warning disable";
1848                         const string w_restore = "warning restore";
1849                         const string checksum = "checksum";
1850
1851                         if (arg == w_disable) {
1852                                 Report.RegisterWarningRegion (Location).WarningDisable (Location.Row);
1853                                 return;
1854                         }
1855
1856                         if (arg == w_restore) {
1857                                 Report.RegisterWarningRegion (Location).WarningEnable (Location.Row);
1858                                 return;
1859                         }
1860
1861                         if (arg.StartsWith (w_disable)) {
1862                                 int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
1863                                 foreach (int code in codes) {
1864                                         if (code != 0)
1865                                                 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1866                                 }
1867                                 return;
1868                         }
1869
1870                         if (arg.StartsWith (w_restore)) {
1871                                 int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
1872                                 Hashtable w_table = Report.warning_ignore_table;
1873                                 foreach (int code in codes) {
1874                                         if (w_table != null && w_table.Contains (code))
1875                                                 Report.Warning (1635, 1, Location, String.Format ("Cannot restore warning `CS{0:0000}' because it was disabled globally", code));
1876                                         Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1877                                 }
1878                                 return;
1879                         }
1880
1881                         if (arg.StartsWith (warning)) {
1882                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
1883                                 return;
1884                         }
1885
1886                         if (arg.StartsWith (checksum)) {
1887                                 if (!PreProcessPragmaChecksum (arg.Substring (checksum.Length)))
1888                                         Warning_InvalidPragmaChecksum ();
1889                                 return;
1890                         }
1891
1892                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
1893                 }
1894
1895                 int[] ParseNumbers (string text)
1896                 {
1897                         string[] string_array = text.Split (',');
1898                         int[] values = new int [string_array.Length];
1899                         int index = 0;
1900                         foreach (string string_code in string_array) {
1901                                 try {
1902                                         values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1903                                 }
1904                                 catch (FormatException) {
1905                                         Report.Warning (1692, 1, Location, "Invalid number");
1906                                 }
1907                         }
1908                         return values;
1909                 }
1910
1911                 bool eval_val (string s)
1912                 {
1913                         if (s == "true")
1914                                 return true;
1915                         if (s == "false")
1916                                 return false;
1917
1918                         return file_name.IsConditionalDefined (s);
1919                 }
1920
1921                 bool pp_primary (ref string s)
1922                 {
1923                         s = s.Trim ();
1924                         int len = s.Length;
1925
1926                         if (len > 0){
1927                                 char c = s [0];
1928
1929                                 if (c == '('){
1930                                         s = s.Substring (1);
1931                                         bool val = pp_expr (ref s, false);
1932                                         if (s.Length > 0 && s [0] == ')'){
1933                                                 s = s.Substring (1);
1934                                                 return val;
1935                                         }
1936                                         Error_InvalidDirective ();
1937                                         return false;
1938                                 }
1939
1940                                 if (is_identifier_start_character (c)){
1941                                         int j = 1;
1942
1943                                         while (j < len){
1944                                                 c = s [j];
1945
1946                                                 if (is_identifier_part_character (c)){
1947                                                         j++;
1948                                                         continue;
1949                                                 }
1950                                                 bool v = eval_val (s.Substring (0, j));
1951                                                 s = s.Substring (j);
1952                                                 return v;
1953                                         }
1954                                         bool vv = eval_val (s);
1955                                         s = "";
1956                                         return vv;
1957                                 }
1958                         }
1959                         Error_InvalidDirective ();
1960                         return false;
1961                 }
1962
1963                 bool pp_unary (ref string s)
1964                 {
1965                         s = s.Trim ();
1966                         int len = s.Length;
1967
1968                         if (len > 0){
1969                                 if (s [0] == '!'){
1970                                         if (len > 1 && s [1] == '='){
1971                                                 Error_InvalidDirective ();
1972                                                 return false;
1973                                         }
1974                                         s = s.Substring (1);
1975                                         return ! pp_primary (ref s);
1976                                 } else
1977                                         return pp_primary (ref s);
1978                         } else {
1979                                 Error_InvalidDirective ();
1980                                 return false;
1981                         }
1982                 }
1983
1984                 bool pp_eq (ref string s)
1985                 {
1986                         bool va = pp_unary (ref s);
1987
1988                         s = s.Trim ();
1989                         int len = s.Length;
1990                         if (len > 0){
1991                                 if (s [0] == '='){
1992                                         if (len > 2 && s [1] == '='){
1993                                                 s = s.Substring (2);
1994                                                 return va == pp_unary (ref s);
1995                                         } else {
1996                                                 Error_InvalidDirective ();
1997                                                 return false;
1998                                         }
1999                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2000                                         s = s.Substring (2);
2001
2002                                         return va != pp_unary (ref s);
2003
2004                                 }
2005                         }
2006
2007                         return va;
2008
2009                 }
2010
2011                 bool pp_and (ref string s)
2012                 {
2013                         bool va = pp_eq (ref s);
2014
2015                         s = s.Trim ();
2016                         int len = s.Length;
2017                         if (len > 0){
2018                                 if (s [0] == '&'){
2019                                         if (len > 2 && s [1] == '&'){
2020                                                 s = s.Substring (2);
2021                                                 return (va & pp_and (ref s));
2022                                         } else {
2023                                                 Error_InvalidDirective ();
2024                                                 return false;
2025                                         }
2026                                 }
2027                         }
2028                         return va;
2029                 }
2030
2031                 //
2032                 // Evaluates an expression for `#if' or `#elif'
2033                 //
2034                 bool pp_expr (ref string s, bool isTerm)
2035                 {
2036                         bool va = pp_and (ref s);
2037                         s = s.Trim ();
2038                         int len = s.Length;
2039                         if (len > 0){
2040                                 char c = s [0];
2041
2042                                 if (c == '|'){
2043                                         if (len > 2 && s [1] == '|'){
2044                                                 s = s.Substring (2);
2045                                                 return va | pp_expr (ref s, isTerm);
2046                                         } else {
2047                                                 Error_InvalidDirective ();
2048                                                 return false;
2049                                         }
2050                                 }
2051                                 if (isTerm) {
2052                                         Error_EndLineExpected ();
2053                                         return false;
2054                                 }
2055                         }
2056
2057                         return va;
2058                 }
2059
2060                 bool eval (string s)
2061                 {
2062                         bool v = pp_expr (ref s, true);
2063                         s = s.Trim ();
2064                         if (s.Length != 0){
2065                                 return false;
2066                         }
2067
2068                         return v;
2069                 }
2070
2071                 void Error_NumericConstantTooLong ()
2072                 {
2073                         Report.Error (1021, Location, "Numeric constant too long");
2074                 }
2075
2076                 void Error_InvalidDirective ()
2077                 {
2078                         Report.Error (1517, Location, "Invalid preprocessor directive");
2079                 }
2080
2081                 void Error_UnexpectedDirective (string extra)
2082                 {
2083                         Report.Error (
2084                                 1028, Location,
2085                                 "Unexpected processor directive ({0})", extra);
2086                 }
2087
2088                 void Error_TokensSeen ()
2089                 {
2090                         Report.Error (1032, Location,
2091                                 "Cannot define or undefine preprocessor symbols after first token in file");
2092                 }
2093
2094                 void Eror_WrongPreprocessorLocation ()
2095                 {
2096                         Report.Error (1040, Location,
2097                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2098                 }
2099
2100                 void Error_EndLineExpected ()
2101                 {
2102                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2103                 }
2104
2105                 void Warning_InvalidPragmaChecksum ()
2106                 {
2107                         Report.Warning (1695, 1, Location,
2108                                         "Invalid #pragma checksum syntax; should be " +
2109                                         "#pragma checksum \"filename\" " +
2110                                         "\"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2111                 }
2112                 //
2113                 // if true, then the code continues processing the code
2114                 // if false, the code stays in a loop until another directive is
2115                 // reached.
2116                 // When caller_is_taking is false we ignore all directives except the ones
2117                 // which can help us to identify where the #if block ends
2118                 bool handle_preprocessing_directive (bool caller_is_taking)
2119                 {
2120                         string cmd, arg;
2121                         bool region_directive = false;
2122
2123                         get_cmd_arg (out cmd, out arg);
2124
2125                         // Eat any trailing whitespaces and single-line comments
2126                         if (arg.IndexOf ("//") != -1)
2127                                 arg = arg.Substring (0, arg.IndexOf ("//"));
2128                         arg = arg.Trim (simple_whitespaces);
2129
2130                         //
2131                         // The first group of pre-processing instructions is always processed
2132                         //
2133                         switch (cmd){
2134                         case "region":
2135                                 region_directive = true;
2136                                 arg = "true";
2137                                 goto case "if";
2138
2139                         case "endregion":
2140                                 if (ifstack == null || ifstack.Count == 0){
2141                                         Error_UnexpectedDirective ("no #region for this #endregion");
2142                                         return true;
2143                                 }
2144                                 int pop = (int) ifstack.Pop ();
2145
2146                                 if ((pop & REGION) == 0)
2147                                         Report.Error (1027, Location, "Expected `#endif' directive");
2148
2149                                 return caller_is_taking;
2150
2151                         case "if":
2152                                 if (ifstack == null)
2153                                         ifstack = new Stack (2);
2154
2155                                 int flags = region_directive ? REGION : 0;
2156                                 if (ifstack.Count == 0){
2157                                         flags |= PARENT_TAKING;
2158                                 } else {
2159                                         int state = (int) ifstack.Peek ();
2160                                         if ((state & TAKING) != 0) {
2161                                                 flags |= PARENT_TAKING;
2162                                         }
2163                                 }
2164
2165                                 if (caller_is_taking && eval (arg)) {
2166                                         ifstack.Push (flags | TAKING);
2167                                         return true;
2168                                 }
2169                                 ifstack.Push (flags);
2170                                 return false;
2171
2172                         case "endif":
2173                                 if (ifstack == null || ifstack.Count == 0){
2174                                         Error_UnexpectedDirective ("no #if for this #endif");
2175                                         return true;
2176                                 } else {
2177                                         pop = (int) ifstack.Pop ();
2178
2179                                         if ((pop & REGION) != 0)
2180                                                 Report.Error (1038, Location, "#endregion directive expected");
2181
2182                                         if (arg.Length != 0) {
2183                                                 Error_EndLineExpected ();
2184                                         }
2185
2186                                         if (ifstack.Count == 0)
2187                                                 return true;
2188
2189                                         int state = (int) ifstack.Peek ();
2190                                         return (state & TAKING) != 0;
2191                                 }
2192
2193                         case "elif":
2194                                 if (ifstack == null || ifstack.Count == 0){
2195                                         Error_UnexpectedDirective ("no #if for this #elif");
2196                                         return true;
2197                                 } else {
2198                                         int state = (int) ifstack.Pop ();
2199
2200                                         if ((state & REGION) != 0) {
2201                                                 Report.Error (1038, Location, "#endregion directive expected");
2202                                                 return true;
2203                                         }
2204
2205                                         if ((state & ELSE_SEEN) != 0){
2206                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2207                                                 return true;
2208                                         }
2209
2210                                         if ((state & TAKING) != 0) {
2211                                                 ifstack.Push (0);
2212                                                 return false;
2213                                         }
2214
2215                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2216                                                 ifstack.Push (state | TAKING);
2217                                                 return true;
2218                                         }
2219
2220                                         ifstack.Push (state);
2221                                         return false;
2222                                 }
2223
2224                         case "else":
2225                                 if (ifstack == null || ifstack.Count == 0){
2226                                         Error_UnexpectedDirective ("no #if for this #else");
2227                                         return true;
2228                                 } else {
2229                                         int state = (int) ifstack.Peek ();
2230
2231                                         if ((state & REGION) != 0) {
2232                                                 Report.Error (1038, Location, "#endregion directive expected");
2233                                                 return true;
2234                                         }
2235
2236                                         if ((state & ELSE_SEEN) != 0){
2237                                                 Error_UnexpectedDirective ("#else within #else");
2238                                                 return true;
2239                                         }
2240
2241                                         ifstack.Pop ();
2242
2243                                         if (arg.Length != 0) {
2244                                                 Error_EndLineExpected ();
2245                                                 return true;
2246                                         }
2247
2248                                         bool ret = false;
2249                                         if ((state & PARENT_TAKING) != 0) {
2250                                                 ret = (state & TAKING) == 0;
2251
2252                                                 if (ret)
2253                                                         state |= TAKING;
2254                                                 else
2255                                                         state &= ~TAKING;
2256                                         }
2257
2258                                         ifstack.Push (state | ELSE_SEEN);
2259
2260                                         return ret;
2261                                 }
2262                         case "define":
2263                                 if (any_token_seen){
2264                                         Error_TokensSeen ();
2265                                         return caller_is_taking;
2266                                 }
2267                                 PreProcessDefinition (true, arg, caller_is_taking);
2268                                 return caller_is_taking;
2269
2270                         case "undef":
2271                                 if (any_token_seen){
2272                                         Error_TokensSeen ();
2273                                         return caller_is_taking;
2274                                 }
2275                                 PreProcessDefinition (false, arg, caller_is_taking);
2276                                 return caller_is_taking;
2277                         }
2278
2279                         //
2280                         // These are only processed if we are in a `taking' block
2281                         //
2282                         if (!caller_is_taking)
2283                                 return false;
2284
2285                         switch (cmd){
2286                         case "error":
2287                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2288                                 return true;
2289
2290                         case "warning":
2291                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2292                                 return true;
2293
2294                         case "pragma":
2295                                 if (RootContext.Version == LanguageVersion.ISO_1) {
2296                                         Report.FeatureIsNotAvailable (Location, "#pragma");
2297                                         return true;
2298                                 }
2299
2300                                 PreProcessPragma (arg);
2301                                 return true;
2302
2303                         case "line":
2304                                 if (!PreProcessLine (arg))
2305                                         Report.Error (
2306                                                 1576, Location,
2307                                                 "The line number specified for #line directive is missing or invalid");
2308                                 return caller_is_taking;
2309                         }
2310
2311                         Report.Error (1024, Location, "Wrong preprocessor directive");
2312                         return true;
2313
2314                 }
2315
2316                 private int consume_string (bool quoted)
2317                 {
2318                         int c;
2319                         string_builder.Length = 0;
2320
2321                         while ((c = get_char ()) != -1){
2322                                 if (c == '"'){
2323                                         if (quoted && peek_char () == '"'){
2324                                                 string_builder.Append ((char) c);
2325                                                 get_char ();
2326                                                 continue;
2327                                         } else {
2328                                                 val = string_builder.ToString ();
2329                                                 return Token.LITERAL_STRING;
2330                                         }
2331                                 }
2332
2333                                 if (c == '\n'){
2334                                         if (!quoted)
2335                                                 Report.Error (1010, Location, "Newline in constant");
2336                                 }
2337
2338                                 if (!quoted){
2339                                         int surrogate;
2340                                         c = escape (c, out surrogate);
2341                                         if (c == -1)
2342                                                 return Token.ERROR;
2343                                         if (surrogate != 0) {
2344                                                 string_builder.Append ((char) c);
2345                                                 c = surrogate;
2346                                         }
2347                                 }
2348                                 string_builder.Append ((char) c);
2349                         }
2350
2351                         Report.Error (1039, Location, "Unterminated string literal");
2352                         return Token.EOF;
2353                 }
2354
2355                 private int consume_identifier (int s)
2356                 {
2357                         int res = consume_identifier (s, false);
2358
2359                         if (doc_state == XmlCommentState.Allowed)
2360                                 doc_state = XmlCommentState.NotAllowed;
2361
2362                         return res;
2363                 }
2364
2365                 private int consume_identifier (int c, bool quoted)
2366                 {
2367                         int pos = 0;
2368
2369                         if (c == '\\') {
2370                                 int surrogate;
2371                                 c = escape (c, out surrogate);
2372                                 if (surrogate != 0) {
2373                                         id_builder [pos++] = (char) c;
2374                                         c = surrogate;
2375                                 }
2376                         }
2377
2378                         id_builder [pos++] = (char) c;
2379                         Location loc = Location;
2380
2381                         while ((c = get_char ()) != -1) {
2382                         loop:
2383                                 if (is_identifier_part_character ((char) c)){
2384                                         if (pos == max_id_size){
2385                                                 Report.Error (645, loc, "Identifier too long (limit is 512 chars)");
2386                                                 return Token.ERROR;
2387                                         }
2388
2389                                         id_builder [pos++] = (char) c;
2390                                 } else if (c == '\\') {
2391                                         int surrogate;
2392                                         c = escape (c, out surrogate);
2393                                         if (surrogate != 0) {
2394                                                 if (is_identifier_part_character ((char) c))
2395                                                         id_builder [pos++] = (char) c;
2396                                                 c = surrogate;
2397                                         }
2398                                         goto loop;
2399                                 } else {
2400                                         putback (c);
2401                                         break;
2402                                 }
2403                         }
2404
2405                         //
2406                         // Optimization: avoids doing the keyword lookup
2407                         // on uppercase letters
2408                         //
2409                         if (id_builder [0] >= '_' && !quoted) {
2410                                 int keyword = GetKeyword (id_builder, pos);
2411                                 if (keyword != -1) {
2412                                         // TODO: No need to store location for keyword, required location cleanup
2413                                         val = loc;
2414                                         return keyword;
2415                                 }
2416                         }
2417
2418                         //
2419                         // Keep identifiers in an array of hashtables to avoid needless
2420                         // allocations
2421                         //
2422                         CharArrayHashtable identifiers_group = identifiers [pos];
2423                         if (identifiers_group != null) {
2424                                 val = identifiers_group [id_builder];
2425                                 if (val != null) {
2426                                         val = new LocatedToken (loc, (string) val);
2427                                         if (quoted)
2428                                                 AddEscapedIdentifier ((LocatedToken) val);
2429                                         return Token.IDENTIFIER;
2430                                 }
2431                         } else {
2432                                 identifiers_group = new CharArrayHashtable (pos);
2433                                 identifiers [pos] = identifiers_group;
2434                         }
2435
2436                         char [] chars = new char [pos];
2437                         Array.Copy (id_builder, chars, pos);
2438
2439                         val = new String (id_builder, 0, pos);
2440                         identifiers_group.Add (chars, val);
2441
2442                         if (RootContext.Version == LanguageVersion.ISO_1) {
2443                                 for (int i = 1; i < chars.Length; i += 3) {
2444                                         if (chars [i] == '_' && (chars [i - 1] == '_' || chars [i + 1] == '_')) {
2445                                                 Report.Error (1638, loc,
2446                                                         "`{0}': Any identifier with double underscores cannot be used when ISO language version mode is specified", val.ToString ());
2447                                         }
2448                                 }
2449                         }
2450
2451                         val = new LocatedToken (loc, (string) val);
2452                         if (quoted)
2453                                 AddEscapedIdentifier ((LocatedToken) val);
2454                         return Token.IDENTIFIER;
2455                 }
2456
2457                 public int xtoken ()
2458                 {
2459                         int d, c;
2460
2461                         // Whether we have seen comments on the current line
2462                         bool comments_seen = false;
2463                         while ((c = get_char ()) != -1) {
2464                                 switch (c) {
2465                                 case '\t':
2466                                         col = ((col + 8) / 8) * 8;
2467                                         continue;
2468
2469                                 case ' ':
2470                                 case '\f':
2471                                 case '\v':
2472                                 case 0xa0:
2473                                 case 0:
2474                                 case 0xFEFF:    // Ignore BOM anywhere in the file
2475                                         continue;
2476
2477 /*                              This is required for compatibility with .NET
2478                                 case 0xEF:
2479                                         if (peek_char () == 0xBB) {
2480                                                 PushPosition ();
2481                                                 get_char ();
2482                                                 if (get_char () == 0xBF)
2483                                                         continue;
2484                                                 PopPosition ();
2485                                         }
2486                                         break;
2487 */
2488                                 case '\r':
2489                                         if (peek_char () != '\n')
2490                                                 advance_line ();
2491                                         else
2492                                                 get_char ();
2493
2494                                         any_token_seen |= tokens_seen;
2495                                         tokens_seen = false;
2496                                         comments_seen = false;
2497                                         continue;
2498
2499                                 case '\\':
2500                                         tokens_seen = true;
2501                                         return consume_identifier (c);
2502
2503                                 case '{':
2504                                         val = Location;
2505                                         return Token.OPEN_BRACE;
2506                                 case '}':
2507                                         val = Location;
2508                                         return Token.CLOSE_BRACE;
2509                                 case '[':
2510                                         // To block doccomment inside attribute declaration.
2511                                         if (doc_state == XmlCommentState.Allowed)
2512                                                 doc_state = XmlCommentState.NotAllowed;
2513                                         return Token.OPEN_BRACKET;
2514                                 case ']':
2515                                         return Token.CLOSE_BRACKET;
2516                                 case '(':
2517                                         val = Location;
2518                                         //
2519                                         // An expression versions of parens can appear in block context only
2520                                         //
2521                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
2522
2523                                                 //
2524                                                 // Optmize most common case where we know that parens
2525                                                 // is not special
2526                                                 //
2527                                                 switch (current_token) {
2528                                                 case Token.IDENTIFIER:
2529                                                 case Token.IF:
2530                                                 case Token.FOR:
2531                                                 case Token.FOREACH:
2532                                                 case Token.TYPEOF:
2533                                                 case Token.WHILE:
2534                                                 case Token.USING:
2535                                                 case Token.DEFAULT:
2536                                                 case Token.DELEGATE:
2537                                                 case Token.OP_GENERICS_GT:
2538                                                         return Token.OPEN_PARENS;
2539                                                 }
2540
2541                                                 // Optimize using peek
2542                                                 int xx = peek_char ();
2543                                                 switch (xx) {
2544                                                 case '(':
2545                                                 case '\'':
2546                                                 case '"':
2547                                                 case '0':
2548                                                 case '1':
2549                                                         return Token.OPEN_PARENS;
2550                                                 }
2551
2552                                                 lambda_arguments_parsing = true;
2553                                                 PushPosition ();
2554                                                 d = TokenizeOpenParens ();
2555                                                 PopPosition ();
2556                                                 lambda_arguments_parsing = false;
2557                                                 return d;
2558                                         }
2559
2560                                         return Token.OPEN_PARENS;
2561                                 case ')':
2562                                         return Token.CLOSE_PARENS;
2563                                 case ',':
2564                                         return Token.COMMA;
2565                                 case ';':
2566                                         return Token.SEMICOLON;
2567                                 case '~':
2568                                         return Token.TILDE;
2569                                 case '?':
2570                                         return TokenizePossibleNullableType ();
2571                                 case '<':
2572                                         if (parsing_generic_less_than++ > 0)
2573                                                 return Token.OP_GENERICS_LT;
2574
2575                                         return TokenizeLessThan ();
2576
2577                                 case '>':
2578                                         d = peek_char ();
2579
2580                                         if (d == '='){
2581                                                 get_char ();
2582                                                 return Token.OP_GE;
2583                                         }
2584
2585                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
2586                                                 parsing_generic_less_than--;
2587                                                 return Token.OP_GENERICS_GT;
2588                                         }
2589
2590                                         if (d == '>') {
2591                                                 get_char ();
2592                                                 d = peek_char ();
2593
2594                                                 if (d == '=') {
2595                                                         get_char ();
2596                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
2597                                                 }
2598                                                 return Token.OP_SHIFT_RIGHT;
2599                                         }
2600
2601                                         return Token.OP_GT;
2602
2603                                 case '+':
2604                                         d = peek_char ();
2605                                         if (d == '+') {
2606                                                 d = Token.OP_INC;
2607                                         } else if (d == '=') {
2608                                                 d = Token.OP_ADD_ASSIGN;
2609                                         } else {
2610                                                 return Token.PLUS;
2611                                         }
2612                                         get_char ();
2613                                         return d;
2614
2615                                 case '-':
2616                                         d = peek_char ();
2617                                         if (d == '-') {
2618                                                 d = Token.OP_DEC;
2619                                         } else if (d == '=')
2620                                                 d = Token.OP_SUB_ASSIGN;
2621                                         else if (d == '>')
2622                                                 d = Token.OP_PTR;
2623                                         else {
2624                                                 return Token.MINUS;
2625                                         }
2626                                         get_char ();
2627                                         return d;
2628
2629                                 case '!':
2630                                         if (peek_char () == '='){
2631                                                 get_char ();
2632                                                 return Token.OP_NE;
2633                                         }
2634                                         return Token.BANG;
2635
2636                                 case '=':
2637                                         d = peek_char ();
2638                                         if (d == '='){
2639                                                 get_char ();
2640                                                 return Token.OP_EQ;
2641                                         }
2642                                         if (d == '>'){
2643                                                 get_char ();
2644                                                 return Token.ARROW;
2645                                         }
2646
2647                                         return Token.ASSIGN;
2648
2649                                 case '&':
2650                                         d = peek_char ();
2651                                         if (d == '&'){
2652                                                 get_char ();
2653                                                 return Token.OP_AND;
2654                                         }
2655                                         if (d == '='){
2656                                                 get_char ();
2657                                                 return Token.OP_AND_ASSIGN;
2658                                         }
2659                                         return Token.BITWISE_AND;
2660
2661                                 case '|':
2662                                         d = peek_char ();
2663                                         if (d == '|'){
2664                                                 get_char ();
2665                                                 return Token.OP_OR;
2666                                         }
2667                                         if (d == '='){
2668                                                 get_char ();
2669                                                 return Token.OP_OR_ASSIGN;
2670                                         }
2671                                         return Token.BITWISE_OR;
2672
2673                                 case '*':
2674                                         if (peek_char () == '='){
2675                                                 get_char ();
2676                                                 return Token.OP_MULT_ASSIGN;
2677                                         }
2678                                         val = Location;
2679                                         return Token.STAR;
2680
2681                                 case '/':
2682                                         d = peek_char ();
2683                                         if (d == '='){
2684                                                 get_char ();
2685                                                 return Token.OP_DIV_ASSIGN;
2686                                         }
2687
2688                                         // Handle double-slash comments.
2689                                         if (d == '/'){
2690                                                 get_char ();
2691                                                 if (RootContext.Documentation != null && peek_char () == '/') {
2692                                                         get_char ();
2693                                                         // Don't allow ////.
2694                                                         if ((d = peek_char ()) != '/') {
2695                                                                 update_comment_location ();
2696                                                                 if (doc_state == XmlCommentState.Allowed)
2697                                                                         handle_one_line_xml_comment ();
2698                                                                 else if (doc_state == XmlCommentState.NotAllowed)
2699                                                                         warn_incorrect_doc_comment ();
2700                                                         }
2701                                                 }
2702                                                 while ((d = get_char ()) != -1 && (d != '\n') && d != '\r');
2703
2704                                                 any_token_seen |= tokens_seen;
2705                                                 tokens_seen = false;
2706                                                 comments_seen = false;
2707                                                 continue;
2708                                         } else if (d == '*'){
2709                                                 get_char ();
2710                                                 bool docAppend = false;
2711                                                 if (RootContext.Documentation != null && peek_char () == '*') {
2712                                                         get_char ();
2713                                                         update_comment_location ();
2714                                                         // But when it is /**/, just do nothing.
2715                                                         if (peek_char () == '/') {
2716                                                                 get_char ();
2717                                                                 continue;
2718                                                         }
2719                                                         if (doc_state == XmlCommentState.Allowed)
2720                                                                 docAppend = true;
2721                                                         else if (doc_state == XmlCommentState.NotAllowed)
2722                                                                 warn_incorrect_doc_comment ();
2723                                                 }
2724
2725                                                 int current_comment_start = 0;
2726                                                 if (docAppend) {
2727                                                         current_comment_start = xml_comment_buffer.Length;
2728                                                         xml_comment_buffer.Append (Environment.NewLine);
2729                                                 }
2730
2731                                                 while ((d = get_char ()) != -1){
2732                                                         if (d == '*' && peek_char () == '/'){
2733                                                                 get_char ();
2734                                                                 comments_seen = true;
2735                                                                 break;
2736                                                         }
2737                                                         if (docAppend)
2738                                                                 xml_comment_buffer.Append ((char) d);
2739
2740                                                         if (d == '\n'){
2741                                                                 any_token_seen |= tokens_seen;
2742                                                                 tokens_seen = false;
2743                                                                 //
2744                                                                 // Reset 'comments_seen' just to be consistent.
2745                                                                 // It doesn't matter either way, here.
2746                                                                 //
2747                                                                 comments_seen = false;
2748                                                         }
2749                                                 }
2750                                                 if (!comments_seen)
2751                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
2752
2753                                                 if (docAppend)
2754                                                         update_formatted_doc_comment (current_comment_start);
2755                                                 continue;
2756                                         }
2757                                         return Token.DIV;
2758
2759                                 case '%':
2760                                         if (peek_char () == '='){
2761                                                 get_char ();
2762                                                 return Token.OP_MOD_ASSIGN;
2763                                         }
2764                                         return Token.PERCENT;
2765
2766                                 case '^':
2767                                         if (peek_char () == '='){
2768                                                 get_char ();
2769                                                 return Token.OP_XOR_ASSIGN;
2770                                         }
2771                                         return Token.CARRET;
2772
2773                                 case ':':
2774                                         if (peek_char () == ':') {
2775                                                 get_char ();
2776                                                 return Token.DOUBLE_COLON;
2777                                         }
2778                                         return Token.COLON;
2779
2780                                 case '0': case '1': case '2': case '3': case '4':
2781                                 case '5': case '6': case '7': case '8': case '9':
2782                                         tokens_seen = true;
2783                                         return is_number (c);
2784
2785                                 case '\n': // white space
2786                                         any_token_seen |= tokens_seen;
2787                                         tokens_seen = false;
2788                                         comments_seen = false;
2789                                         continue;
2790
2791                                 case '.':
2792                                         tokens_seen = true;
2793                                         d = peek_char ();
2794                                         if (d >= '0' && d <= '9')
2795                                                 return is_number (c);
2796                                         return Token.DOT;
2797
2798                                 case '#':
2799                                         if (tokens_seen || comments_seen) {
2800                                                 Eror_WrongPreprocessorLocation ();
2801                                                 return Token.ERROR;
2802                                         }
2803
2804                                         if (handle_preprocessing_directive (true))
2805                                                 continue;
2806
2807                                         bool directive_expected = false;
2808                                         while ((c = get_char ()) != -1) {
2809                                                 if (col == 1) {
2810                                                         directive_expected = true;
2811                                                 } else if (!directive_expected) {
2812                                                         // TODO: Implement comment support for disabled code and uncomment this code
2813 //                                                      if (c == '#') {
2814 //                                                              Eror_WrongPreprocessorLocation ();
2815 //                                                              return Token.ERROR;
2816 //                                                      }
2817                                                         continue;
2818                                                 }
2819
2820                                                 if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
2821                                                         continue;
2822
2823                                                 if (c == '#') {
2824                                                         if (handle_preprocessing_directive (false))
2825                                                                 break;
2826                                                 }
2827                                                 directive_expected = false;
2828                                         }
2829
2830                                         if (c != -1) {
2831                                                 tokens_seen = false;
2832                                                 continue;
2833                                         }
2834
2835                                         return Token.EOF;
2836
2837                                 case '"':
2838                                         return consume_string (false);
2839
2840                                 case '\'':
2841                                         return TokenizeBackslash ();
2842
2843                                 case '@':
2844                                         c = get_char ();
2845                                         if (c == '"') {
2846                                                 tokens_seen = true;
2847                                                 return consume_string (true);
2848                                         }
2849
2850                                         if (is_identifier_start_character (c)){
2851                                                 return consume_identifier (c, true);
2852                                         }
2853
2854                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
2855                                         return Token.ERROR;
2856
2857                                 case EvalStatementParserCharacter:
2858                                         return Token.EVAL_STATEMENT_PARSER;
2859                                 case EvalCompilationUnitParserCharacter:
2860                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
2861                                 case EvalUsingDeclarationsParserCharacter:
2862                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
2863                                 }
2864
2865                                 if (is_identifier_start_character (c)) {
2866                                         tokens_seen = true;
2867                                         return consume_identifier (c);
2868                                 }
2869
2870                                 error_details = ((char)c).ToString ();
2871                                 return Token.ERROR;
2872                         }
2873
2874                         return Token.EOF;
2875                 }
2876
2877                 int TokenizeBackslash ()
2878                 {
2879                         int c = get_char ();
2880                         tokens_seen = true;
2881                         if (c == '\'') {
2882                                 error_details = "Empty character literal";
2883                                 Report.Error (1011, Location, error_details);
2884                                 return Token.ERROR;
2885                         }
2886                         if (c == '\r' || c == '\n') {
2887                                 Report.Error (1010, Location, "Newline in constant");
2888                                 return Token.ERROR;
2889                         }
2890
2891                         int d;
2892                         c = escape (c, out d);
2893                         if (c == -1)
2894                                 return Token.ERROR;
2895                         if (d != 0)
2896                                 throw new NotImplementedException ();
2897
2898                         val = (char) c;
2899                         c = get_char ();
2900
2901                         if (c != '\'') {
2902                                 Report.Error (1012, Location, "Too many characters in character literal");
2903
2904                                 // Try to recover, read until newline or next "'"
2905                                 while ((c = get_char ()) != -1) {
2906                                         if (c == '\n' || c == '\'')
2907                                                 break;
2908                                 }
2909                                 return Token.ERROR;
2910                         }
2911
2912                         return Token.LITERAL_CHARACTER;
2913                 }
2914
2915                 int TokenizeLessThan ()
2916                 {
2917                         int d;
2918                         if (handle_typeof) {
2919                                 PushPosition ();
2920                                 if (parse_generic_dimension (out d)) {
2921                                         val = d;
2922                                         DiscardPosition ();
2923                                         return Token.GENERIC_DIMENSION;
2924                                 }
2925                                 PopPosition ();
2926                         }
2927
2928                         // Save current position and parse next token.
2929                         PushPosition ();
2930                         if (parse_less_than ()) {
2931                                 if (parsing_generic_declaration && token () != Token.DOT) {
2932                                         d = Token.OP_GENERICS_LT_DECL;
2933                                 } else {
2934                                         d = Token.OP_GENERICS_LT;
2935                                 }
2936                                 PopPosition ();
2937                                 return d;
2938                         }
2939
2940                         PopPosition ();
2941                         parsing_generic_less_than = 0;
2942
2943                         d = peek_char ();
2944                         if (d == '<') {
2945                                 get_char ();
2946                                 d = peek_char ();
2947
2948                                 if (d == '=') {
2949                                         get_char ();
2950                                         return Token.OP_SHIFT_LEFT_ASSIGN;
2951                                 }
2952                                 return Token.OP_SHIFT_LEFT;
2953                         }
2954
2955                         if (d == '=') {
2956                                 get_char ();
2957                                 return Token.OP_LE;
2958                         }
2959                         return Token.OP_LT;
2960                 }
2961
2962                 //
2963                 // Handles one line xml comment
2964                 //
2965                 private void handle_one_line_xml_comment ()
2966                 {
2967                         int c;
2968                         while ((c = peek_char ()) == ' ')
2969                                 get_char (); // skip heading whitespaces.
2970                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
2971                                 xml_comment_buffer.Append ((char) get_char ());
2972                         }
2973                         if (c == '\r' || c == '\n')
2974                                 xml_comment_buffer.Append (Environment.NewLine);
2975                 }
2976
2977                 //
2978                 // Remove heading "*" in Javadoc-like xml documentation.
2979                 //
2980                 private void update_formatted_doc_comment (int current_comment_start)
2981                 {
2982                         int length = xml_comment_buffer.Length - current_comment_start;
2983                         string [] lines = xml_comment_buffer.ToString (
2984                                 current_comment_start,
2985                                 length).Replace ("\r", "").Split ('\n');
2986
2987                         // The first line starts with /**, thus it is not target
2988                         // for the format check.
2989                         for (int i = 1; i < lines.Length; i++) {
2990                                 string s = lines [i];
2991                                 int idx = s.IndexOf ('*');
2992                                 string head = null;
2993                                 if (idx < 0) {
2994                                         if (i < lines.Length - 1)
2995                                                 return;
2996                                         head = s;
2997                                 } else
2998                                         head = s.Substring (0, idx);
2999                                 foreach (char c in head)
3000                                         if (c != ' ')
3001                                                 return;
3002                                 lines [i] = s.Substring (idx + 1);
3003                         }
3004                         xml_comment_buffer.Remove (current_comment_start, length);
3005                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3006                 }
3007
3008                 //
3009                 // Updates current comment location.
3010                 //
3011                 private void update_comment_location ()
3012                 {
3013                         if (current_comment_location.IsNull) {
3014                                 // "-2" is for heading "//" or "/*"
3015                                 current_comment_location =
3016                                         new Location (ref_line, hidden ? -1 : col - 2);
3017                         }
3018                 }
3019
3020                 //
3021                 // Checks if there was incorrect doc comments and raise
3022                 // warnings.
3023                 //
3024                 public void check_incorrect_doc_comment ()
3025                 {
3026                         if (xml_comment_buffer.Length > 0)
3027                                 warn_incorrect_doc_comment ();
3028                 }
3029
3030                 //
3031                 // Raises a warning when tokenizer found incorrect doccomment
3032                 // markup.
3033                 //
3034                 private void warn_incorrect_doc_comment ()
3035                 {
3036                         if (doc_state != XmlCommentState.Error) {
3037                                 doc_state = XmlCommentState.Error;
3038                                 // in csc, it is 'XML comment is not placed on
3039                                 // a valid language element'. But that does not
3040                                 // make sense.
3041                                 Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
3042                         }
3043                 }
3044
3045                 //
3046                 // Consumes the saved xml comment lines (if any)
3047                 // as for current target member or type.
3048                 //
3049                 public string consume_doc_comment ()
3050                 {
3051                         if (xml_comment_buffer.Length > 0) {
3052                                 string ret = xml_comment_buffer.ToString ();
3053                                 reset_doc_comment ();
3054                                 return ret;
3055                         }
3056                         return null;
3057                 }
3058
3059                 void reset_doc_comment ()
3060                 {
3061                         xml_comment_buffer.Length = 0;
3062                         current_comment_location = Location.Null;
3063                 }
3064
3065                 public void cleanup ()
3066                 {
3067                         if (ifstack != null && ifstack.Count >= 1) {
3068                                 int state = (int) ifstack.Pop ();
3069                                 if ((state & REGION) != 0)
3070                                         Report.Error (1038, Location, "#endregion directive expected");
3071                                 else
3072                                         Report.Error (1027, Location, "Expected `#endif' directive");
3073                         }
3074                 }
3075         }
3076
3077         //
3078         // Indicates whether it accepts XML documentation or not.
3079         //
3080         public enum XmlCommentState {
3081                 // comment is allowed in this state.
3082                 Allowed,
3083                 // comment is not allowed in this state.
3084                 NotAllowed,
3085                 // once comments appeared when it is NotAllowed, then the
3086                 // state is changed to it, until the state is changed to
3087                 // .Allowed.
3088                 Error
3089         }
3090 }
3091