mcs/mcs/cs-tokenizer.cs

   1 //\r
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler\r
   3 //\r
   4 // Author: Miguel de Icaza (miguel@gnu.org)\r
   5 //\r
   6 // Licensed under the terms of the GNU GPL\r
   7 //\r
   8 // (C) 2001 Ximian, Inc (http://www.ximian.com)\r
   9 //\r
  10 \r
  11 /*\r
  12   Todo:\r
  13 \r
  14   Do something with the integer and float suffixes, pass full datatype?\r
  15   Make sure we accept the proper Unicode ranges, per the spec.\r
  16 \r
  17   Open issues:\r
  18 \r
  19   * Data type handling\r
  20   \r
  21           Currently I am returning different tokens for the various\r
  22           kinds of floating point types (float, double, decimal) and I\r
  23           am only returning a single token for all integer values\r
  24           (integer, unsigned int, etc) as an experiment as to see\r
  25           which mechanism is better.\r
  26         \r
  27           I do not know yet how I will be doing the mapping of "int"\r
  28           to things like System.Int32 and so on.  I am confused.  MAN\r
  29           I AM C\r
  30         \r
  31           Indeed, this might be the core of the problem, I should\r
  32           *probably* just return a TYPE token and have the value of\r
  33           the token be stuff like `System.Int32', `System.UInt32',\r
  34           `System.Double' and so on.  I will see.\r
  35 \r
  36   * Error reporting.\r
  37 \r
  38           I was returning Token.ERROR on errors and setting an\r
  39           internal error string with the details, but it might make sense\r
  40           to just use exceptions.\r
  41 \r
  42           Change of mind: I think I want to keep returning errors *UNLESS* the\r
  43           parser is catching errors from the tokenizer (at that point, there is\r
  44           not really any reason to use exceptions) so that I can continue the\r
  45           parsing \r
  46 \r
  47   * IDEA\r
  48 \r
  49           I think I have solved the problem.  The idea is to not even *bother*\r
  50           about handling data types a lot here (except for fitting data into\r
  51           the proper places), but let the upper layer handle it.\r
  52 \r
  53           Ie, treat LITERAL_CHARACTER, LITERAL_INTEGER, LITERAL_FLOAT, LITERAL_DOUBLE, and\r
  54           return then as `LITERAL_LITERAL' with maybe subdetail information\r
  55 \r
  56 */\r
  57 \r
  58 using System;\r
  59 using System.Text;\r
  60 using CSC;\r
  61 using System.Collections;\r
  62 using System.IO;\r
  63 using System.Globalization;\r
  64 \r
  65 namespace CSC\r
  66 {\r
  67         using CSC;\r
  68         \r
  69         /// <summary>\r
  70         ///    Tokenizer for C# source code. \r
  71         /// </summary>\r
  72         \r
  73         public class Tokenizer : yyParser.yyInput\r
  74         {\r
  75                 StreamReader reader;\r
  76                 public string ref_name;\r
  77                 public int ref_line = 1;\r
  78                 public int line = 1;\r
  79                 public int col = 1;\r
  80                 public int current_token;\r
  81                 bool handle_get_set = false;\r
  82 \r
  83                 public string location {\r
  84                         get {\r
  85                                 string det;\r
  86 \r
  87                                 if (current_token == Token.ERROR)\r
  88                                         det = "detail: " + error_details;\r
  89                                 else\r
  90                                         det = "";\r
  91                                 \r
  92                                 return "Line:     "+line+" Col: "+col + "\n" +\r
  93                                        "VirtLine: "+ref_line +\r
  94                                        " Token: "+current_token + " " + det;\r
  95                         }\r
  96                 }\r
  97 \r
  98                 public bool properties {\r
  99                         get {\r
 100                                 return handle_get_set;\r
 101                         }\r
 102 \r
 103                         set {\r
 104                                 handle_get_set = value;\r
 105                         }\r
 106                 }\r
 107                 \r
 108                 //\r
 109                 // Class variables\r
 110                 // \r
 111                 static Hashtable keywords;\r
 112                 static NumberStyles styles;\r
 113                 static NumberFormatInfo csharp_format_info;\r
 114                 \r
 115                 //\r
 116                 // Values for the associated token returned\r
 117                 //\r
 118                 System.Text.StringBuilder number;\r
 119                 int putback_char;\r
 120                 Object val;\r
 121                 \r
 122                 //\r
 123                 // Details about the error encoutered by the tokenizer\r
 124                 //\r
 125                 string error_details;\r
 126                 \r
 127                 public string error {\r
 128                         get {\r
 129                                 return error_details;\r
 130                         }\r
 131                 }\r
 132                 \r
 133                 public int Line {\r
 134                         get {\r
 135                                 return line;\r
 136                         }\r
 137                 }\r
 138 \r
 139                 public int Col {\r
 140                         get {\r
 141                                 return col;\r
 142                         }\r
 143                 }\r
 144                 \r
 145                 static void initTokens ()\r
 146                 {\r
 147                         keywords = new Hashtable ();\r
 148 \r
 149                         keywords.Add ("abstract", Token.ABSTRACT);\r
 150                         keywords.Add ("as", Token.AS);\r
 151                         keywords.Add ("base", Token.BASE);\r
 152                         keywords.Add ("bool", Token.BOOL);\r
 153                         keywords.Add ("break", Token.BREAK);\r
 154                         keywords.Add ("byte", Token.BYTE);\r
 155                         keywords.Add ("case", Token.CASE);\r
 156                         keywords.Add ("catch", Token.CATCH);\r
 157                         keywords.Add ("char", Token.CHAR);\r
 158                         keywords.Add ("checked", Token.CHECKED);\r
 159                         keywords.Add ("class", Token.CLASS);\r
 160                         keywords.Add ("const", Token.CONST);\r
 161                         keywords.Add ("continue", Token.CONTINUE);\r
 162                         keywords.Add ("decimal", Token.DECIMAL);\r
 163                         keywords.Add ("default", Token.DEFAULT);\r
 164                         keywords.Add ("delegate", Token.DELEGATE);\r
 165                         keywords.Add ("do", Token.DO);\r
 166                         keywords.Add ("double", Token.DOUBLE);\r
 167                         keywords.Add ("else", Token.ELSE);\r
 168                         keywords.Add ("enum", Token.ENUM);\r
 169                         keywords.Add ("event", Token.EVENT);\r
 170                         keywords.Add ("explicit", Token.EXPLICIT);\r
 171                         keywords.Add ("extern", Token.EXTERN);\r
 172                         keywords.Add ("false", Token.FALSE);\r
 173                         keywords.Add ("finally", Token.FINALLY);\r
 174                         keywords.Add ("fixed", Token.FIXED);\r
 175                         keywords.Add ("float", Token.FLOAT);\r
 176                         keywords.Add ("for", Token.FOR);\r
 177                         keywords.Add ("foreach", Token.FOREACH);\r
 178                         keywords.Add ("goto", Token.GOTO);\r
 179                         keywords.Add ("get", Token.GET);\r
 180                         keywords.Add ("if", Token.IF);\r
 181                         keywords.Add ("implicit", Token.IMPLICIT);\r
 182                         keywords.Add ("in", Token.IN);\r
 183                         keywords.Add ("int", Token.INT);\r
 184                         keywords.Add ("interface", Token.INTERFACE);\r
 185                         keywords.Add ("internal", Token.INTERNAL);\r
 186                         keywords.Add ("is", Token.IS);\r
 187                         keywords.Add ("lock ", Token.LOCK );\r
 188                         keywords.Add ("long", Token.LONG);\r
 189                         keywords.Add ("namespace", Token.NAMESPACE);\r
 190                         keywords.Add ("new", Token.NEW);\r
 191                         keywords.Add ("null", Token.NULL);\r
 192                         keywords.Add ("object", Token.OBJECT);\r
 193                         keywords.Add ("operator", Token.OPERATOR);\r
 194                         keywords.Add ("out", Token.OUT);\r
 195                         keywords.Add ("override", Token.OVERRIDE);\r
 196                         keywords.Add ("params", Token.PARAMS);\r
 197                         keywords.Add ("private", Token.PRIVATE);\r
 198                         keywords.Add ("protected", Token.PROTECTED);\r
 199                         keywords.Add ("public", Token.PUBLIC);\r
 200                         keywords.Add ("readonly", Token.READONLY);\r
 201                         keywords.Add ("ref", Token.REF);\r
 202                         keywords.Add ("return", Token.RETURN);\r
 203                         keywords.Add ("sbyte", Token.SBYTE);\r
 204                         keywords.Add ("sealed", Token.SEALED);\r
 205                         keywords.Add ("set", Token.SET);\r
 206                         keywords.Add ("short", Token.SHORT);\r
 207                         keywords.Add ("sizeof", Token.SIZEOF);\r
 208                         keywords.Add ("static", Token.STATIC);\r
 209                         keywords.Add ("string", Token.STRING);\r
 210                         keywords.Add ("struct", Token.STRUCT);\r
 211                         keywords.Add ("switch", Token.SWITCH);\r
 212                         keywords.Add ("this", Token.THIS);\r
 213                         keywords.Add ("throw", Token.THROW);\r
 214                         keywords.Add ("true", Token.TRUE);\r
 215                         keywords.Add ("try", Token.TRY);\r
 216                         keywords.Add ("typeof", Token.TYPEOF);\r
 217                         keywords.Add ("uint", Token.UINT);\r
 218                         keywords.Add ("ulong", Token.ULONG);\r
 219                         keywords.Add ("unchecked", Token.UNCHECKED);\r
 220                         keywords.Add ("unsafe", Token.UNSAFE);\r
 221                         keywords.Add ("ushort", Token.USHORT);\r
 222                         keywords.Add ("using", Token.USING);\r
 223                         keywords.Add ("virtual", Token.VIRTUAL);\r
 224                         keywords.Add ("void", Token.VOID);\r
 225                         keywords.Add ("while", Token.WHILE);\r
 226                 }\r
 227 \r
 228                 //\r
 229                 // Class initializer\r
 230                 // \r
 231                 static Tokenizer ()\r
 232                 {\r
 233                         initTokens ();\r
 234                         csharp_format_info = new NumberFormatInfo ();\r
 235                         csharp_format_info.CurrencyDecimalSeparator = ".";\r
 236                         styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint;\r
 237                 }\r
 238 \r
 239                 bool is_keyword (string name)\r
 240                 {\r
 241                         bool res;\r
 242                         \r
 243                         res = keywords.Contains (name);\r
 244                         if ((name == "get" || name == "set") && handle_get_set == false)\r
 245                                 return false;\r
 246                         return res;\r
 247                 }\r
 248 \r
 249                 int getKeyword (string name)\r
 250                 {\r
 251                         return (int) (keywords [name]);\r
 252                 }\r
 253                 \r
 254                 public Tokenizer (System.IO.Stream input, string fname)\r
 255                 {\r
 256                         this.ref_name = fname;\r
 257                         reader = new System.IO.StreamReader (input);\r
 258                         putback_char = -1;\r
 259                 }\r
 260 \r
 261                 bool is_identifier_start_character (char c)\r
 262                 {\r
 263                         return CharacterInfo.IsLetter (c) || c == '_' ;\r
 264                 }\r
 265 \r
 266                 bool is_identifier_part_character (char c)\r
 267                 {\r
 268                         return (CharacterInfo.IsLetter (c) || CharacterInfo.IsDigit (c) || c == '_');\r
 269                 }\r
 270 \r
 271                 int is_punct (char c, ref bool doread)\r
 272                 {\r
 273                         int idx = "{}[](),:;~+-*/%&|^!=<>?".IndexOf (c);\r
 274                         int d;\r
 275                         int t;\r
 276 \r
 277                         doread = false;\r
 278 \r
 279                         switch (c){\r
 280                         case '{':\r
 281                                 return Token.OPEN_BRACE;\r
 282                         case '}':\r
 283                                 return Token.CLOSE_BRACE;\r
 284                         case '[':\r
 285                                 return Token.OPEN_BRACKET;\r
 286                         case ']':\r
 287                                 return Token.CLOSE_BRACKET;\r
 288                         case '(':\r
 289                                 return Token.OPEN_PARENS;\r
 290                         case ')':\r
 291                                 return Token.CLOSE_PARENS;\r
 292                         case ',':\r
 293                                 return Token.COMMA;\r
 294                         case ':':\r
 295                                 return Token.COLON;\r
 296                         case ';':\r
 297                                 return Token.SEMICOLON;\r
 298                         case '~':\r
 299                                 return Token.TILDE;\r
 300                         case '?':\r
 301                                 return Token.INTERR;\r
 302                         }\r
 303 \r
 304                         d = peekChar ();\r
 305                         if (c == '+'){\r
 306                                 \r
 307                                 if (d == '+')\r
 308                                         t = Token.OP_INC;\r
 309                                 else if (d == '=')\r
 310                                         t = Token.OP_ADD_ASSIGN;\r
 311                                 else\r
 312                                         return Token.PLUS;\r
 313                                 doread = true;\r
 314                                 return t;\r
 315                         }\r
 316                         if (c == '-'){\r
 317                                 if (d == '-')\r
 318                                         t = Token.OP_DEC;\r
 319                                 else if (d == '=')\r
 320                                         t = Token.OP_SUB_ASSIGN;\r
 321                                 else if (d == '>')\r
 322                                         return Token.OP_PTR;\r
 323                                 else\r
 324                                         return Token.MINUS;\r
 325                                 doread = true;\r
 326                                 return t;\r
 327                         }\r
 328 \r
 329                         if (c == '!'){\r
 330                                 if (d == '='){\r
 331                                         doread = true;\r
 332                                         return Token.OP_NE;\r
 333                                 }\r
 334                                 return Token.BANG;\r
 335                         }\r
 336 \r
 337                         if (c == '='){\r
 338                                 if (d == '='){\r
 339                                         doread = true;\r
 340                                         return Token.OP_EQ;\r
 341                                 }\r
 342                                 return Token.ASSIGN;\r
 343                         }\r
 344 \r
 345                         if (c == '&'){\r
 346                                 if (d == '&'){\r
 347                                         doread = true;\r
 348                                         return Token.OP_AND;\r
 349                                 } else if (d == '='){\r
 350                                         doread = true;\r
 351                                         return Token.OP_AND_ASSIGN;\r
 352                                 }\r
 353                                 return Token.BITWISE_AND;\r
 354                         }\r
 355 \r
 356                         if (c == '|'){\r
 357                                 if (d == '|'){\r
 358                                         doread = true;\r
 359                                         return Token.OP_OR;\r
 360                                 } else if (d == '='){\r
 361                                         doread = true;\r
 362                                         return Token.OP_OR_ASSIGN;\r
 363                                 }\r
 364                                 return Token.BITWISE_OR;\r
 365                         }\r
 366 \r
 367                         if (c == '*'){\r
 368                                 if (d == '='){\r
 369                                         doread = true;\r
 370                                         return Token.OP_MULT_ASSIGN;\r
 371                                 }\r
 372                                 return Token.STAR;\r
 373                         }\r
 374 \r
 375                         if (c == '/'){\r
 376                                 if (d == '='){\r
 377                                         doread = true;\r
 378                                         return Token.OP_DIV_ASSIGN;\r
 379                                 }\r
 380                                 return Token.DIV;\r
 381                         }\r
 382 \r
 383                         if (c == '%'){\r
 384                                 if (d == '='){\r
 385                                         doread = true;\r
 386                                         return Token.OP_MOD_ASSIGN;\r
 387                                 }\r
 388                                 return Token.PERCENT;\r
 389                         }\r
 390 \r
 391                         if (c == '^'){\r
 392                                 if (d == '='){\r
 393                                         doread = true;\r
 394                                         return Token.OP_XOR_ASSIGN;\r
 395                                 }\r
 396                                 return Token.CARRET;\r
 397                         }\r
 398 \r
 399                         if (c == '<'){\r
 400                                 if (d == '<'){\r
 401                                         getChar ();\r
 402                                         d = peekChar ();\r
 403 \r
 404                                         if (d == '='){\r
 405                                                 doread = true;\r
 406                                                 return Token.OP_SHIFT_LEFT_ASSIGN;\r
 407                                         }\r
 408                                         return Token.OP_SHIFT_LEFT;\r
 409                                 } else if (d == '='){\r
 410                                         doread = true;\r
 411                                         return Token.OP_LE;\r
 412                                 }\r
 413                                 return Token.OP_LT;\r
 414                         }\r
 415 \r
 416                         if (c == '>'){\r
 417                                 if (d == '>'){\r
 418                                         getChar ();\r
 419                                         d = peekChar ();\r
 420 \r
 421                                         if (d == '='){\r
 422                                                 doread = true;\r
 423                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;\r
 424                                         }\r
 425                                         return Token.OP_SHIFT_RIGHT;\r
 426                                 } else if (d == '='){\r
 427                                         doread = true;\r
 428                                         return Token.OP_GE;\r
 429                                 }\r
 430                                 return Token.OP_GT;\r
 431                         }\r
 432                         return Token.ERROR;\r
 433                 }\r
 434 \r
 435                 bool decimal_digits (int c)\r
 436                 {\r
 437                         int d;\r
 438                         bool seen_digits = false;\r
 439                         \r
 440                         if (c != -1)\r
 441                                 number.Append ((char) c);\r
 442                         \r
 443                         while ((d = peekChar ()) != -1){\r
 444                                 if (CharacterInfo.IsDigit ((char)d)){\r
 445                                         number.Append ((char) d);\r
 446                                         getChar ();\r
 447                                         seen_digits = true;\r
 448                                 } else\r
 449                                         break;\r
 450                         }\r
 451                         return seen_digits;\r
 452                 }\r
 453 \r
 454                 void hex_digits (int c)\r
 455                 {\r
 456                         int d;\r
 457 \r
 458                         if (c != -1)\r
 459                                 number.Append ((char) c);\r
 460                         while ((d = peekChar ()) != -1){\r
 461                                 char e = Char.ToUpper ((char) d);\r
 462                                 \r
 463                                 if (CharacterInfo.IsDigit (e) ||\r
 464                                     (e >= 'A' && e <= 'F')){\r
 465                                         number.Append ((char) e);\r
 466                                         getChar ();\r
 467                                 } else\r
 468                                         break;\r
 469                         }\r
 470                 }\r
 471                 \r
 472                 int real_type_suffix (int c)\r
 473                 {\r
 474                         int t;\r
 475                         \r
 476                         switch (c){\r
 477                         case 'F': case 'f':\r
 478                                 t =  Token.LITERAL_FLOAT;\r
 479                                 break;\r
 480                         case 'D': case 'd':\r
 481                                 t = Token.LITERAL_DOUBLE;\r
 482                                 break;\r
 483                         case 'M': case 'm':\r
 484                                  t= Token.LITERAL_DECIMAL;\r
 485                                 break;\r
 486                         default:\r
 487                                 return Token.NONE;\r
 488                         }\r
 489                         getChar ();\r
 490                         return t;\r
 491                 }\r
 492 \r
 493                 int integer_type_suffix (int c)\r
 494                 {\r
 495                         // FIXME: Handle U and L suffixes.\r
 496                         // We also need to see in which kind of\r
 497                         // Int the thing fits better according to the spec.\r
 498                         return Token.LITERAL_INTEGER;\r
 499                 }\r
 500                 \r
 501                 void adjust_int (int t)\r
 502                 {\r
 503                         val = new System.Int32();\r
 504                         val = System.Int32.Parse (number.ToString (), 0);\r
 505                 }\r
 506 \r
 507                 int adjust_real (int t)\r
 508                 {\r
 509                         string s = number.ToString ();\r
 510 \r
 511                         Console.WriteLine (s);\r
 512                         switch (t){\r
 513                         case Token.LITERAL_DECIMAL:\r
 514                                 val = new System.Decimal ();\r
 515                                 val = System.Decimal.Parse (\r
 516                                         s, styles, csharp_format_info);\r
 517                                 break;\r
 518                         case Token.LITERAL_DOUBLE:\r
 519                                 val = new System.Double ();\r
 520                                 val = System.Double.Parse (\r
 521                                         s, styles, csharp_format_info);\r
 522                                 break;\r
 523                         case Token.LITERAL_FLOAT:\r
 524                                 val = new System.Double ();\r
 525                                 val = (float) System.Double.Parse (\r
 526                                         s, styles, csharp_format_info);\r
 527                                 break;\r
 528 \r
 529                         case Token.NONE:\r
 530                                 val = new System.Double ();\r
 531                                 val = System.Double.Parse (\r
 532                                         s, styles, csharp_format_info);\r
 533                                 t = Token.LITERAL_DOUBLE;\r
 534                                 break;\r
 535                         }\r
 536                         return t;\r
 537                 }\r
 538 \r
 539                 //\r
 540                 // Invoked if we know we have .digits or digits\r
 541                 //\r
 542                 int is_number (int c)\r
 543                 {\r
 544                         bool is_real = false;\r
 545                         number = new System.Text.StringBuilder ();\r
 546                         int type;\r
 547 \r
 548                         number.Length = 0;\r
 549 \r
 550                         if (CharacterInfo.IsDigit ((char)c)){\r
 551                                 if (peekChar () == 'x' || peekChar () == 'X'){\r
 552                                         getChar ();\r
 553                                         hex_digits (-1);\r
 554                                         val = new System.Int32 ();\r
 555                                         val = System.Int32.Parse (number.ToString (), NumberStyles.HexNumber);\r
 556                                         return integer_type_suffix (peekChar ());\r
 557                                 }\r
 558                                 decimal_digits (c);\r
 559                                 c = getChar ();\r
 560                         }\r
 561 \r
 562                         //\r
 563                         // We need to handle the case of\r
 564                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)\r
 565                         //\r
 566                         if (c == '.'){\r
 567                                 if (decimal_digits ('.')){\r
 568                                         is_real = true;\r
 569                                         c = peekChar ();\r
 570                                 } else {\r
 571                                         putback ('.');\r
 572                                         number.Length -= 1;\r
 573                                         adjust_int (Token.LITERAL_INTEGER);\r
 574                                         return Token.LITERAL_INTEGER;\r
 575                                 }\r
 576                         }\r
 577                         \r
 578                         if (c == 'e' || c == 'E'){\r
 579                                 is_real = true;\r
 580                                 number.Append ("e");\r
 581                                 getChar ();\r
 582                                 \r
 583                                 c = peekChar ();\r
 584                                 if (c == '+'){\r
 585                                         number.Append ((char) c);\r
 586                                         getChar ();\r
 587                                         c = peekChar ();\r
 588                                 } else if (c == '-'){\r
 589                                         number.Append ((char) c);\r
 590                                         getChar ();\r
 591                                         c = peekChar ();\r
 592                                 }\r
 593                                 decimal_digits (-1);\r
 594                                 c = peekChar ();\r
 595                         }\r
 596 \r
 597                         type = real_type_suffix (c);\r
 598                         if (type == Token.NONE && !is_real){\r
 599                                 type = integer_type_suffix (c);\r
 600                                 adjust_int (type);\r
 601                                 putback (c);\r
 602                                 return type;\r
 603                         } else\r
 604                                 is_real = true;\r
 605 \r
 606                         if (is_real)\r
 607                                 return adjust_real (type);\r
 608 \r
 609                         Console.WriteLine ("This should not be reached");\r
 610                         throw new Exception ("Is Number should never reach this point");\r
 611                 }\r
 612                         \r
 613                 int escape (int c)\r
 614                 {\r
 615                         int d;\r
 616                         int v;\r
 617 \r
 618                         d = peekChar ();\r
 619                         if (c != '\\')\r
 620                                 return c;\r
 621                         \r
 622                         switch (d){\r
 623                         case 'a':\r
 624                                 v = '\a'; break;\r
 625                         case 'b':\r
 626                                 v = '\b'; break;\r
 627                         case 'n':\r
 628                                 v = '\n'; break;\r
 629                         case 't':\r
 630                                 v = '\t'; break;\r
 631                         case 'v':\r
 632                                 v = '\v'; break;\r
 633                         case 'r':\r
 634                                 v = 'c'; break;\r
 635                         case '\\':\r
 636                                 v = '\\'; break;\r
 637                         case 'f':\r
 638                                 v = '\f'; break;\r
 639                         case '0':\r
 640                                 v = 0; break;\r
 641                         case '"':\r
 642                                 v = '"'; break;\r
 643                         case '\'':\r
 644                                 v = '\''; break;\r
 645                         default:\r
 646                                 error_details = "cs1009: Unrecognized escape sequence " + (char)d;\r
 647                                 return -1;\r
 648                         }\r
 649                         getChar ();\r
 650                         return v;\r
 651                 }\r
 652 \r
 653                 int getChar ()\r
 654                 {\r
 655                         if (putback_char != -1){\r
 656                                 int x = putback_char;\r
 657                                 putback_char = -1;\r
 658 \r
 659                                 return x;\r
 660                         }\r
 661                         return reader.Read ();\r
 662                 }\r
 663 \r
 664                 int peekChar ()\r
 665                 {\r
 666                         if (putback_char != -1)\r
 667                                 return putback_char;\r
 668                         return reader.Peek ();\r
 669                 }\r
 670 \r
 671                 void putback (int c)\r
 672                 {\r
 673                         if (putback_char != -1)\r
 674                                 throw new Exception ("This should not happen putback on putback");\r
 675                         putback_char = c;\r
 676                 }\r
 677 \r
 678                 public bool advance ()\r
 679                 {\r
 680                         return peekChar () != -1;\r
 681                 }\r
 682 \r
 683                 public Object Value {\r
 684                         get {\r
 685                                 return val;\r
 686                         }\r
 687                 }\r
 688 \r
 689                 public Object value ()\r
 690                 {\r
 691                         return val;\r
 692                 }\r
 693                 \r
 694                 public int token ()\r
 695                 {\r
 696                         current_token = xtoken ();\r
 697                         return current_token;\r
 698                 }\r
 699                 \r
 700                 public int xtoken ()\r
 701                 {\r
 702                         int t;\r
 703                         bool allow_keyword = false;\r
 704                         bool doread = false;\r
 705                         int c;\r
 706 \r
 707                         val = null;\r
 708                         for (;(c = getChar ()) != -1; col++) {\r
 709                         \r
 710                                 if (is_identifier_start_character ((char) c)){\r
 711                                         System.Text.StringBuilder id = new System.Text.StringBuilder ();\r
 712                                         string ids;\r
 713                                         \r
 714                                         id.Append ((char) c);\r
 715                                         \r
 716                                         while ((c = peekChar ()) != -1) {\r
 717                                                 if (is_identifier_part_character ((char) c)){\r
 718                                                         id.Append ((char)getChar ());\r
 719                                                         col++;\r
 720                                                 } else \r
 721                                                         break;\r
 722                                         }\r
 723                                         \r
 724                                         ids = id.ToString ();\r
 725                                         \r
 726                                         if (!is_keyword (ids)){\r
 727                                                 val = id.ToString ();\r
 728                                                 return Token.IDENTIFIER;\r
 729                                         }\r
 730                                         \r
 731                                         if (allow_keyword) {\r
 732                                                 val = ids;\r
 733                                                 return Token.IDENTIFIER;\r
 734                                         }\r
 735 \r
 736                                         if (ids == "true")\r
 737                                                 return Token.TRUE;\r
 738                                         else if (ids == "false")\r
 739                                                 return Token.FALSE;\r
 740                                         else if (ids == "null")\r
 741                                                 return Token.NULL;\r
 742                                         \r
 743                                         return getKeyword (ids);\r
 744                                 }\r
 745 \r
 746                                 if (c == '.'){\r
 747                                         if (CharacterInfo.IsDigit ((char) peekChar ()))\r
 748                                                 return is_number (c);\r
 749                                         return Token.DOT;\r
 750                                 }\r
 751                                 \r
 752                                 if (CharacterInfo.IsDigit ((char) c))\r
 753                                         return is_number (c);\r
 754 \r
 755                                 // Handle double-slash comments.\r
 756                                 if (c == '/'){\r
 757                                         int d = peekChar ();\r
 758                                 \r
 759                                         if (d == '/'){\r
 760                                                 getChar ();\r
 761                                                 while ((d = getChar ()) != -1 && (d != '\n'))\r
 762                                                         col++;\r
 763                                                 line++;\r
 764                                                 ref_line++;\r
 765                                                 continue;\r
 766                                         } else if (d == '*'){\r
 767                                                 getChar ();\r
 768 \r
 769                                                 while ((d = getChar ()) != -1){\r
 770                                                         if (d == '*' && peekChar () == '/'){\r
 771                                                                 getChar ();\r
 772                                                                 col++;\r
 773                                                                 break;\r
 774                                                         }\r
 775                                                         if (d == '\n'){\r
 776                                                                 line++;\r
 777                                                                 ref_line++;\r
 778                                                         }\r
 779                                                         col++;\r
 780                                                 }\r
 781                                                 continue;\r
 782                                         }\r
 783                                 }\r
 784 \r
 785                                 /* For now, ignore pre-processor commands */\r
 786                                 if (col == 1 && c == '#'){\r
 787                                         System.Text.StringBuilder s = new System.Text.StringBuilder ();\r
 788                                         \r
 789                                         while ((c = getChar ()) != -1 && (c != '\n')){\r
 790                                                 s.Append ((char) c);\r
 791                                         }\r
 792                                         if (String.Compare (s.ToString (), 0, "line", 0, 4) == 0){\r
 793                                                 string arg = s.ToString ().Substring (5);\r
 794                                                 int pos;\r
 795 \r
 796                                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){\r
 797                                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));\r
 798                                                         pos++;\r
 799 \r
 800                                                         char [] quotes = { '\"' };\r
 801 \r
 802                                                         ref_name = arg.Substring (pos);\r
 803                                                         ref_name.TrimStart (quotes);\r
 804                                                         ref_name.TrimEnd (quotes);\r
 805                                                 } else\r
 806                                                         ref_line = System.Int32.Parse (arg);\r
 807                                         }\r
 808                                         line++;\r
 809                                         ref_line++;\r
 810                                         continue;\r
 811                                 }\r
 812                                 \r
 813                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){\r
 814                                         if (doread){\r
 815                                                 getChar ();\r
 816                                                 col++;\r
 817                                         }\r
 818                                         return t;\r
 819                                 }\r
 820                                 \r
 821                                 if (c == '"'){\r
 822                                         System.Text.StringBuilder s = new System.Text.StringBuilder ();\r
 823 \r
 824                                         while ((c = getChar ()) != -1){\r
 825                                                 if (c == '"'){\r
 826                                                         val = s.ToString ();\r
 827                                                         return Token.LITERAL_STRING;\r
 828                                                 }\r
 829 \r
 830                                                 c = escape (c);\r
 831                                                 if (c == -1)\r
 832                                                         return Token.ERROR;\r
 833                                                 s.Append ((char) c);\r
 834                                         }\r
 835                                 }\r
 836 \r
 837                                 if (c == '\''){\r
 838                                         c = getChar ();\r
 839                                         if (c == '\''){\r
 840                                                 error_details = "CS1011: Empty character literal";\r
 841                                                 return Token.ERROR;\r
 842                                         }\r
 843                                         c = escape (c);\r
 844                                         if (c == -1)\r
 845                                                 return Token.ERROR;\r
 846                                         val = new System.Char ();\r
 847                                         val = (char) c;\r
 848                                         c = getChar ();\r
 849                                         if (c != '\''){\r
 850                                                 error_details = "CS1012: Too many characters in character literal";\r
 851                                                 // Try to recover, read until newline or next "'"\r
 852                                                 while ((c = getChar ()) != -1){\r
 853                                                         if (c == '\n' || c == '\'')\r
 854                                                                 break;\r
 855                                                         \r
 856                                                 }\r
 857                                                 return Token.ERROR;\r
 858                                         }\r
 859                                         return Token.LITERAL_CHARACTER;\r
 860                                 }\r
 861                                 \r
 862                                 // white space\r
 863                                 if (c == '\n'){\r
 864                                         line++;\r
 865                                         ref_line++;\r
 866                                         col = 0;\r
 867                                         continue;\r
 868                                 }\r
 869                                 if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r'){\r
 870                                         if (c == '\t')\r
 871                                                 col = (((col + 8) / 8) * 8) - 1;\r
 872                                         \r
 873                                         continue;\r
 874                                 }\r
 875 \r
 876                                 if (c == '@'){\r
 877                                         allow_keyword = true;\r
 878                                         continue;\r
 879                                 }\r
 880 \r
 881                                 error_details = ((char)c).ToString ();\r
 882                                 \r
 883                                 return Token.ERROR;\r
 884                         }\r
 885 \r
 886                         return Token.EOF;\r
 887                 }\r
 888         }\r
 889 }\r