mcs/mcs/cs-tokenizer.cs

   1 //\r
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler\r
   3 //\r
   4 // Author: Miguel de Icaza (miguel@gnu.org)\r
   5 //\r
   6 // Licensed under the terms of the GNU GPL\r
   7 //\r
   8 // (C) 2001 Ximian, Inc (http://www.ximian.com)\r
   9 //\r
  10 \r
  11 /*\r
  12   Todo:\r
  13 \r
  14   Do something with the integer and float suffixes, pass full datatype?\r
  15   Make sure we accept the proper Unicode ranges, per the spec.\r
  16 \r
  17   * Error reporting.\r
  18 \r
  19           I was returning Token.ERROR on errors and setting an\r
  20           internal error string with the details, but it might make sense\r
  21           to just use exceptions.\r
  22 \r
  23           Change of mind: I think I want to keep returning errors *UNLESS* the\r
  24           parser is catching errors from the tokenizer (at that point, there is\r
  25           not really any reason to use exceptions) so that I can continue the\r
  26           parsing \r
  27 \r
  28   * IDEA\r
  29 \r
  30           I think I have solved the problem.  The idea is to not even *bother*\r
  31           about handling data types a lot here (except for fitting data into\r
  32           the proper places), but let the upper layer handle it.\r
  33 \r
  34           Ie, treat LITERAL_CHARACTER, LITERAL_INTEGER, LITERAL_FLOAT, LITERAL_DOUBLE, and\r
  35           return then as `LITERAL_LITERAL' with maybe subdetail information\r
  36 \r
  37 */\r
  38 \r
  39 using System;\r
  40 using System.Text;\r
  41 using System.Collections;\r
  42 using System.IO;\r
  43 using System.Globalization;\r
  44 \r
  45 namespace CIR\r
  46 {\r
  47         /// <summary>\r
  48         ///    Tokenizer for C# source code. \r
  49         /// </summary>\r
  50 \r
  51         public class Tokenizer : yyParser.yyInput\r
  52         {\r
  53                 TextReader reader;\r
  54                 public string ref_name;\r
  55                 public int ref_line = 1;\r
  56                 public int line = 1;\r
  57                 public int col = 1;\r
  58                 public int current_token;\r
  59                 bool handle_get_set = false;\r
  60 \r
  61                 //\r
  62                 // Returns a verbose representation of the current location\r
  63                 //\r
  64                 public string location {\r
  65                         get {\r
  66                                 string det;\r
  67 \r
  68                                 if (current_token == Token.ERROR)\r
  69                                         det = "detail: " + error_details;\r
  70                                 else\r
  71                                         det = "";\r
  72                                 \r
  73                                 // return "Line:     "+line+" Col: "+col + "\n" +\r
  74                                 //       "VirtLine: "+ref_line +\r
  75                                 //       " Token: "+current_token + " " + det;\r
  76 \r
  77                                 return ref_name + " " + "(" + line + "," + col + "), Token:" + current_token + " " + det;\r
  78                         }\r
  79                 }\r
  80 \r
  81                 public bool properties {\r
  82                         get {\r
  83                                 return handle_get_set;\r
  84                         }\r
  85 \r
  86                         set {\r
  87                                 handle_get_set = value;\r
  88                         }\r
  89                 }\r
  90                 \r
  91                 //\r
  92                 // Class variables\r
  93                 // \r
  94                 static Hashtable keywords;\r
  95                 static NumberStyles styles;\r
  96                 static NumberFormatInfo csharp_format_info;\r
  97                 \r
  98                 //\r
  99                 // Values for the associated token returned\r
 100                 //\r
 101                 System.Text.StringBuilder number;\r
 102                 int putback_char;\r
 103                 Object val;\r
 104                 \r
 105                 //\r
 106                 // Details about the error encoutered by the tokenizer\r
 107                 //\r
 108                 string error_details;\r
 109                 \r
 110                 public string error {\r
 111                         get {\r
 112                                 return error_details;\r
 113                         }\r
 114                 }\r
 115                 \r
 116                 public int Line {\r
 117                         get {\r
 118                                 return line;\r
 119                         }\r
 120                 }\r
 121 \r
 122                 public int Col {\r
 123                         get {\r
 124                                 return col;\r
 125                         }\r
 126                 }\r
 127                 \r
 128                 static void initTokens ()\r
 129                 {\r
 130                         keywords = new Hashtable ();\r
 131 \r
 132                         keywords.Add ("abstract", Token.ABSTRACT);\r
 133                         keywords.Add ("as", Token.AS);\r
 134                         keywords.Add ("add", Token.ADD);\r
 135                         keywords.Add ("base", Token.BASE);\r
 136                         keywords.Add ("bool", Token.BOOL);\r
 137                         keywords.Add ("break", Token.BREAK);\r
 138                         keywords.Add ("byte", Token.BYTE);\r
 139                         keywords.Add ("case", Token.CASE);\r
 140                         keywords.Add ("catch", Token.CATCH);\r
 141                         keywords.Add ("char", Token.CHAR);\r
 142                         keywords.Add ("checked", Token.CHECKED);\r
 143                         keywords.Add ("class", Token.CLASS);\r
 144                         keywords.Add ("const", Token.CONST);\r
 145                         keywords.Add ("continue", Token.CONTINUE);\r
 146                         keywords.Add ("decimal", Token.DECIMAL);\r
 147                         keywords.Add ("default", Token.DEFAULT);\r
 148                         keywords.Add ("delegate", Token.DELEGATE);\r
 149                         keywords.Add ("do", Token.DO);\r
 150                         keywords.Add ("double", Token.DOUBLE);\r
 151                         keywords.Add ("else", Token.ELSE);\r
 152                         keywords.Add ("enum", Token.ENUM);\r
 153                         keywords.Add ("event", Token.EVENT);\r
 154                         keywords.Add ("explicit", Token.EXPLICIT);\r
 155                         keywords.Add ("extern", Token.EXTERN);\r
 156                         keywords.Add ("false", Token.FALSE);\r
 157                         keywords.Add ("finally", Token.FINALLY);\r
 158                         keywords.Add ("fixed", Token.FIXED);\r
 159                         keywords.Add ("float", Token.FLOAT);\r
 160                         keywords.Add ("for", Token.FOR);\r
 161                         keywords.Add ("foreach", Token.FOREACH);\r
 162                         keywords.Add ("goto", Token.GOTO);\r
 163                         keywords.Add ("get", Token.GET);\r
 164                         keywords.Add ("if", Token.IF);\r
 165                         keywords.Add ("implicit", Token.IMPLICIT);\r
 166                         keywords.Add ("in", Token.IN);\r
 167                         keywords.Add ("int", Token.INT);\r
 168                         keywords.Add ("interface", Token.INTERFACE);\r
 169                         keywords.Add ("internal", Token.INTERNAL);\r
 170                         keywords.Add ("is", Token.IS);\r
 171                         keywords.Add ("lock ", Token.LOCK );\r
 172                         keywords.Add ("long", Token.LONG);\r
 173                         keywords.Add ("namespace", Token.NAMESPACE);\r
 174                         keywords.Add ("new", Token.NEW);\r
 175                         keywords.Add ("null", Token.NULL);\r
 176                         keywords.Add ("object", Token.OBJECT);\r
 177                         keywords.Add ("operator", Token.OPERATOR);\r
 178                         keywords.Add ("out", Token.OUT);\r
 179                         keywords.Add ("override", Token.OVERRIDE);\r
 180                         keywords.Add ("params", Token.PARAMS);\r
 181                         keywords.Add ("private", Token.PRIVATE);\r
 182                         keywords.Add ("protected", Token.PROTECTED);\r
 183                         keywords.Add ("public", Token.PUBLIC);\r
 184                         keywords.Add ("readonly", Token.READONLY);\r
 185                         keywords.Add ("ref", Token.REF);\r
 186                         keywords.Add ("remove", Token.REMOVE);\r
 187                         keywords.Add ("return", Token.RETURN);\r
 188                         keywords.Add ("sbyte", Token.SBYTE);\r
 189                         keywords.Add ("sealed", Token.SEALED);\r
 190                         keywords.Add ("set", Token.SET);\r
 191                         keywords.Add ("short", Token.SHORT);\r
 192                         keywords.Add ("sizeof", Token.SIZEOF);\r
 193                         keywords.Add ("static", Token.STATIC);\r
 194                         keywords.Add ("string", Token.STRING);\r
 195                         keywords.Add ("struct", Token.STRUCT);\r
 196                         keywords.Add ("switch", Token.SWITCH);\r
 197                         keywords.Add ("this", Token.THIS);\r
 198                         keywords.Add ("throw", Token.THROW);\r
 199                         keywords.Add ("true", Token.TRUE);\r
 200                         keywords.Add ("try", Token.TRY);\r
 201                         keywords.Add ("typeof", Token.TYPEOF);\r
 202                         keywords.Add ("uint", Token.UINT);\r
 203                         keywords.Add ("ulong", Token.ULONG);\r
 204                         keywords.Add ("unchecked", Token.UNCHECKED);\r
 205                         keywords.Add ("unsafe", Token.UNSAFE);\r
 206                         keywords.Add ("ushort", Token.USHORT);\r
 207                         keywords.Add ("using", Token.USING);\r
 208                         keywords.Add ("virtual", Token.VIRTUAL);\r
 209                         keywords.Add ("void", Token.VOID);\r
 210                         keywords.Add ("while", Token.WHILE);\r
 211                 }\r
 212 \r
 213                 //\r
 214                 // Class initializer\r
 215                 // \r
 216                 static Tokenizer ()\r
 217                 {\r
 218                         initTokens ();\r
 219                         csharp_format_info = new NumberFormatInfo ();\r
 220                         csharp_format_info.CurrencyDecimalSeparator = ".";\r
 221                         styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint;\r
 222                 }\r
 223 \r
 224                 bool is_keyword (string name)\r
 225                 {\r
 226                         bool res;\r
 227                         \r
 228                         res = keywords.Contains (name);\r
 229                         if ((name == "get" || name == "set") && handle_get_set == false)\r
 230                                 return false;\r
 231                         return res;\r
 232                 }\r
 233 \r
 234                 int getKeyword (string name)\r
 235                 {\r
 236                         return (int) (keywords [name]);\r
 237                 }\r
 238 \r
 239                 public Location Location {\r
 240                         get {\r
 241                                 return new Location (ref_line);\r
 242                         }\r
 243                 }\r
 244                 \r
 245                 public Tokenizer (System.IO.TextReader input, string fname)\r
 246                 {\r
 247                         this.ref_name = fname;\r
 248                         reader = input;\r
 249                         putback_char = -1;\r
 250 \r
 251                         Location.Push (fname);\r
 252                 }\r
 253 \r
 254                 bool is_identifier_start_character (char c)\r
 255                 {\r
 256                         return Char.IsLetter (c) || c == '_' ;\r
 257                 }\r
 258 \r
 259                 bool is_identifier_part_character (char c)\r
 260                 {\r
 261                         return (Char.IsLetter (c) || Char.IsDigit (c) || c == '_');\r
 262                 }\r
 263 \r
 264                 int is_punct (char c, ref bool doread)\r
 265                 {\r
 266                         int idx = "{}[](),:;~+-*/%&|^!=<>?".IndexOf (c);\r
 267                         int d;\r
 268                         int t;\r
 269 \r
 270                         doread = false;\r
 271 \r
 272                         switch (c){\r
 273                         case '{':\r
 274                                 return Token.OPEN_BRACE;\r
 275                         case '}':\r
 276                                 return Token.CLOSE_BRACE;\r
 277                         case '[':\r
 278                                 return Token.OPEN_BRACKET;\r
 279                         case ']':\r
 280                                 return Token.CLOSE_BRACKET;\r
 281                         case '(':\r
 282                                 return Token.OPEN_PARENS;\r
 283                         case ')':\r
 284                                 return Token.CLOSE_PARENS;\r
 285                         case ',':\r
 286                                 return Token.COMMA;\r
 287                         case ':':\r
 288                                 return Token.COLON;\r
 289                         case ';':\r
 290                                 return Token.SEMICOLON;\r
 291                         case '~':\r
 292                                 return Token.TILDE;\r
 293                         case '?':\r
 294                                 return Token.INTERR;\r
 295                         }\r
 296 \r
 297                         d = peekChar ();\r
 298                         if (c == '+'){\r
 299                                 \r
 300                                 if (d == '+')\r
 301                                         t = Token.OP_INC;\r
 302                                 else if (d == '=')\r
 303                                         t = Token.OP_ADD_ASSIGN;\r
 304                                 else\r
 305                                         return Token.PLUS;\r
 306                                 doread = true;\r
 307                                 return t;\r
 308                         }\r
 309                         if (c == '-'){\r
 310                                 if (d == '-')\r
 311                                         t = Token.OP_DEC;\r
 312                                 else if (d == '=')\r
 313                                         t = Token.OP_SUB_ASSIGN;\r
 314                                 else if (d == '>')\r
 315                                         return Token.OP_PTR;\r
 316                                 else\r
 317                                         return Token.MINUS;\r
 318                                 doread = true;\r
 319                                 return t;\r
 320                         }\r
 321 \r
 322                         if (c == '!'){\r
 323                                 if (d == '='){\r
 324                                         doread = true;\r
 325                                         return Token.OP_NE;\r
 326                                 }\r
 327                                 return Token.BANG;\r
 328                         }\r
 329 \r
 330                         if (c == '='){\r
 331                                 if (d == '='){\r
 332                                         doread = true;\r
 333                                         return Token.OP_EQ;\r
 334                                 }\r
 335                                 return Token.ASSIGN;\r
 336                         }\r
 337 \r
 338                         if (c == '&'){\r
 339                                 if (d == '&'){\r
 340                                         doread = true;\r
 341                                         return Token.OP_AND;\r
 342                                 } else if (d == '='){\r
 343                                         doread = true;\r
 344                                         return Token.OP_AND_ASSIGN;\r
 345                                 }\r
 346                                 return Token.BITWISE_AND;\r
 347                         }\r
 348 \r
 349                         if (c == '|'){\r
 350                                 if (d == '|'){\r
 351                                         doread = true;\r
 352                                         return Token.OP_OR;\r
 353                                 } else if (d == '='){\r
 354                                         doread = true;\r
 355                                         return Token.OP_OR_ASSIGN;\r
 356                                 }\r
 357                                 return Token.BITWISE_OR;\r
 358                         }\r
 359 \r
 360                         if (c == '*'){\r
 361                                 if (d == '='){\r
 362                                         doread = true;\r
 363                                         return Token.OP_MULT_ASSIGN;\r
 364                                 }\r
 365                                 return Token.STAR;\r
 366                         }\r
 367 \r
 368                         if (c == '/'){\r
 369                                 if (d == '='){\r
 370                                         doread = true;\r
 371                                         return Token.OP_DIV_ASSIGN;\r
 372                                 }\r
 373                                 return Token.DIV;\r
 374                         }\r
 375 \r
 376                         if (c == '%'){\r
 377                                 if (d == '='){\r
 378                                         doread = true;\r
 379                                         return Token.OP_MOD_ASSIGN;\r
 380                                 }\r
 381                                 return Token.PERCENT;\r
 382                         }\r
 383 \r
 384                         if (c == '^'){\r
 385                                 if (d == '='){\r
 386                                         doread = true;\r
 387                                         return Token.OP_XOR_ASSIGN;\r
 388                                 }\r
 389                                 return Token.CARRET;\r
 390                         }\r
 391 \r
 392                         if (c == '<'){\r
 393                                 if (d == '<'){\r
 394                                         getChar ();\r
 395                                         d = peekChar ();\r
 396 \r
 397                                         if (d == '='){\r
 398                                                 doread = true;\r
 399                                                 return Token.OP_SHIFT_LEFT_ASSIGN;\r
 400                                         }\r
 401                                         return Token.OP_SHIFT_LEFT;\r
 402                                 } else if (d == '='){\r
 403                                         doread = true;\r
 404                                         return Token.OP_LE;\r
 405                                 }\r
 406                                 return Token.OP_LT;\r
 407                         }\r
 408 \r
 409                         if (c == '>'){\r
 410                                 if (d == '>'){\r
 411                                         getChar ();\r
 412                                         d = peekChar ();\r
 413 \r
 414                                         if (d == '='){\r
 415                                                 doread = true;\r
 416                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;\r
 417                                         }\r
 418                                         return Token.OP_SHIFT_RIGHT;\r
 419                                 } else if (d == '='){\r
 420                                         doread = true;\r
 421                                         return Token.OP_GE;\r
 422                                 }\r
 423                                 return Token.OP_GT;\r
 424                         }\r
 425                         return Token.ERROR;\r
 426                 }\r
 427 \r
 428                 bool decimal_digits (int c)\r
 429                 {\r
 430                         int d;\r
 431                         bool seen_digits = false;\r
 432                         \r
 433                         if (c != -1)\r
 434                                 number.Append ((char) c);\r
 435                         \r
 436                         while ((d = peekChar ()) != -1){\r
 437                                 if (Char.IsDigit ((char)d)){\r
 438                                         number.Append ((char) d);\r
 439                                         getChar ();\r
 440                                         seen_digits = true;\r
 441                                 } else\r
 442                                         break;\r
 443                         }\r
 444                         return seen_digits;\r
 445                 }\r
 446 \r
 447                 void hex_digits (int c)\r
 448                 {\r
 449                         int d;\r
 450 \r
 451                         if (c != -1)\r
 452                                 number.Append ((char) c);\r
 453                         while ((d = peekChar ()) != -1){\r
 454                                 char e = Char.ToUpper ((char) d);\r
 455                                 \r
 456                                 if (Char.IsDigit (e) ||\r
 457                                     (e >= 'A' && e <= 'F')){\r
 458                                         number.Append ((char) e);\r
 459                                         getChar ();\r
 460                                 } else\r
 461                                         break;\r
 462                         }\r
 463                 }\r
 464                 \r
 465                 int real_type_suffix (int c)\r
 466                 {\r
 467                         int t;\r
 468                         \r
 469                         switch (c){\r
 470                         case 'F': case 'f':\r
 471                                 t =  Token.LITERAL_FLOAT;\r
 472                                 break;\r
 473                         case 'D': case 'd':\r
 474                                 t = Token.LITERAL_DOUBLE;\r
 475                                 break;\r
 476                         case 'M': case 'm':\r
 477                                  t= Token.LITERAL_DECIMAL;\r
 478                                 break;\r
 479                         default:\r
 480                                 return Token.NONE;\r
 481                         }\r
 482                         //getChar ();\r
 483                         return t;\r
 484                 }\r
 485 \r
 486                 int integer_type_suffix (int c)\r
 487                 {\r
 488                         // FIXME: Handle U and L suffixes.\r
 489                         // We also need to see in which kind of\r
 490                         // Int the thing fits better according to the spec.\r
 491                         return Token.LITERAL_INTEGER;\r
 492                 }\r
 493                 \r
 494                 void adjust_int (int t)\r
 495                 {\r
 496                         val = new System.Int32();\r
 497                         val = System.Int32.Parse (number.ToString (), 0);\r
 498                 }\r
 499 \r
 500                 int adjust_real (int t)\r
 501                 {\r
 502                         string s = number.ToString ();\r
 503 \r
 504                         switch (t){\r
 505                         case Token.LITERAL_DECIMAL:\r
 506                                 val = new System.Decimal ();\r
 507                                 val = System.Decimal.Parse (\r
 508                                         s, styles, csharp_format_info);\r
 509                                 break;\r
 510                         case Token.LITERAL_DOUBLE:\r
 511                                 val = new System.Double ();\r
 512                                 val = System.Double.Parse (\r
 513                                         s, styles, csharp_format_info);\r
 514                                 break;\r
 515                         case Token.LITERAL_FLOAT:\r
 516                                 val = new System.Double ();\r
 517                                 val = (float) System.Double.Parse (\r
 518                                         s, styles, csharp_format_info);\r
 519                                 break;\r
 520 \r
 521                         case Token.NONE:\r
 522                                 val = new System.Double ();\r
 523                                 val = System.Double.Parse (\r
 524                                         s, styles, csharp_format_info);\r
 525                                 t = Token.LITERAL_DOUBLE;\r
 526                                 break;\r
 527                         }\r
 528                         return t;\r
 529                 }\r
 530 \r
 531                 //\r
 532                 // Invoked if we know we have .digits or digits\r
 533                 //\r
 534                 int is_number (int c)\r
 535                 {\r
 536                         bool is_real = false;\r
 537                         number = new System.Text.StringBuilder ();\r
 538                         int type;\r
 539 \r
 540                         number.Length = 0;\r
 541 \r
 542                         if (Char.IsDigit ((char)c)){\r
 543                                 if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){\r
 544                                         ulong ul;\r
 545                                         getChar ();\r
 546                                         hex_digits (-1);\r
 547 \r
 548                                         string s = number.ToString ();\r
 549 \r
 550                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);\r
 551                                         if ((ul & 0xffffffff00000000) == 0){\r
 552                                                 uint ui = (uint) ul;\r
 553                                                 \r
 554                                                 if ((ui & 0x80000000) != 0)\r
 555                                                         val = ui;\r
 556                                                 else\r
 557                                                         val = (int) ui;\r
 558                                         } else {\r
 559                                                 if ((ul & 0x8000000000000000) != 0)\r
 560                                                         val = ul;\r
 561                                                 else\r
 562                                                         val = (long) ul;\r
 563                                         }\r
 564 \r
 565                                         return integer_type_suffix (peekChar ());\r
 566                                 }\r
 567                                 decimal_digits (c);\r
 568                                 c = getChar ();\r
 569                         }\r
 570 \r
 571                         //\r
 572                         // We need to handle the case of\r
 573                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)\r
 574                         //\r
 575                         if (c == '.'){\r
 576                                 if (decimal_digits ('.')){\r
 577                                         is_real = true;\r
 578                                         c = peekChar ();\r
 579                                 } else {\r
 580                                         putback ('.');\r
 581                                         number.Length -= 1;\r
 582                                         adjust_int (Token.LITERAL_INTEGER);\r
 583                                         return Token.LITERAL_INTEGER;\r
 584                                 }\r
 585                         }\r
 586                         \r
 587                         if (c == 'e' || c == 'E'){\r
 588                                 is_real = true;\r
 589                                 number.Append ("e");\r
 590                                 getChar ();\r
 591                                 \r
 592                                 c = peekChar ();\r
 593                                 if (c == '+'){\r
 594                                         number.Append ((char) c);\r
 595                                         getChar ();\r
 596                                         c = peekChar ();\r
 597                                 } else if (c == '-'){\r
 598                                         number.Append ((char) c);\r
 599                                         getChar ();\r
 600                                         c = peekChar ();\r
 601                                 }\r
 602                                 decimal_digits (-1);\r
 603                                 c = peekChar ();\r
 604                         }\r
 605 \r
 606                         type = real_type_suffix (c);\r
 607                         if (type == Token.NONE && !is_real){\r
 608                                 type = integer_type_suffix (c);\r
 609                                 adjust_int (type);\r
 610                                 putback (c);\r
 611                                 return type;\r
 612                         } else\r
 613                                 is_real = true;\r
 614 \r
 615                         if (is_real)\r
 616                                 return adjust_real (type);\r
 617 \r
 618                         Console.WriteLine ("This should not be reached");\r
 619                         throw new Exception ("Is Number should never reach this point");\r
 620                 }\r
 621                         \r
 622                 int escape (int c)\r
 623                 {\r
 624                         int d;\r
 625                         int v;\r
 626 \r
 627                         d = peekChar ();\r
 628                         if (c != '\\')\r
 629                                 return c;\r
 630                         \r
 631                         switch (d){\r
 632                         case 'a':\r
 633                                 v = '\a'; break;\r
 634                         case 'b':\r
 635                                 v = '\b'; break;\r
 636                         case 'n':\r
 637                                 v = '\n'; break;\r
 638                         case 't':\r
 639                                 v = '\t'; break;\r
 640                         case 'v':\r
 641                                 v = '\v'; break;\r
 642                         case 'r':\r
 643                                 v = 'c'; break;\r
 644                         case '\\':\r
 645                                 v = '\\'; break;\r
 646                         case 'f':\r
 647                                 v = '\f'; break;\r
 648                         case '0':\r
 649                                 v = 0; break;\r
 650                         case '"':\r
 651                                 v = '"'; break;\r
 652                         case '\'':\r
 653                                 v = '\''; break;\r
 654                         default:\r
 655                                 error_details = "cs1009: Unrecognized escape sequence " + (char)d;\r
 656                                 return -1;\r
 657                         }\r
 658                         getChar ();\r
 659                         return v;\r
 660                 }\r
 661 \r
 662                 int getChar ()\r
 663                 {\r
 664                         if (putback_char != -1){\r
 665                                 int x = putback_char;\r
 666                                 putback_char = -1;\r
 667 \r
 668                                 return x;\r
 669                         }\r
 670                         return reader.Read ();\r
 671                 }\r
 672 \r
 673                 int peekChar ()\r
 674                 {\r
 675                         if (putback_char != -1)\r
 676                                 return putback_char;\r
 677                         return reader.Peek ();\r
 678                 }\r
 679 \r
 680                 void putback (int c)\r
 681                 {\r
 682                         if (putback_char != -1)\r
 683                                 throw new Exception ("This should not happen putback on putback");\r
 684                         putback_char = c;\r
 685                 }\r
 686 \r
 687                 public bool advance ()\r
 688                 {\r
 689                         return peekChar () != -1;\r
 690                 }\r
 691 \r
 692                 public Object Value {\r
 693                         get {\r
 694                                 return val;\r
 695                         }\r
 696                 }\r
 697 \r
 698                 public Object value ()\r
 699                 {\r
 700                         return val;\r
 701                 }\r
 702                 \r
 703                 public int token ()\r
 704                 {\r
 705                         current_token = xtoken ();\r
 706                         return current_token;\r
 707                 }\r
 708                 \r
 709                 public int xtoken ()\r
 710                 {\r
 711                         int t;\r
 712                         bool allow_keyword_as_ident = false;\r
 713                         bool doread = false;\r
 714                         int c;\r
 715 \r
 716                         val = null;\r
 717                         for (;(c = getChar ()) != -1; col++) {\r
 718                         \r
 719                                 if (is_identifier_start_character ((char) c)){\r
 720                                         System.Text.StringBuilder id = new System.Text.StringBuilder ();\r
 721                                         string ids;\r
 722                                         \r
 723                                         id.Append ((char) c);\r
 724                                         \r
 725                                         while ((c = peekChar ()) != -1) {\r
 726                                                 if (is_identifier_part_character ((char) c)){\r
 727                                                         id.Append ((char)getChar ());\r
 728                                                         col++;\r
 729                                                 } else \r
 730                                                         break;\r
 731                                         }\r
 732                                         \r
 733                                         ids = id.ToString ();\r
 734 \r
 735                                         if (!is_keyword (ids) || allow_keyword_as_ident) {\r
 736                                                 val = ids;\r
 737                                                 return Token.IDENTIFIER;\r
 738                                         }\r
 739 \r
 740                                         // true, false and null are in the hash anyway.\r
 741                                         return getKeyword (ids);\r
 742 \r
 743                                 }\r
 744 \r
 745                                 if (c == '.'){\r
 746                                         if (Char.IsDigit ((char) peekChar ()))\r
 747                                                 return is_number (c);\r
 748                                         return Token.DOT;\r
 749                                 }\r
 750                                 \r
 751                                 if (Char.IsDigit ((char) c))\r
 752                                         return is_number (c);\r
 753 \r
 754                                 // Handle double-slash comments.\r
 755                                 if (c == '/'){\r
 756                                         int d = peekChar ();\r
 757                                 \r
 758                                         if (d == '/'){\r
 759                                                 getChar ();\r
 760                                                 while ((d = getChar ()) != -1 && (d != '\n'))\r
 761                                                         col++;\r
 762                                                 line++;\r
 763                                                 ref_line++;\r
 764                                                 continue;\r
 765                                         } else if (d == '*'){\r
 766                                                 getChar ();\r
 767 \r
 768                                                 while ((d = getChar ()) != -1){\r
 769                                                         if (d == '*' && peekChar () == '/'){\r
 770                                                                 getChar ();\r
 771                                                                 col++;\r
 772                                                                 break;\r
 773                                                         }\r
 774                                                         if (d == '\n'){\r
 775                                                                 line++;\r
 776                                                                 ref_line++;\r
 777                                                         }\r
 778                                                         col++;\r
 779                                                 }\r
 780                                                 continue;\r
 781                                         }\r
 782                                 }\r
 783 \r
 784                                 /* For now, ignore pre-processor commands */\r
 785                                 if (col == 1 && c == '#'){\r
 786                                         System.Text.StringBuilder s = new System.Text.StringBuilder ();\r
 787                                         \r
 788                                         while ((c = getChar ()) != -1 && (c != '\n')){\r
 789                                                 s.Append ((char) c);\r
 790                                         }\r
 791                                         if (String.Compare (s.ToString (), 0, "line", 0, 4) == 0){\r
 792                                                 string arg = s.ToString ().Substring (5);\r
 793                                                 int pos;\r
 794 \r
 795                                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){\r
 796                                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));\r
 797                                                         pos++;\r
 798 \r
 799                                                         char [] quotes = { '\"' };\r
 800 \r
 801                                                         ref_name = arg.Substring (pos);\r
 802                                                         ref_name.TrimStart (quotes);\r
 803                                                         ref_name.TrimEnd (quotes);\r
 804                                                 } else\r
 805                                                         ref_line = System.Int32.Parse (arg);\r
 806                                         }\r
 807                                         line++;\r
 808                                         ref_line++;\r
 809                                         continue;\r
 810                                 }\r
 811                                 \r
 812                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){\r
 813                                         if (doread){\r
 814                                                 getChar ();\r
 815                                                 col++;\r
 816                                         }\r
 817                                         return t;\r
 818                                 }\r
 819                                 \r
 820                                 if (c == '"'){\r
 821                                         System.Text.StringBuilder s = new System.Text.StringBuilder ();\r
 822 \r
 823                                         while ((c = getChar ()) != -1){\r
 824                                                 if (c == '"'){\r
 825                                                         val = s.ToString ();\r
 826                                                         return Token.LITERAL_STRING;\r
 827                                                 }\r
 828 \r
 829                                                 c = escape (c);\r
 830                                                 if (c == -1)\r
 831                                                         return Token.ERROR;\r
 832                                                 s.Append ((char) c);\r
 833                                         }\r
 834                                 }\r
 835 \r
 836                                 if (c == '\''){\r
 837                                         c = getChar ();\r
 838                                         if (c == '\''){\r
 839                                                 error_details = "CS1011: Empty character literal";\r
 840                                                 return Token.ERROR;\r
 841                                         }\r
 842                                         c = escape (c);\r
 843                                         if (c == -1)\r
 844                                                 return Token.ERROR;\r
 845                                         val = new System.Char ();\r
 846                                         val = (char) c;\r
 847                                         c = getChar ();\r
 848                                         if (c != '\''){\r
 849                                                 error_details = "CS1012: Too many characters in character literal";\r
 850                                                 // Try to recover, read until newline or next "'"\r
 851                                                 while ((c = getChar ()) != -1){\r
 852                                                         if (c == '\n' || c == '\'')\r
 853                                                                 break;\r
 854                                                         \r
 855                                                 }\r
 856                                                 return Token.ERROR;\r
 857                                         }\r
 858                                         return Token.LITERAL_CHARACTER;\r
 859                                 }\r
 860                                 \r
 861                                 // white space\r
 862                                 if (c == '\n'){\r
 863                                         line++;\r
 864                                         ref_line++;\r
 865                                         col = 0;\r
 866                                         continue;\r
 867                                 }\r
 868                                 if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r'){\r
 869                                         if (c == '\t')\r
 870                                                 col = (((col + 8) / 8) * 8) - 1;\r
 871                                         \r
 872                                         continue;\r
 873                                 }\r
 874 \r
 875                                 if (c == '@'){\r
 876                                         allow_keyword_as_ident = true;\r
 877                                         continue;\r
 878                                 }\r
 879 \r
 880                                 error_details = ((char)c).ToString ();\r
 881                                 \r
 882                                 return Token.ERROR;\r
 883                         }\r
 884 \r
 885                         return Token.EOF;\r
 886                 }\r
 887         }\r
 888 }\r
 889 \r