mcs/class/Microsoft.JScript/Microsoft.JScript/TokenStream.cs

   1 //
   2 // TokenStream.cs: Port of Mozilla's Rhino TokenStream
   3 //                 This class implements the JScript scanner
   4 //
   5 // Author:
   6 //      Cesar Lopez Nataren (cesar@ciencias.unam.mx)
   7 //
   8 // (C) 2004, Cesar Lopez Nataren
   9 //
  10
  11 //
  12 // Permission is hereby granted, free of charge, to any person obtaining
  13 // a copy of this software and associated documentation files (the
  14 // "Software"), to deal in the Software without restriction, including
  15 // without limitation the rights to use, copy, modify, merge, publish,
  16 // distribute, sublicense, and/or sell copies of the Software, and to
  17 // permit persons to whom the Software is furnished to do so, subject to
  18 // the following conditions:
  19 //
  20 // The above copyright notice and this permission notice shall be
  21 // included in all copies or substantial portions of the Software.
  22 //
  23 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  27 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  28 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  29 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  30 //
  31
  32 using System;
  33 using System.IO;
  34 using System.Collections;
  35 using System.Globalization;
  36
  37 namespace Microsoft.JScript {
  38 internal class TokenStream {
  39
  40         //
  41         // fields
  42         //
  43         string source_name;
  44         internal string SourceName {
  45                 get { return source_name; }
  46                 set { source_name = value; }
  47         }
  48
  49         int line_number;
  50         internal int LineNumber {
  51                 get { return line_number; }
  52                 set { line_number = value; }
  53         }
  54
  55         bool hit_eof;
  56         internal bool EOF {
  57                 get { return hit_eof; }
  58         }
  59
  60         int token_number;
  61         internal int TokenNumber {
  62                 get { return token_number; }
  63                 set { token_number = value; }
  64         }
  65
  66         int pushback_token;
  67
  68         // tokenize newlines
  69         bool significant_eol;
  70
  71         int string_buffer_top;
  72         char [] string_buffer = new char [128];
  73
  74         // Room backtrace from to < on failed match of the last - in <!--
  75         int [] unget_buffer = new int [3];
  76         int unget_cursor;
  77
  78         int line_start;
  79         int line_end_char;
  80
  81         string source_string;
  82         char [] source_buffer;
  83         int source_end;
  84         int source_cursor;
  85
  86         static int EOF_CHAR = -1;
  87         static int EOL_HINT_MASK = 0xdfd0;
  88
  89         StreamReader source_reader;
  90
  91         bool dirty_line;
  92
  93         string _string;
  94         internal string GetString {
  95                 get { return _string; }
  96         }
  97
  98         static bool reserved_keyword_as_identifier;
  99
 100         double number;
 101         internal double GetNumber {
 102                 get { return number; }
 103         }
 104
 105
 106         int op;
 107         internal int GetOp ()
 108         {
 109                 return op;
 110         }
 111
 112         internal bool allow_reg_exp;
 113
 114         internal string reg_exp_flags;
 115
 116         //
 117         // methods
 118         //
 119
 120         internal TokenStream (StreamReader source_reader, string source_string, string source_name, int line_number)
 121         {
 122                 pushback_token = Token.EOF;
 123                 SourceName = source_name;
 124                 this.line_number = line_number;
 125                 if (source_reader != null) {
 126                         if (source_string != null)
 127                                 ;
 128                         this.source_reader = source_reader;
 129                         source_buffer = new char [512];
 130                         source_end = 0;
 131                 } else {
 132                         if (source_string == null)
 133                                 ;
 134                         this.source_string = source_string;
 135                         source_end = source_string.Length;
 136                 }
 137                 source_cursor = 0;
 138         }
 139
 140         static bool IsKeyword (string s)
 141         {
 142                 return Token.EOF != StringToKeyword (s);
 143         }
 144
 145         static int StringToKeyword (string name)
 146         {
 147                 // The following assumes that Token.EOF == 0
 148                 int
 149                 Id_break         = Token.BREAK,
 150                 Id_case          = Token.CASE,
 151                 Id_continue      = Token.CONTINUE,
 152                 Id_default       = Token.DEFAULT,
 153                 Id_delete        = Token.DELPROP,
 154                 Id_do            = Token.DO,
 155                 Id_else          = Token.ELSE,
 156                 Id_export        = Token.EXPORT,
 157                 Id_false         = Token.FALSE,
 158                 Id_for           = Token.FOR,
 159                 Id_function      = Token.FUNCTION,
 160                 Id_if            = Token.IF,
 161                 Id_in            = Token.IN,
 162                 Id_new           = Token.NEW,
 163                 Id_null          = Token.NULL,
 164                 Id_return        = Token.RETURN,
 165                 Id_switch        = Token.SWITCH,
 166                 Id_this          = Token.THIS,
 167                 Id_true          = Token.TRUE,
 168                 Id_typeof        = Token.TYPEOF,
 169                 Id_var           = Token.VAR,
 170                 Id_void          = Token.VOID,
 171                 Id_while         = Token.WHILE,
 172                 Id_with          = Token.WITH,
 173
 174                 // the following are #ifdef RESERVE_JAVA_KEYWORDS in jsscan.c
 175                 Id_abstract      = Token.RESERVED,
 176                 Id_boolean       = Token.RESERVED,
 177                 Id_byte          = Token.RESERVED,
 178                 Id_catch         = Token.CATCH,
 179                 Id_char          = Token.RESERVED,
 180                 Id_class         = Token.RESERVED,
 181                 Id_const         = Token.RESERVED,
 182                 Id_debugger      = Token.RESERVED,
 183                 Id_double        = Token.RESERVED,
 184                 Id_enum          = Token.RESERVED,
 185                 Id_extends       = Token.RESERVED,
 186                 Id_final         = Token.RESERVED,
 187                 Id_finally       = Token.FINALLY,
 188                 Id_float         = Token.RESERVED,
 189                 Id_goto          = Token.RESERVED,
 190                 Id_implements    = Token.RESERVED,
 191                 Id_import        = Token.IMPORT,
 192                 Id_instanceof    = Token.INSTANCEOF,
 193                 Id_int           = Token.RESERVED,
 194                 Id_interface     = Token.RESERVED,
 195                 Id_long          = Token.RESERVED,
 196                 Id_native        = Token.RESERVED,
 197                 Id_package       = Token.RESERVED,
 198                 Id_private       = Token.RESERVED,
 199                 Id_protected     = Token.RESERVED,
 200                 Id_public        = Token.RESERVED,
 201                 Id_short         = Token.RESERVED,
 202                 Id_static        = Token.RESERVED,
 203                 Id_super         = Token.RESERVED,
 204                 Id_synchronized  = Token.RESERVED,
 205                 Id_throw         = Token.THROW,
 206                 Id_throws        = Token.RESERVED,
 207                 Id_transient     = Token.RESERVED,
 208                 Id_try           = Token.TRY,
 209                 Id_volatile      = Token.RESERVED;
 210
 211                 int id;
 212                 string s = name;
 213
 214                 L0: {
 215                         id = 0;
 216                         string X = String.Empty;
 217                         int c;
 218
 219                         L: {
 220                                 switch (s.Length) {
 221                                 case 2: c = s [1];
 222                                         if (c == 'f') {
 223                                                 if (s [0] == 'i') {
 224                                                         id = Id_if;
 225                                                         goto LEAVE_L0;
 226                                                 }
 227                                         } else if (c == 'n') {
 228                                                 if (s [0] == 'i') {
 229                                                         id = Id_in;
 230                                                         goto LEAVE_L0;
 231                                                 }
 232                                         } else if (c == 'o') {
 233                                                 if (s [0] == 'd') {
 234                                                         id = Id_do;
 235                                                         goto LEAVE_L0;
 236                                                 }
 237                                         }
 238                                         goto LEAVE_L;
 239                                 case 3:
 240                                         switch (s [0]) {
 241                                         case 'f':
 242                                                 if (s [2] == 'r' && s [1] == 'o') {
 243                                                         id = Id_for;
 244                                                         goto LEAVE_L0;
 245                                                 }
 246                                         goto LEAVE_L;
 247                                         case 'i':
 248                                                 if (s [2] == 't' && s [1] == 'n') {
 249                                                         id = Id_int;
 250                                                         goto LEAVE_L0;
 251                                                 }
 252                                                 goto LEAVE_L;
 253                                         case 'n':
 254                                                 if (s [2] == 'w' && s [1] == 'e') {
 255                                                         id = Id_new;
 256                                                         goto LEAVE_L0;
 257                                                 }
 258                                                 goto LEAVE_L;
 259                                         case 't':
 260                                                 if (s [2] == 'y' && s [1] == 'r') {
 261                                                         id = Id_try;
 262                                                         goto LEAVE_L0;
 263                                                 }
 264                                                 goto LEAVE_L;
 265                                         case 'v':
 266                                                 if (s [2] == 'r' && s [1] == 'a') {
 267                                                         id = Id_var;
 268                                                         goto LEAVE_L0;
 269                                                 }
 270                                                 goto LEAVE_L;
 271                                         }
 272                                         goto LEAVE_L;
 273                                 case 4:
 274                                         switch (s [0]) {
 275                                         case 'b':
 276                                                 X = "byte";
 277                                                 id = Id_byte;
 278                                                 goto LEAVE_L;
 279                                         case 'c':
 280                                                 c = s [3];
 281                                                 if (c == 'e') {
 282                                                         if (s [2] == 's' && s [1] == 'a') {
 283                                                                 id = Id_case;
 284                                                                 goto LEAVE_L0;
 285                                                         }
 286                                                 } else if (c == 'r') {
 287                                                         if (s [2] == 'a' && s [1] == 'h') {
 288                                                                 id = Id_char;
 289                                                                 goto LEAVE_L0;
 290                                                         }
 291                                                 }
 292                                                 goto LEAVE_L;
 293                                         case 'e':
 294                                                 c = s [3];
 295                                                 if (c == 'e') {
 296                                                         if (s [2] == 's' && s [1] == 'l') {
 297                                                                 id = Id_else;
 298                                                                 goto LEAVE_L0;
 299                                                         }
 300                                                 } else if (c == 'm') {
 301                                                         if (s [2] == 'u' && s [1] == 'n') {
 302                                                                 id = Id_enum;
 303                                                                 goto LEAVE_L0;
 304                                                         }
 305                                                 }
 306                                                 goto LEAVE_L;
 307                                         case 'g':
 308                                                 X = "goto";
 309                                                 id = Id_goto;
 310                                                 goto LEAVE_L;
 311                                         case 'l':
 312                                                 X = "long";
 313                                                 id = Id_long;
 314                                                 goto LEAVE_L;
 315                                         case 't':
 316                                                 c = s [3];
 317                                                 if (c == 'e') {
 318                                                         if (s [2] == 'u' && s [1] == 'r') {
 319                                                                 id = Id_true;
 320                                                                 goto LEAVE_L0;
 321                                                         }
 322                                                 } else if (c == 's') {
 323                                                         if (s [2] == 'i' && s [1] == 'h') {
 324                                                                 id = Id_this;
 325                                                                 goto LEAVE_L0;
 326                                                         }
 327                                                 }
 328                                                 goto LEAVE_L;
 329                                         case 'v':
 330                                                 X = "void";
 331                                                 id = Id_void;
 332                                                 goto LEAVE_L;
 333                                         case 'w':
 334                                                 X = "with";
 335                                                 id = Id_with;
 336                                                 goto LEAVE_L;
 337                                         }
 338                                         goto LEAVE_L;
 339                                 case 5:
 340                                         switch (s [2]) {
 341                                         case 'a':
 342                                                 X = "class";
 343                                                 id = Id_class;
 344                                                 goto LEAVE_L;
 345                                         case 'e':
 346                                                 X = "break";
 347                                                 id = Id_break;
 348                                                 goto LEAVE_L;
 349                                         case 'i':
 350                                                 X = "while";
 351                                                 id = Id_while;
 352                                                 goto LEAVE_L;
 353                                         case 'l':
 354                                                 X = "false";
 355                                                 id = Id_false;
 356                                                 goto LEAVE_L;
 357                                         case 'n':
 358                                                 c = s [0];
 359                                                 if (c == 'c') {
 360                                                         X = "const";
 361                                                         id = Id_const;
 362                                                 } else if (c == 'f') {
 363                                                         X = "final";
 364                                                         id = Id_final;
 365                                                 }
 366                                                 goto LEAVE_L;
 367                                         case 'o':
 368                                                 c = s [0];
 369                                                 if (c == 'f') {
 370                                                         X = "float";
 371                                                         id = Id_float;
 372                                                 } else if (c == 's') {
 373                                                         X = "short";
 374                                                         id = Id_short;
 375                                                 }
 376                                                 goto LEAVE_L;
 377                                         case 'p':
 378                                                 X = "super";
 379                                                 id = Id_super;
 380                                                 goto LEAVE_L;
 381                                         case 'r':
 382                                                 X = "throw";
 383                                                 id = Id_throw;
 384                                                 goto LEAVE_L;
 385                                         case 't':
 386                                                 X = "catch";
 387                                                 id = Id_catch;
 388                                                 goto LEAVE_L;
 389                                         }
 390                                         goto LEAVE_L;
 391                                 case 6:
 392                                         switch (s [1]) {
 393                                         case 'a':
 394                                                 X = "native";
 395                                                 id = Id_native;
 396                                                 goto LEAVE_L;
 397                                         case 'e':
 398                                                 c = s [0];
 399                                                 if (c == 'd') {
 400                                                         X = "delete";
 401                                                         id = Id_delete;
 402                                                 } else if (c == 'r') {
 403                                                         X = "return";
 404                                                         id = Id_return;
 405                                                 }
 406                                                 goto LEAVE_L;
 407                                         case 'h':
 408                                                 X = "throws";
 409                                                 id = Id_throws;
 410                                                 goto LEAVE_L;
 411                                         case 'm':
 412                                                 X = "import";
 413                                                 id = Id_import;
 414                                                 goto LEAVE_L;
 415                                         case 'o':
 416                                                 X = "double";
 417                                                 id = Id_double;
 418                                                 goto LEAVE_L;
 419                                         case 't':
 420                                                 X = "static";
 421                                                 id = Id_static;
 422                                                 goto LEAVE_L;
 423                                         case 'u':
 424                                                 X = "public";
 425                                                 id = Id_public;
 426                                                 goto LEAVE_L;
 427                                         case 'w':
 428                                                 X = "switch";
 429                                                 id = Id_switch;
 430                                                 goto LEAVE_L;
 431                                         case 'x':
 432                                                 X = "export";
 433                                                 id = Id_export;
 434                                                 goto LEAVE_L;
 435                                         case 'y':
 436                                                 X = "typeof";
 437                                                 id = Id_typeof;
 438                                                 goto LEAVE_L;
 439                                         }
 440                                         goto LEAVE_L;
 441                                 case 7:
 442                                         switch (s [1]) {
 443                                         case 'a':
 444                                                 X = "package";
 445                                                 id = Id_package;
 446                                                 goto LEAVE_L;
 447                                         case 'e':
 448                                                 X = "default";
 449                                                 id = Id_default;
 450                                                 goto LEAVE_L;
 451                                         case 'i':
 452                                                 X = "finally";
 453                                                 id = Id_finally;
 454                                                 goto LEAVE_L;
 455                                         case 'o':
 456                                                 X = "boolean";
 457                                                 id = Id_boolean;
 458                                                 goto LEAVE_L;
 459                                         case 'r':
 460                                                 X = "private";
 461                                                 id = Id_private;
 462                                                 goto LEAVE_L;
 463                                         case 'x':
 464                                                 X = "extends";
 465                                                 id = Id_extends;
 466                                                 goto LEAVE_L;
 467                                         }
 468                                         goto LEAVE_L;
 469                                 case 8:
 470                                         switch (s [0]) {
 471                                         case 'a':
 472                                                 X = "abstract";
 473                                                 id = Id_abstract;
 474                                                 goto LEAVE_L;
 475                                         case 'c':
 476                                                 X = "continue";
 477                                                 id = Id_continue;
 478                                                 goto LEAVE_L;
 479                                         case 'd':
 480                                                 X = "debbuger";
 481                                                 id = Id_debugger;
 482                                                 goto LEAVE_L;
 483                                         case 'f':
 484                                                 X = "function";
 485                                                 id = Id_function;
 486                                                 goto LEAVE_L;
 487                                         case 'v':
 488                                                 X = "volatile";
 489                                                 id = Id_volatile;
 490                                                 goto LEAVE_L;
 491                                         }
 492                                         goto LEAVE_L;
 493                                 case 9:
 494                                         c = s [0];
 495                                         if (c == 'i') {
 496                                                 X = "interface";
 497                                                 id = Id_interface;
 498                                         } else if (c == 'p') {
 499                                                 X = "protected";
 500                                                 id = Id_protected;
 501                                         } else if (c == 't') {
 502                                                 X = "transient";
 503                                                 id = Id_transient;
 504                                         }
 505                                         goto LEAVE_L;
 506                                 case 10:
 507                                         c = s [1];
 508                                         if (c == 'm') {
 509                                                 X = "implements";
 510                                                 id = Id_implements;
 511                                         } else if (c == 'n') {
 512                                                 X = "instanceof";
 513                                                 id = Id_instanceof;
 514                                         }
 515                                         goto LEAVE_L;
 516                                 case 12:
 517                                         X = "synchronized";
 518                                         id = Id_synchronized;
 519                                         goto LEAVE_L;
 520                                 }
 521                         }
 522                         LEAVE_L:
 523                                 if (X != null && X != s && !X.Equals (s))
 524                                         id = 0;
 525                 }
 526                 LEAVE_L0:
 527                 if (id == 0)
 528                         return Token.EOF;
 529                 return id & 0xff;
 530         }
 531
 532
 533         //
 534         // return and pop the token from the stream if it matches otherwise return null
 535         //
 536         internal bool MatchToken (int to_match)
 537         {
 538                 int token = GetToken ();
 539                 if (token == to_match)
 540                         return true;
 541                 // did not match, push back the token
 542                 TokenNumber--;
 543                 pushback_token = token;
 544                 return false;
 545         }
 546
 547         internal void UnGetToken (int tt)
 548         {
 549                 // Can not unreadmore than one token
 550                 if (pushback_token != Token.EOF && tt != Token.ERROR)
 551                         ;
 552                 pushback_token = tt;
 553                 TokenNumber--;
 554         }
 555
 556         internal int PeekToken ()
 557         {
 558                 int result = GetToken ();
 559                 pushback_token = result;
 560                 TokenNumber--;
 561                 return result;
 562         }
 563
 564         internal int PeekTokenSameLine ()
 565         {
 566                 significant_eol = true;
 567                 int result = GetToken ();
 568                 pushback_token = result;
 569                 TokenNumber--;
 570                 significant_eol = false;
 571                 return result;
 572         }
 573
 574         internal int GetToken ()
 575         {
 576                 int c;
 577                 TokenNumber++;
 578
 579                 // Check for pushed-back token
 580                 if (pushback_token != Token.EOF) {
 581                         int result = pushback_token;
 582                         pushback_token = Token.EOF;
 583                         if (result != Token.EOL || significant_eol)
 584                                 return result;
 585                 }
 586
 587                 retry:
 588                         for (;;) {
 589                                 // Eat whitespace, possibly sensitive to newlines
 590                                 for (;;) {
 591                                         c = GetChar ();
 592                                         if (c == EOF_CHAR)
 593                                                 return Token.EOF;
 594                                         else if (c == '\n') {
 595                                                 dirty_line = false;
 596                                                 if (significant_eol)
 597                                                         return Token.EOL;
 598                                         } else if (!IsJSSpace (c)) {
 599                                                 if (c != '-')
 600                                                         dirty_line = true;
 601                                                 break;
 602                                         }
 603                                 }
 604
 605                                 // identifier/keyword/instanceof?
 606                                 // watch out for starting with a <backslash>
 607                                 bool identifier_start;
 608                                 bool is_unicode_escape_start = false;
 609
 610                                 if (c == '\\') {
 611                                         c = GetChar ();
 612                                         if (c == 'u') {
 613                                                 identifier_start = true;
 614                                                 is_unicode_escape_start = true;
 615                                                 string_buffer_top = 0;
 616                                         } else {
 617                                                 identifier_start = false;
 618                                                 UnGetChar (c);
 619                                                 c = '\\';
 620                                         }
 621                                 } else {
 622                                         identifier_start = IsJavaIdentifierStart ((char) c);
 623                                         if (identifier_start) {
 624                                                 string_buffer_top = 0;
 625                                                 AddToString (c);
 626                                         }
 627                                 }
 628
 629                                 if (identifier_start) {
 630                                         bool contains_escape = is_unicode_escape_start;
 631                                         for (;;) {
 632                                                 if (is_unicode_escape_start) {
 633                                                         // strictly speaking we should probably push-back
 634                                                         // all the bad characters if the <backslash>uXXXX
 635                                                         // sequence is malformed. But since there isn't a
 636                                                         // correct context(is there?) for a bad Unicode
 637                                                         // escape sequence in an identifier, we can report
 638                                                         // an error here.
 639                                                         int escape_val = 0;
 640                                                         for (int i = 0; i != 4; ++i) {
 641                                                                 c = GetChar ();
 642                                                                 escape_val = (escape_val << 4) | xDigitToInt (c);
 643                                                                 // Next check takes care about c < 0 and bad escape
 644                                                                 if (escape_val < 0)
 645                                                                         break;
 646                                                         }
 647                                                         if (escape_val < 0) {
 648                                                                 ReportCurrentLineError ("msg.invalid.escape");
 649                                                                 return Token.ERROR;
 650                                                         }
 651                                                         AddToString (escape_val);
 652                                                         is_unicode_escape_start = false;
 653                                                 } else {
 654                                                         c = GetChar ();
 655                                                         if (c == '\\') {
 656                                                                 c = GetChar ();
 657                                                                 if (c == 'u') {
 658                                                                         is_unicode_escape_start = true;
 659                                                                         contains_escape = true;
 660                                                                 } else {
 661                                                                         ReportCurrentLineError ("msg.illegal.character");
 662                                                                         return Token.ERROR;
 663                                                                 }
 664                                                         } else {
 665                                                                 if (c == EOF_CHAR || !IsJavaIdentifierPart ((char) c))
 666                                                                         break;
 667                                                                 AddToString (c);
 668                                                         }
 669                                                 }
 670                                         }
 671                                         UnGetChar (c);
 672
 673                                         string str = GetStringFromBuffer ();
 674                                         if (!contains_escape) {
 675                                                 // OPT we shouldn't have to make a string (object!) to
 676                                                 // check if it's a keyword.
 677
 678                                                 // Return the corresponding token if it's a keyword
 679                                                 int result = StringToKeyword (str);
 680                                                 if (result != Token.EOF) {
 681                                                         if (result != Token.RESERVED)
 682                                                                 return result;
 683                                                         else if (!reserved_keyword_as_identifier)
 684                                                                 return result;
 685                                                         else {
 686                                                                 // If implementation permits to use future reserved
 687                                                                 // keywords in violation with the EcmaScript,
 688                                                                 // treat it as name but issue warning
 689                                                                 ReportCurrentLineWarning ("msg.reserved.keyword", str);
 690                                                                 Console.WriteLine ("Warning: using future reserved keyword as name");
 691                                                         }
 692                                                 }
 693                                         }
 694                                         _string = String.Intern (str);
 695                                         return Token.NAME;
 696                                 }
 697
 698                                 // is it a number?
 699                                 if (IsDigit (c) || (c == '.' && IsDigit (PeekChar ()))) {
 700                                         string_buffer_top = 0;
 701                                         int _base = 10;
 702
 703                                         if (c == '0') {
 704                                                 c = GetChar ();
 705                                                 if (c == 'x' || c == 'X') {
 706                                                         _base = 16;
 707                                                         c = GetChar ();
 708                                                 } else if (IsDigit (c))
 709                                                         _base = 8;
 710                                                 else
 711                                                         AddToString ('0');
 712                                         }
 713
 714                                         if (_base == 16) {
 715                                                 while (0 <= xDigitToInt (c)) {
 716                                                         AddToString (c);
 717                                                         c = GetChar ();
 718                                                 }
 719                                         } else {
 720                                                 while ('0' <= c && c <= '9') {
 721                                                         /*
 722                                                          * We permit 08 and 09 as decimal numbers, which
 723                                                          * makes our behavior a superset of the ECMA
 724                                                          * numeric grammar.  We might not always be so
 725                                                          * permissive, so we warn about it.
 726                                                          */
 727                                                         if (_base == 8 && c >= '8') {
 728                                                                 ReportCurrentLineWarning ("msg.bad.octal.literal", c == '8' ? "8" : "9");
 729                                                                 _base = 10;
 730                                                         }
 731                                                         AddToString (c);
 732                                                         c = GetChar ();
 733                                                 }
 734                                         }
 735
 736                                         bool is_integer = true;
 737
 738                                         if (_base == 10 && (c == '.' || c == 'e' || c == 'E')) {
 739                                                 is_integer = false;
 740                                                 if (c == '.') {
 741                                                         do {
 742                                                                 AddToString (c);
 743                                                                 c = GetChar ();
 744                                                         } while (IsDigit (c));
 745                                                 }
 746                                                 if (c == 'e' || c == 'E') {
 747                                                         AddToString (c);
 748                                                         c = GetChar ();
 749                                                         if (c == '+' || c == '-') {
 750                                                                 AddToString (c);
 751                                                                 c = GetChar ();
 752                                                         }
 753                                                         if (!IsDigit (c)) {
 754                                                                 ReportCurrentLineError ("msg.missing.exponent");
 755                                                                 return Token.ERROR;
 756                                                         }
 757                                                         do {
 758                                                                 AddToString (c);
 759                                                                 c = GetChar ();
 760                                                         } while (IsDigit (c));
 761                                                 }
 762                                         }
 763                                         UnGetChar (c);
 764                                         string num_string = GetStringFromBuffer ();
 765
 766                                         double dval;
 767                                         if (_base == 10 && !is_integer) {
 768                                                 try {
 769                                                         // Use C# conversion to number from string
 770                                                         dval = Double.Parse (num_string, CultureInfo.InvariantCulture);
 771                                                 } catch (FormatException ex) {
 772                                                         ReportCurrentLineError ("msg.caught.nfe");
 773                                                         return Token.ERROR;
 774                                                 } catch (OverflowException) {
 775                                                         dval = Double.NaN;
 776                                                 }
 777                                         } else
 778                                                 dval = StringToNumber (num_string, 0, _base);
 779
 780                                         number = dval;
 781                                         return Token.NUMBER;
 782                                 }
 783
 784                                 // is it a string?
 785                                 if (c == '"' || c == '\'') {
 786                                         // We attempt to accumulate a string the fast way, by
 787                                         // building it directly out of the reader.  But if there
 788                                         // are any escaped characters in the string, we revert to
 789                                         // building it out of a StringBuffer.
 790
 791                                         int quote_char = c;
 792                                         string_buffer_top = 0;
 793                                         c = GetChar ();
 794
 795                                 strLoop: while (c != quote_char) {
 796                                         if (c == '\n' || c == EOF_CHAR) {
 797                                                 UnGetChar (c);
 798                                                 ReportCurrentLineError ("msg.unterminated.string.lit");
 799                                                 return Token.ERROR;
 800                                         }
 801
 802                                         if (c == '\\') {
 803                                                 // We've hit an escaped character
 804                                                 int escape_val;
 805
 806                                                 c = GetChar ();
 807                                                 switch (c) {
 808                                                 case 'b': c = '\b'; break;
 809                                                 case 'f': c = '\f'; break;
 810                                                 case 'n': c = '\n'; break;
 811                                                 case 'r': c = '\r'; break;
 812                                                 case 't': c = '\t'; break;
 813
 814                                                 // \v a late addition to the ECMA spec,
 815                                                 // it is not in Java, so use 0xb
 816                                                 case 'v': c = 0xb; break;
 817
 818                                                 case 'u':
 819                                                         // Get 4 hex digits; if the u escape is not
 820                                                         // followed by 4 hex digits, use 'u' + the
 821                                                         // literal character sequence that follows.
 822                                                         int escape_start = string_buffer_top;
 823                                                         AddToString ('u');
 824                                                         escape_val = 0;
 825                                                         for (int i = 0; i != 4; ++i) {
 826                                                                 c = GetChar ();
 827                                                                 escape_val = (escape_val << 4) | xDigitToInt (c);
 828                                                                 if (escape_val < 0)
 829                                                                         goto strLoop;
 830                                                                 AddToString (c);
 831                                                         }
 832                                                         // prepare for replace of stored 'u' sequence
 833                                                         // by escape value
 834                                                         string_buffer_top = escape_start;
 835                                                         c = escape_val;
 836                                                         break;
 837                                                 case 'x':
 838                                                         // Get 2 hex digits, defaulting to 'x'+literal
 839                                                         // sequence, as above.
 840                                                         c = GetChar ();
 841                                                         escape_val = xDigitToInt (c);
 842                                                         if (escape_val < 0) {
 843                                                                 AddToString ('x');
 844                                                                 goto strLoop;
 845                                                         } else {
 846                                                                 int c1 = c;
 847                                                                 c = GetChar ();
 848                                                                 escape_val = (escape_val << 4) | xDigitToInt (c);
 849                                                                 if (escape_val < 0) {
 850                                                                         AddToString ('x');
 851                                                                         AddToString (c1);
 852                                                                         goto strLoop;
 853                                                                 } else // got 2 hex digits
 854                                                                         c = escape_val;
 855                                                         }
 856                                                         break;
 857                                                 default:
 858                                                         if ('0' <= c && c < '8') {
 859                                                                 int val = c - '0';
 860                                                                 c = GetChar ();
 861                                                                 if ('0' <= c && c < '8') {
 862                                                                         val = 8 * val + c - '0';
 863                                                                         c = GetChar ();
 864                                                                         if ('0' <= c && c < '8' && val <= 037) {
 865                                                                                 // c is 3rd char of octal sequence only
 866                                                                                 // if the resulting val <= 0377
 867                                                                                 val = 8 * val + c - '0';
 868                                                                                 c = GetChar ();
 869                                                                         }
 870                                                                 }
 871                                                                 UnGetChar (c);
 872                                                                 c = val;
 873                                                         }
 874                                                         break;
 875                                                 }
 876                                         }
 877                                         AddToString (c);
 878                                         c = GetChar ();
 879                                 }
 880                                         string str = GetStringFromBuffer ();
 881                                         _string = String.Intern (str);
 882                                         return Token.STRING;
 883                                 }
 884
 885                                 switch (c) {
 886                                 case ';': return Token.SEMI;
 887                                 case '[': return Token.LB;
 888                                 case ']': return Token.RB;
 889                                 case '{': return Token.LC;
 890                                 case '}': return Token.RC;
 891                                 case '(': return Token.LP;
 892                                 case ')': return Token.RP;
 893                                 case ',': return Token.COMMA;
 894                                 case '?': return Token.HOOK;
 895                                 case ':': return Token.COLON;
 896                                 case '.': return Token.DOT;
 897
 898                                 case '|':
 899                                         if (MatchChar ('|'))
 900                                                 return Token.OR;
 901                                         else if (MatchChar ('=')) {
 902                                                 op = Token.BITOR;
 903                                                 return Token.ASSIGNOP;
 904                                         } else
 905                                                 return Token.BITOR;
 906
 907                                 case '^':
 908                                         if (MatchChar ('=')) {
 909                                                 op = Token.BITXOR;
 910                                                 return Token.ASSIGNOP;
 911                                         } else
 912                                                 return Token.BITXOR;
 913
 914                                 case '&':
 915                                         if (MatchChar ('&'))
 916                                                 return Token.AND;
 917                                         else if (MatchChar ('=')) {
 918                                                 op = Token.BITAND;
 919                                                 return Token.ASSIGNOP;
 920                                         } else
 921                                                 return Token.BITAND;
 922
 923                                 case '=':
 924                                         if (MatchChar ('=')) {
 925                                                 if (MatchChar ('='))
 926                                                         return Token.SHEQ;
 927                                                 else
 928                                                         return Token.EQ;
 929                                         } else
 930                                                 return Token.ASSIGN;
 931
 932                                 case '!':
 933                                         if (MatchChar ('=')) {
 934                                                 if (MatchChar ('='))
 935                                                         return Token.SHNE;
 936                                                 else
 937                                                         return Token.NE;
 938                                         } else
 939                                                 return Token.NOT;
 940
 941                                 case '<':
 942                                         /* NB:treat HTML begin-comment as comment-till-eol */
 943                                         if (MatchChar ('!')) {
 944                                                 if (MatchChar ('-')) {
 945                                                         if (MatchChar ('-')) {
 946                                                                 SkipLine ();
 947                                                                 goto retry;
 948                                                         }
 949                                                         UnGetChar ('-');
 950                                                 }
 951                                                 UnGetChar ('!');
 952                                         }
 953                                         if (MatchChar ('<')) {
 954                                                 if (MatchChar ('=')) {
 955                                                         op = Token.LSH;
 956                                                         return Token.ASSIGNOP;
 957                                                 } else
 958                                                         return Token.LSH;
 959                                         } else {
 960                                                 if (MatchChar ('='))
 961                                                         return Token.LE;
 962                                                 else
 963                                                         return Token.LT;
 964                                         }
 965
 966                                 case '>':
 967                                         if (MatchChar ('>')) {
 968                                                 if (MatchChar ('>')) {
 969                                                         if (MatchChar ('=')) {
 970                                                                 op = Token.URSH;
 971                                                                 return Token.ASSIGNOP;
 972                                                         } else
 973                                                                 return Token.URSH;
 974                                                 } else {
 975                                                         if (MatchChar ('=')) {
 976                                                                 op = Token.RSH;
 977                                                                 return Token.ASSIGNOP;
 978                                                         } else
 979                                                                 return Token.RSH;
 980                                                 }
 981                                         } else {
 982                                                 if (MatchChar ('='))
 983                                                         return Token.GE;
 984                                                 else
 985                                                         return Token.GT;
 986                                         }
 987
 988                                 case '*':
 989                                         if (MatchChar ('=')) {
 990                                                 op = Token.MUL;
 991                                                 return Token.ASSIGNOP;
 992                                         } else
 993                                                 return Token.MUL;
 994
 995                                 case '/':
 996                                         // is it a // comment?
 997                                         if (MatchChar ('/')) {
 998                                                 SkipLine ();
 999                                                 goto retry;
1000                                         }
1001                                         if (MatchChar ('*')) {
1002                                                 bool look_for_slash = false;
1003                                                 for (;;) {
1004                                                         c = GetChar ();
1005                                                         if (c == EOF_CHAR) {
1006                                                                 ReportCurrentLineError ("msg.unterminated.comment");
1007                                                                 return Token.ERROR;
1008                                                         } else if (c == '*')
1009                                                                 look_for_slash = true;
1010                                                         else if (c == '/') {
1011                                                                 if (look_for_slash)
1012                                                                         goto retry;
1013                                                         } else
1014                                                                 look_for_slash = false;
1015                                                 }
1016                                         }
1017
1018                                         // is it a RegExp?
1019                                         if (allow_reg_exp) {
1020                                                 string_buffer_top = 0;
1021                                                 while ((c = GetChar ()) != '/') {
1022                                                         if (c == '\n' || c == EOF_CHAR) {
1023                                                                 UnGetChar (c);
1024                                                                 ReportCurrentLineError ("msg.unterminated.re.lit");
1025                                                                 return Token.ERROR;
1026                                                         }
1027                                                         if (c == '\\') {
1028                                                                 AddToString (c);
1029                                                                 c = GetChar ();
1030                                                         }
1031                                                         AddToString (c);
1032                                                 }
1033                                                 int re_end = string_buffer_top;
1034
1035                                                 while (true) {
1036                                                         if (MatchChar ('g'))
1037                                                                 AddToString ('g');
1038                                                         else if (MatchChar ('i'))
1039                                                                 AddToString ('i');
1040                                                         else if (MatchChar ('m'))
1041                                                                 AddToString ('m');
1042                                                         else
1043                                                                 break;
1044                                                 }
1045
1046                                                 if (IsAlpha (PeekChar ())) {
1047                                                         ReportCurrentLineError ("msg.invalid.re.flag");
1048                                                         return Token.ERROR;
1049                                                 }
1050
1051                                                 _string = to_string (string_buffer).Substring (0, re_end);
1052                                                 reg_exp_flags = to_string (string_buffer).Substring (re_end, string_buffer_top - re_end);
1053                                                 return Token.REGEXP;
1054                                         }
1055
1056                                         if (MatchChar ('=')) {
1057                                                 op = Token.DIV;
1058                                                 return Token.ASSIGNOP;
1059                                         } else
1060                                                 return Token.DIV;
1061
1062                                 case '%':
1063                                         if (MatchChar ('=')) {
1064                                                 op = Token.MOD;
1065                                                 return Token.ASSIGNOP;
1066                                         } else
1067                                                 return Token.MOD;
1068
1069                                 case '~':
1070                                         return Token.BITNOT;
1071
1072                                 case '+':
1073                                         if (MatchChar ('=')) {
1074                                                 op = Token.ADD;
1075                                                 return Token.ASSIGNOP;
1076                                         } else if (MatchChar ('+'))
1077                                                 return Token.INC;
1078                                         else
1079                                                 return Token.ADD;
1080
1081                                 case '-':
1082                                         if (MatchChar ('=')) {
1083                                                 op = Token.SUB;
1084                                                 c = Token.ASSIGNOP;
1085                                         } else if (MatchChar ('-')) {
1086                                                 if (!dirty_line) {
1087                                                         // treat HTML end-comment after possible whitespace
1088                                                         // after line start as comment-utill-eol
1089                                                         if (MatchChar ('>')) {
1090                                                                 SkipLine ();
1091                                                                 goto retry;
1092                                                         }
1093                                                 }
1094                                                 c = Token.DEC;
1095                                         } else
1096                                                 c = Token.SUB;
1097                                         dirty_line = true;
1098                                         return c;
1099
1100                                 default:
1101                                         ReportCurrentLineError ("msg.illegal.character");
1102                                         return Token.ERROR;
1103                                 }
1104                         }
1105         }
1106
1107
1108         static bool IsAlpha (int c)
1109         {
1110                 // Use 'Z' < 'a'
1111                 if (c <= 'Z')
1112                         return 'A' <= c;
1113                 else
1114                         return 'a' <= c && c <= 'z';
1115         }
1116
1117         double StringToNumber (string s, int start, int radix)
1118         {
1119                 char digit_max = '9';
1120                 char lower_case_bound = 'a';
1121                 char upper_case_bound = 'A';
1122                 int len = s.Length;
1123
1124                 if (radix > 10) {
1125                         lower_case_bound = (char) ('a' + radix - 10);
1126                         upper_case_bound = (char) ('A' + radix - 10);
1127                 }
1128
1129                 int end;
1130                 double sum = 0.0;
1131
1132                 for (end = start; end < len; end++) {
1133                         char c = s [end];
1134                         int new_digit;
1135                         if ('0' <= c && c <= digit_max)
1136                                 new_digit = c - '0';
1137                         else if ('a' <= c && c < lower_case_bound)
1138                                 new_digit = c - 'a' + 10;
1139                         else if ('A' <= c && c < upper_case_bound)
1140                                 new_digit = c - 'A' + 10;
1141                         else
1142                                 break;
1143                         sum = sum * radix + new_digit;
1144                 }
1145
1146                 if (start == end)
1147                         return Double.NaN;
1148
1149                 if (sum >= 9007199254740992.0) {
1150                         if (radix == 10) {
1151                                 /* If we're accumulating a decimal number and the number
1152                                  * is >= 2^53, then the result from the repeated multiply-add
1153                                  * above may be inaccurate.  Call Java to get the correct
1154                                  * answer.
1155                                  */
1156                                 try {
1157                                         return Double.Parse (s, CultureInfo.InvariantCulture);
1158                                 } catch (FormatException fe) {
1159                                         return Double.NaN;
1160                                 }
1161                         } else if (radix == 2 || radix == 4 || radix == 8 ||
1162                                    radix == 16 || radix == 32) {
1163                                 /* The number may also be inaccurate for one of these bases.
1164                                  * This happens if the addition in value*radix + digit causes
1165                                  * a round-down to an even least significant mantissa bit
1166                                  * when the first dropped bit is a one.  If any of the
1167                                  * following digits in the number (which haven't been added
1168                                  * in yet) are nonzero then the correct action would have
1169                                  * been to round up instead of down.  An example of this
1170                                  * occurs when reading the number 0x1000000000000081, which
1171                                  * rounds to 0x1000000000000000 instead of 0x1000000000000100.
1172                                  */
1173                                 int bit_shift_in_char = 1;
1174                                 int digit = 0;
1175
1176                                 const int SKIP_LEADING_ZEROS = 0;
1177                                 const int FIRST_EXACT_53_BITS = 1;
1178                                 const int AFTER_BIT_53         = 2;
1179                                 const int ZEROS_AFTER_54 = 3;
1180                                 const int MIXED_AFTER_54 = 4;
1181
1182                                 int state = SKIP_LEADING_ZEROS;
1183                                 int exact_bits_limit = 53;
1184                                 double factor = 0.0;
1185                                 bool bit53 = false;
1186                                 // bit54 is the 54th bit (the first dropped from the mantissa)
1187                                 bool bit54 = false;
1188
1189                                 for (;;) {
1190                                         if (bit_shift_in_char == 1) {
1191                                                 if (start == end)
1192                                                         break;
1193                                                 digit = s [start++];
1194                                                 if ('0' <= digit && digit <= '9')
1195                                                         digit -= '0';
1196                                                 else if ('a' <= digit && digit <= 'z')
1197                                                         digit -= 'a' - 10;
1198                                                 else
1199                                                         digit -= 'A' - 10;
1200                                                 bit_shift_in_char = radix;
1201                                         }
1202                                         bit_shift_in_char >>= 1;
1203                                         bool bit = (digit & bit_shift_in_char) != 0;
1204
1205                                         switch (state) {
1206                                         case SKIP_LEADING_ZEROS:
1207                                                 if (bit) {
1208                                                         --exact_bits_limit;
1209                                                         sum = 1.0;
1210                                                         state = FIRST_EXACT_53_BITS;
1211                                                 }
1212                                                 break;
1213                                         case FIRST_EXACT_53_BITS:
1214                                                 sum *= 2.0;
1215                                                 if (bit)
1216                                                         sum += 1.0;
1217                                                 --exact_bits_limit;
1218                                                 if (exact_bits_limit == 0) {
1219                                                         bit53 = bit;
1220                                                         state = AFTER_BIT_53;
1221                                                 }
1222                                                 break;
1223                                         case AFTER_BIT_53:
1224                                                 bit54 = bit;
1225                                                 factor = 2.0;
1226                                                 state = ZEROS_AFTER_54;
1227                                                 break;
1228                                         // FIXME: check if this work
1229                                         case ZEROS_AFTER_54:
1230                                         case MIXED_AFTER_54:
1231                                                 if (state == ZEROS_AFTER_54 && bit) {
1232                                                         state = MIXED_AFTER_54;
1233                                                 }
1234                                                 // fallthrough
1235                                                 factor *= 2;
1236                                                 break;
1237                                         }
1238                                 }
1239                                 switch (state) {
1240                                 case SKIP_LEADING_ZEROS:
1241                                         sum = 0.0;
1242                                         break;
1243                                 case FIRST_EXACT_53_BITS:
1244                                 case AFTER_BIT_53:
1245                                         // do nothing
1246                                         break;
1247                                 case ZEROS_AFTER_54:
1248                                         // x1.1 -> x1 + 1 (round up)
1249                                         // x0.1 -> x0 (round down)
1250                                         if (bit54 & bit53)
1251                                                 sum += 1.0;
1252                                         sum *= factor;
1253                                         break;
1254                                 case MIXED_AFTER_54:
1255                                         // x.100...1.. -> x + 1 (round up)
1256                                         // x.0anything -> x (round down)
1257                                         if (bit54)
1258                                                 sum += 1.0;
1259                                         sum *= factor;
1260                                         break;
1261                                 }
1262                         }
1263                         /* We don't worry about inaccurate numbers for any other base. */
1264                 }
1265                 return sum;
1266         }
1267
1268         bool IsDigit (int c)
1269         {
1270                 return '0' <= c && c <= '9';
1271         }
1272
1273         static int xDigitToInt (int c)
1274         {
1275                 // use 0..9 < A..Z < a..z
1276                 if (c <= '9') {
1277                         c -= '0';
1278                         if (0 <= c)
1279                                 return c;
1280                 } else if (c <= 'F') {
1281                         if ('A' <= c)
1282                                 return c - ('A' - 10);
1283                 } else if (c <= 'f') {
1284                         if ('a' <= c)
1285                                 return c - ('a' - 10);
1286                 }
1287                 return -1;
1288         }
1289
1290
1291         internal static bool IsJSSpace (int c)
1292         {
1293                 if (c < 127)
1294                         return c == 0x20 || c == 0x9 || c == 0xC || c == 0xB;
1295                 else
1296                         return c == 0xA0 || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.SpaceSeparator;
1297         }
1298
1299         internal static bool IsJSLineTerminator (int c)
1300         {
1301                 return c == '\n' || c == '\r' || c == 0x2028 || c == 0x2029;
1302         }
1303
1304         static bool IsJSFormatChar (int c)
1305         {
1306                 return (c > 127) && (Char.GetUnicodeCategory ((char) c) == UnicodeCategory.Format);
1307         }
1308
1309         string GetStringFromBuffer ()
1310         {
1311                 return new string (string_buffer, 0, string_buffer_top);
1312         }
1313
1314         void AddToString (int c)
1315         {
1316                 int N = string_buffer_top;
1317                 if (N == string_buffer.Length) {
1318                         char [] tmp = new char [string_buffer.Length * 2];
1319                         Array.Copy (string_buffer, 0, tmp, 0, N);
1320                         string_buffer = tmp;
1321                 }
1322                 string_buffer [N] = (char) c;
1323                 string_buffer_top = N + 1;
1324         }
1325
1326         void UnGetChar (int c)
1327         {
1328                 // can not unread past across line boundary
1329                 if (unget_cursor != 0 && unget_buffer [unget_cursor - 1] == '\n')
1330                         ;
1331                 unget_buffer [unget_cursor++] = c;
1332         }
1333
1334         bool MatchChar (int test)
1335         {
1336                 int c = GetChar ();
1337                 if (c == test)
1338                         return true;
1339                 else {
1340                         UnGetChar (c);
1341                         return false;
1342                 }
1343         }
1344
1345         int PeekChar ()
1346         {
1347                 int c = GetChar ();
1348                 UnGetChar (c);
1349                 return c;
1350         }
1351
1352
1353         int GetChar ()
1354         {
1355                 if (unget_cursor != 0)
1356                         return unget_buffer [--unget_cursor];
1357
1358                 for (;;) {
1359                         int c;
1360                         if (source_string != null) {
1361                                 if (source_cursor == source_end) {
1362                                         hit_eof = true;
1363                                         return EOF_CHAR;
1364                                 }
1365                                 c = source_string [source_cursor++];
1366                         } else {
1367                                 if (source_cursor == source_end) {
1368                                         if (!FillSourceBuffer ()) {
1369                                                 hit_eof = true;
1370                                                 return EOF_CHAR;
1371                                         }
1372                                 }
1373                                 c = source_buffer [source_cursor++];
1374                         }
1375
1376                         if (line_end_char >= 0) {
1377                                 if (line_end_char == '\r' && c == '\n') {
1378                                         line_end_char = '\n';
1379                                         continue;
1380                                 }
1381                                 line_end_char = -1;
1382                                 line_start = source_cursor - 1;
1383                                 LineNumber++;
1384                         }
1385
1386                         if (c <= 127) {
1387                                 if (c == '\n' || c == '\r') {
1388                                         line_end_char = c;
1389                                         c = '\n';
1390                                 }
1391                         } else {
1392                                 if (IsJSFormatChar (c))
1393                                         continue;
1394                                 if ((c & EOL_HINT_MASK) == 0 && IsJSLineTerminator (c)) {
1395                                         line_end_char = c;
1396                                         c = '\n';
1397                                 }
1398                         }
1399                         return c;
1400                 }
1401         }
1402
1403         void SkipLine ()
1404         {
1405                 // skip to end of line
1406                 int c;
1407                 while ((c = GetChar ()) != EOF_CHAR && c != '\n')
1408                         ;
1409                 UnGetChar (c);
1410         }
1411
1412         bool FillSourceBuffer ()
1413         {
1414                 if (source_string == null)
1415                         ;
1416                 if (source_end == source_buffer.Length) {
1417                         if (line_start != 0) {
1418                                 Array.Copy (source_buffer, line_start, source_buffer, 0, source_end - line_start);
1419                                 source_end -= line_start;
1420                                 source_cursor -= line_start;
1421                                 line_start = 0;
1422                         } else {
1423                                 char [] tmp = new char [source_buffer.Length *  2];
1424                                 Array.Copy (source_buffer, 0, tmp, 0, source_end);
1425                                 source_buffer = tmp;
1426                         }
1427                 }
1428                 int n = source_reader.Read (source_buffer, source_end, source_buffer.Length - source_end);
1429                 if (n == 0)
1430                         return false;
1431                 source_end += n;
1432                 return true;
1433         }
1434
1435         internal void ReportCurrentLineWarning (string message, string str)
1436         {
1437                 Console.WriteLine ("warning: {0}, {1}, {2}, {3}", message, SourceName, LineNumber, str);
1438         }
1439
1440         internal void ReportCurrentLineError (string message)
1441         {
1442                 Console.WriteLine ("{0} ({1}, 0): error: {2}", SourceName, LineNumber, message);
1443         }
1444
1445         // FIXME: we don't check for combining mark yet
1446         static bool IsJavaIdentifierPart (char c)
1447         {
1448                 UnicodeCategory unicode_category = Char.GetUnicodeCategory (c);
1449                 return Char.IsLetter (c) || unicode_category == UnicodeCategory.CurrencySymbol ||
1450                         unicode_category == UnicodeCategory.ConnectorPunctuation || Char.IsDigit (c) ||
1451                         unicode_category == UnicodeCategory.LetterNumber ||
1452                         unicode_category == UnicodeCategory.NonSpacingMark || IsIdentifierIgnorable (c);
1453         }
1454
1455         static bool IsIdentifierIgnorable (char c)
1456         {
1457                 return (c >= '\u0000' && c <= '\u0008') || (c >= '\u000E' && c <= '\u001B') ||
1458                         (c >= '\u007F' && c <= '\u009F') || Char.GetUnicodeCategory (c) == UnicodeCategory.Format;
1459         }
1460
1461         static bool IsJavaIdentifierStart (char c)
1462         {
1463                 UnicodeCategory unicode_category = Char.GetUnicodeCategory (c);
1464                 return Char.IsLetter (c) || unicode_category == UnicodeCategory.LetterNumber ||
1465                         unicode_category == UnicodeCategory.CurrencySymbol ||
1466                         unicode_category == UnicodeCategory.ConnectorPunctuation;
1467         }
1468
1469         internal static string to_string (Array a)
1470         {
1471                 string s = String.Empty;
1472                 foreach (char c in a)
1473                         s += c;
1474                 return s;
1475         }
1476 }
1477 }