mcs/mbas/mb-tokenizer.cs

   1 //
   2 // Mono.MonoBASIC.Tokenizer.cs: The Tokenizer for the MonoBASIC compiler
   3 //
   4 // Author: A Rafael D Teixeira (rafaelteixeirabr@hotmail.com)
   5 //
   6 // Based on cs-tokenizer.cs by Miguel de Icaza (miguel@gnu.org)
   7 //
   8 // Licensed under the terms of the GNU GPL
   9 //
  10 // Copyright (C) 2001 A Rafael D Teixeira
  11 //
  12
  13 namespace Mono.MonoBASIC
  14 {
  15         using System;
  16         using System.Text;
  17         using System.Collections;
  18         using System.IO;
  19         using System.Globalization;
  20         using Mono.Languages;
  21         using Mono.MonoBASIC;
  22
  23         /// <summary>
  24         ///    Tokenizer for MonoBASIC source code.
  25         /// </summary>
  26
  27         public class Tokenizer : yyParser.yyInput
  28         {
  29                 TextReader reader;
  30                 string file_name;
  31                 string ref_name;
  32                 int ref_line = 0;
  33                 int line = 0;
  34                 int col = 1;
  35                 public int current_token = Token.ERROR;
  36                 bool handle_get_set = false;
  37                 bool cant_have_a_type_character = false;
  38
  39                 public int ExpandedTabsSize = 4;
  40
  41                 public string location {
  42                         get {
  43                                 string det;
  44
  45                                 if (current_token == Token.ERROR)
  46                                         det = "detail: " + error_details;
  47                                 else
  48                                         det = "";
  49
  50                                 return "Line:     "+line+" Col: "+col + "\n" +
  51                                        "VirtLine: "+ref_line +
  52                                        " Token: "+current_token + " " + det;
  53                         }
  54                 }
  55
  56                 public bool properties {
  57                         get {
  58                                 return handle_get_set;
  59                         }
  60
  61                         set {
  62                                 handle_get_set = value;
  63                         }
  64                 }
  65
  66                 //
  67                 // Class variables
  68                 //
  69                 static Hashtable keywords;
  70                 static NumberStyles styles;
  71                 static NumberFormatInfo csharp_format_info;
  72
  73                 //
  74                 // Values for the associated token returned
  75                 //
  76                 StringBuilder number;
  77                 int putback_char = -1;
  78                 Object val;
  79
  80                 //
  81                 // Details about the error encoutered by the tokenizer
  82                 //
  83                 string error_details;
  84
  85                 public string error {
  86                         get {
  87                                 return error_details;
  88                         }
  89                 }
  90
  91
  92                 public string Source {
  93                         get {
  94                                 return file_name;
  95                         }
  96
  97                         set {
  98                                 file_name = value;
  99                                 ref_name = value;
 100                                 Location.SetCurrentSource(file_name);
 101                         }
 102                 }
 103
 104                 public string EffectiveSource {
 105                         get {
 106                                 return ref_name;
 107                         }
 108                         set {
 109                                 ref_name = value;
 110                                 Location.SetCurrentSource(ref_name);
 111                         }
 112                 }
 113
 114                 public int Line {
 115                         get {
 116                                 return line;
 117                         }
 118                 }
 119
 120                 public int EffectiveLine {
 121                         get {
 122                                 return ref_line;
 123                         }
 124                         set {
 125                                 ref_line = value;
 126                         }
 127                 }
 128
 129                 public int Col {
 130                         get {
 131                                 return col;
 132                         }
 133                 }
 134
 135                 static void initTokens ()
 136                 {
 137                         keywords = new Hashtable ();
 138
 139                         keywords.Add ("addhandler", Token.ADDHANDLER);
 140                         keywords.Add ("addressof", Token.ADDRESSOF);
 141                         keywords.Add ("alias", Token.ALIAS);
 142                         keywords.Add ("and", Token.AND);
 143                         keywords.Add ("andalso", Token.ANDALSO);
 144                         keywords.Add ("ansi", Token.ANSI);
 145                         keywords.Add ("as", Token.AS);
 146                         keywords.Add ("assembly", Token.ASSEMBLY);
 147                         keywords.Add ("auto", Token.AUTO);
 148                         keywords.Add ("binary", Token.BINARY); // Not a VB.NET Keyword
 149                         keywords.Add ("boolean", Token.BOOLEAN);
 150                         keywords.Add ("byref", Token.BYREF);
 151                         keywords.Add ("byte", Token.BYTE);
 152                         keywords.Add ("byval", Token.BYVAL);
 153                         keywords.Add ("call", Token.CALL);
 154                         keywords.Add ("case", Token.CASE);
 155                         keywords.Add ("catch", Token.CATCH);
 156                         keywords.Add ("cbool", Token.CBOOL);
 157                         keywords.Add ("cbyte", Token.CBYTE);
 158                         keywords.Add ("cchar", Token.CCHAR);
 159                         keywords.Add ("cdate", Token.CDATE);
 160                         keywords.Add ("cdec", Token.CDEC);
 161                         keywords.Add ("cdbl", Token.CDBL);
 162                         keywords.Add ("char", Token.CHAR);
 163                         keywords.Add ("cint", Token.CINT);
 164                         keywords.Add ("class", Token.CLASS);
 165                         keywords.Add ("clng", Token.CLNG);
 166                         keywords.Add ("cobj", Token.COBJ);
 167                         keywords.Add ("compare", Token.COMPARE); // Not a VB.NET Keyword
 168                         keywords.Add ("const", Token.CONST);
 169                         keywords.Add ("cshort", Token.CSHORT);
 170                         keywords.Add ("csng", Token.CSNG);
 171                         keywords.Add ("cstr", Token.CSTR);
 172                         keywords.Add ("ctype", Token.CTYPE);
 173                         keywords.Add ("date", Token.DATE);
 174                         keywords.Add ("decimal", Token.DECIMAL);
 175                         keywords.Add ("declare", Token.DECLARE);
 176                         keywords.Add ("default", Token.DEFAULT);
 177                         keywords.Add ("delegate", Token.DELEGATE);
 178                         keywords.Add ("dim", Token.DIM);
 179                         keywords.Add ("directcast", Token.DIRECTCAST);
 180                         keywords.Add ("do", Token.DO);
 181                         keywords.Add ("double", Token.DOUBLE);
 182                         keywords.Add ("each", Token.EACH);
 183                         keywords.Add ("else", Token.ELSE);
 184                         keywords.Add ("elseif", Token.ELSEIF);
 185                         keywords.Add ("end", Token.END);
 186                         keywords.Add ("endif", Token.ENDIF); // An unused VB.NET keyword
 187                         keywords.Add ("enum", Token.ENUM);
 188                         keywords.Add ("erase", Token.ERASE);
 189                         keywords.Add ("error", Token.ERROR);
 190                         keywords.Add ("event", Token.EVENT);
 191                         keywords.Add ("exit", Token.EXIT);
 192                         keywords.Add ("explicit", Token.EXPLICIT); // Not a VB.NET keyword
 193                         keywords.Add ("false", Token.FALSE);
 194                         keywords.Add ("finally", Token.FINALLY);
 195                         keywords.Add ("for", Token.FOR);
 196                         keywords.Add ("friend", Token.FRIEND);
 197                         keywords.Add ("function", Token.FUNCTION);
 198                         keywords.Add ("get", Token.GET);
 199                         keywords.Add ("gettype", Token.GETTYPE);
 200                         keywords.Add ("gosub", Token.GOSUB); // An unused VB.NET keyword
 201                         keywords.Add ("goto", Token.GOTO);
 202                         keywords.Add ("handles", Token.HANDLES);
 203                         keywords.Add ("if", Token.IF);
 204                         keywords.Add ("implements", Token.IMPLEMENTS);
 205                         keywords.Add ("imports", Token.IMPORTS);
 206                         keywords.Add ("in", Token.IN);
 207                         keywords.Add ("inherits", Token.INHERITS);
 208                         keywords.Add ("integer", Token.INTEGER);
 209                         keywords.Add ("interface", Token.INTERFACE);
 210                         keywords.Add ("is", Token.IS);
 211                         keywords.Add ("let ", Token.LET ); // An unused VB.NET keyword
 212                         keywords.Add ("lib ", Token.LIB );
 213                         keywords.Add ("like ", Token.LIKE );
 214                         keywords.Add ("long", Token.LONG);
 215                         keywords.Add ("loop", Token.LOOP);
 216                         keywords.Add ("me", Token.ME);
 217                         keywords.Add ("mod", Token.MOD);
 218                         keywords.Add ("module", Token.MODULE);
 219                         keywords.Add ("mustinherit", Token.MUSTINHERIT);
 220                         keywords.Add ("mustoverride", Token.MUSTOVERRIDE);
 221                         keywords.Add ("mybase", Token.MYBASE);
 222                         keywords.Add ("myclass", Token.MYCLASS);
 223                         keywords.Add ("namespace", Token.NAMESPACE);
 224                         keywords.Add ("new", Token.NEW);
 225                         keywords.Add ("next", Token.NEXT);
 226                         keywords.Add ("not", Token.NOT);
 227                         keywords.Add ("nothing", Token.NOTHING);
 228                         keywords.Add ("notinheritable", Token.NOTINHERITABLE);
 229                         keywords.Add ("notoverridable", Token.NOTOVERRIDABLE);
 230                         keywords.Add ("object", Token.OBJECT);
 231                         keywords.Add ("off", Token.OFF); // Not a VB.NET Keyword
 232                         keywords.Add ("on", Token.ON);
 233                         keywords.Add ("option", Token.OPTION);
 234                         keywords.Add ("optional", Token.OPTIONAL);
 235                         keywords.Add ("or", Token.OR);
 236                         keywords.Add ("orelse", Token.ORELSE);
 237                         keywords.Add ("overloads", Token.OVERLOADS);
 238                         keywords.Add ("overridable", Token.OVERRIDABLE);
 239                         keywords.Add ("overrides", Token.OVERRIDES);
 240                         keywords.Add ("paramarray", Token.PARAM_ARRAY);
 241                         keywords.Add ("preserve", Token.PRESERVE);
 242                         keywords.Add ("private", Token.PRIVATE);
 243                         keywords.Add ("property", Token.PROPERTY);
 244                         keywords.Add ("protected", Token.PROTECTED);
 245                         keywords.Add ("public", Token.PUBLIC);
 246                         keywords.Add ("raiseevent", Token.RAISEEVENT);
 247                         keywords.Add ("readonly", Token.READONLY);
 248                         keywords.Add ("redim", Token.REDIM);
 249                         keywords.Add ("rem", Token.REM);
 250                         keywords.Add ("removehandler", Token.REMOVEHANDLER);
 251                         keywords.Add ("resume", Token.RESUME);
 252                         keywords.Add ("return", Token.RETURN);
 253                         keywords.Add ("select", Token.SELECT);
 254                         keywords.Add ("set", Token.SET);
 255                         keywords.Add ("shadows", Token.SHADOWS);
 256                         keywords.Add ("shared", Token.SHARED);
 257                         keywords.Add ("short", Token.SHORT);
 258                         keywords.Add ("single", Token.SINGLE);
 259                         keywords.Add ("sizeof", Token.SIZEOF); // Not a VB.NET Keyword
 260                         keywords.Add ("static", Token.STATIC);
 261                         keywords.Add ("step", Token.STEP);
 262                         keywords.Add ("stop", Token.STOP);
 263                         keywords.Add ("strict", Token.STRICT); // Not a VB.NET Keyword
 264                         keywords.Add ("string", Token.STRING);
 265                         keywords.Add ("structure", Token.STRUCTURE);
 266                         keywords.Add ("sub", Token.SUB);
 267                         keywords.Add ("synclock", Token.SYNCLOCK);
 268                         keywords.Add ("text", Token.TEXT); // Not a VB.NET Keyword
 269                         keywords.Add ("then", Token.THEN);
 270                         keywords.Add ("throw", Token.THROW);
 271                         keywords.Add ("to", Token.TO);
 272                         keywords.Add ("true", Token.TRUE);
 273                         keywords.Add ("try", Token.TRY);
 274                         keywords.Add ("typeof", Token.TYPEOF);
 275                         keywords.Add ("unicode", Token.UNICODE);
 276                         keywords.Add ("until", Token.UNTIL);
 277                         keywords.Add ("variant", Token.VARIANT); // An unused VB.NET keyword
 278                         keywords.Add ("wend", Token.WEND); // An unused VB.NET keyword
 279                         keywords.Add ("when", Token.WHEN);
 280                         keywords.Add ("while", Token.WHILE);
 281                         keywords.Add ("with", Token.WITH);
 282                         keywords.Add ("withevents", Token.WITHEVENTS);
 283                         keywords.Add ("writeonly", Token.WRITEONLY);
 284                         keywords.Add ("xor", Token.XOR);
 285
 286                         if (Parser.UseExtendedSyntax){
 287                                 keywords.Add ("yield", Token.YIELD);
 288                         }
 289
 290                 }
 291
 292                 static Tokenizer ()
 293                 {
 294                         initTokens ();
 295                         csharp_format_info = new NumberFormatInfo ();
 296                         csharp_format_info.CurrencyDecimalSeparator = ".";
 297                         styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint;
 298                 }
 299
 300                 public Tokenizer (System.IO.TextReader input, string fname, ArrayList defines)
 301                 {
 302                         this.Source = fname;
 303
 304                         reader = input;
 305
 306                         // putback an EOL at the beginning of a stream. This is a convenience that
 307                         // allows pre-processor directives to be added to the beginning of a vb file.
 308                         putback('\n');
 309                 }
 310
 311                 bool is_keyword (string name)
 312                 {
 313                         bool res;
 314                         name = name.ToLower();
 315
 316                         res = keywords.Contains(name);
 317                         if ((name == "GET" || name == "SET") && handle_get_set == false)
 318                                 return false;
 319                         return res;
 320                 }
 321
 322                 int getKeyword (string name)
 323                 {
 324                         return (int) (keywords [name.ToLower()]);
 325                 }
 326
 327                 public Location Location {
 328                         get {
 329                                 return new Location (ref_line, col);
 330                         }
 331                 }
 332
 333                 public bool PropertyParsing {
 334                         get {
 335                                 return handle_get_set;
 336                         }
 337
 338                         set {
 339                                 handle_get_set = value;
 340                         }
 341                 }
 342
 343                 bool is_identifier_start_character (char c)
 344                 {
 345                         return Char.IsLetter (c) || c == '_' ;
 346                 }
 347
 348                 bool is_identifier_part_character (char c)
 349                 {
 350                         return (Char.IsLetter (c) || Char.IsDigit (c) || c == '_');
 351                 }
 352
 353                 int is_punct (char c, ref bool doread)
 354                 {
 355                         int d;
 356                         int t;
 357
 358                         doread = false;
 359
 360                         error_details = c.ToString();
 361
 362                         d = peekChar ();
 363
 364                         switch (c){
 365                         case '[':
 366                                 return Token.OPEN_BRACKET;
 367                         case ']':
 368                                 return Token.CLOSE_BRACKET;
 369                         case '{':
 370                                 return Token.OPEN_BRACE;
 371                         case '}':
 372                                 return Token.CLOSE_BRACE;
 373                         case '(':
 374                                 return Token.OPEN_PARENS;
 375                         case ')':
 376                                 return Token.CLOSE_PARENS;
 377                         case ',':
 378                                 return Token.COMMA;
 379                         case '?':
 380                                 return Token.INTERR;
 381                         case '!':
 382                                 if (is_identifier_start_character((char)d) || cant_have_a_type_character)
 383                                         return Token.EXCLAMATION;
 384                                 return Token.SINGLETYPECHAR;
 385                         case '$':
 386                                 if (cant_have_a_type_character)
 387                                         return Token.ERROR;
 388                                 return Token.DOLAR_SIGN;
 389                         case '@':
 390                                 if (cant_have_a_type_character)
 391                                         return Token.ERROR;
 392                                 return Token.AT_SIGN;
 393                         case '%':
 394                                 if (cant_have_a_type_character)
 395                                         return Token.ERROR;
 396                                 return Token.PERCENT;
 397                         case '#':
 398                                 if(tokens_seen)
 399                                 {
 400                                         if (cant_have_a_type_character)
 401                                                 return ExtractDateTimeLiteral();
 402                                         else
 403                                                 return Token.NUMBER_SIGN;
 404                                 }
 405                                 else
 406                                 {
 407                                         tokens_seen = true;
 408                                         return Token.HASH;
 409                                 }
 410                         case '&':
 411                                 if (!cant_have_a_type_character)
 412                                         return Token.LONGTYPECHAR;
 413                                 t = handle_integer_literal_in_other_bases(d);
 414                                 if (t == Token.NONE) {
 415                                         if (d == '=') {
 416                                                 doread = true;
 417                                                 t = Token.OP_CONCAT_ASSIGN;
 418                                         } else
 419                                                 t = Token.OP_CONCAT;
 420                                 }
 421                                 return t;
 422                         }
 423
 424                         if (c == '+'){
 425                                 if (d == '+')
 426                                         t = Token.OP_INC;
 427                                 else if (d == '=')
 428                                         t = Token.OP_ADD_ASSIGN;
 429                                 else
 430                                         return Token.PLUS;
 431                                 doread = true;
 432                                 return t;
 433                         }
 434                         if (c == '-'){
 435                                 if (d == '=')
 436                                         t = Token.OP_SUB_ASSIGN;
 437                                 else
 438                                         return Token.MINUS;
 439                                 doread = true;
 440                                 return t;
 441                         }
 442
 443                         if (c == '='){
 444                                 return Token.ASSIGN;
 445                         }
 446
 447                         if (c == '*'){
 448                                 if (d == '='){
 449                                         doread = true;
 450                                         return Token.OP_MULT_ASSIGN;
 451                                 }
 452                                 return Token.STAR;
 453                         }
 454
 455                         if (c == '/'){
 456                                 if (d == '='){
 457                                         doread = true;
 458                                         return Token.OP_DIV_ASSIGN;
 459                                 }
 460                                 return Token.DIV;
 461                         }
 462
 463                         if (c == '\\'){
 464                                 if (d == '='){
 465                                         doread = true;
 466                                         return Token.OP_IDIV_ASSIGN;
 467                                 }
 468                                 return Token.OP_IDIV;
 469                         }
 470
 471                         if (c == '^'){
 472                                 if (d == '='){
 473                                         doread = true;
 474                                         return Token.OP_EXP_ASSIGN;
 475                                 }
 476                                 return Token.OP_EXP;
 477                         }
 478
 479                         if (c == '<'){
 480                                 if (d == '>')
 481                                 {
 482                                         doread = true;
 483                                         return Token.OP_NE;
 484                                 }
 485                                 if (d == '='){
 486                                         doread = true;
 487                                         return Token.OP_LE;
 488                                 }
 489                                 if (d == '<')
 490                                 {
 491                                         doread = true;
 492                                         return Token.OP_SHIFT_LEFT;
 493                                 }
 494                                 return Token.OP_LT;
 495                         }
 496
 497                         if (c == '>'){
 498                                 if (d == '='){
 499                                         doread = true;
 500                                         return Token.OP_GE;
 501                                 }
 502                                 if (d == '>')
 503                                 {
 504                                         doread = true;
 505                                         return Token.OP_SHIFT_RIGHT;
 506                                 }
 507                                 return Token.OP_GT;
 508                         }
 509
 510                         if (c == ':'){
 511                                 if (d == '='){
 512                                         doread = true;
 513                                         return Token.ATTR_ASSIGN;
 514                                 }
 515                                 return Token.COLON;
 516                         }
 517
 518                         return Token.ERROR;
 519                 }
 520
 521                 bool decimal_digits (int c)
 522                 {
 523                         int d;
 524                         bool seen_digits = false;
 525
 526                         if (c != -1)
 527                                 number.Append ((char) c);
 528
 529                         while ((d = peekChar ()) != -1){
 530                                 if (Char.IsDigit ((char)d)){
 531                                         number.Append ((char) d);
 532                                         getChar ();
 533                                         seen_digits = true;
 534                                 } else
 535                                         break;
 536                         }
 537                         return seen_digits;
 538                 }
 539
 540
 541                 int real_type_suffix (int c)
 542                 {
 543                         int t;
 544
 545                         switch (c){
 546                         case 'F': case 'f':
 547                                 t =  Token.LITERAL_SINGLE;
 548                                 break;
 549                         case 'R': case 'r':
 550                                 t = Token.LITERAL_DOUBLE;
 551                                 break;
 552                         case 'D': case 'd':
 553                                  t= Token.LITERAL_DECIMAL;
 554                                 break;
 555                         default:
 556                                 return Token.NONE;
 557                         }
 558                         getChar ();
 559                         return t;
 560                 }
 561
 562                 int integer_type_suffix (int c)
 563                 {
 564                         int t;
 565
 566                         try {
 567
 568                                 switch (c){
 569                                 case 'S': case 's':
 570                                         t =  Token.LITERAL_INTEGER; // SHORT ?
 571                                         val = ((IConvertible)val).ToInt16(null);
 572                                         break;
 573                                 case 'I': case 'i':
 574                                         t = Token.LITERAL_INTEGER;
 575                                         val = ((IConvertible)val).ToInt32(null);
 576                                         break;
 577                                 case 'L': case 'l':
 578                                          t= Token.LITERAL_INTEGER; // LONG ?
 579                                          val = ((IConvertible)val).ToInt64(null);
 580                                         break;
 581                                 default:
 582                                         if ((long)val <= System.Int32.MaxValue &&
 583                                                 (long)val >= System.Int32.MinValue) {
 584                                                 val = ((IConvertible)val).ToInt32(null);
 585                                                 return Token.LITERAL_INTEGER;
 586                                         } else {
 587                                                 val = ((IConvertible)val).ToInt64(null);
 588                                                 return Token.LITERAL_INTEGER; // LONG ?
 589                                         }
 590                                 }
 591                                 getChar ();
 592                                 return t;
 593                         } catch (Exception e) {
 594                                 val = e.ToString();
 595                                 return Token.ERROR;
 596                         }
 597                 }
 598
 599                 int adjust_real (int t)
 600                 {
 601                         string s = number.ToString ();
 602
 603                         switch (t){
 604                         case Token.LITERAL_DECIMAL:
 605                                 val = new System.Decimal ();
 606                                 val = System.Decimal.Parse (
 607                                         s, styles, csharp_format_info);
 608                                 break;
 609                         case Token.LITERAL_DOUBLE:
 610                                 val = new System.Double ();
 611                                 val = System.Double.Parse (
 612                                         s, styles, csharp_format_info);
 613                                 break;
 614                         case Token.LITERAL_SINGLE:
 615                                 val = new System.Double ();
 616                                 val = (float) System.Double.Parse (
 617                                         s, styles, csharp_format_info);
 618                                 break;
 619
 620                         case Token.NONE:
 621                                 val = new System.Double ();
 622                                 val = System.Double.Parse (
 623                                         s, styles, csharp_format_info);
 624                                 t = Token.LITERAL_DOUBLE;
 625                                 break;
 626                         }
 627                         return t;
 628                 }
 629
 630                 long hex_digits ()
 631                 {
 632                         StringBuilder hexNumber = new StringBuilder ();
 633
 634                         int d;
 635
 636                         while ((d = peekChar ()) != -1){
 637                                 char e = Char.ToUpper ((char) d);
 638
 639                                 if (Char.IsDigit (e) || (e >= 'A' && e <= 'F')){
 640                                         hexNumber.Append (e);
 641                                         getChar ();
 642                                 } else
 643                                         break;
 644                         }
 645                         return System.Int64.Parse (hexNumber.ToString(), NumberStyles.HexNumber);
 646                 }
 647
 648                 long octal_digits ()
 649                 {
 650                         long valueToReturn = 0;
 651
 652                         int d;
 653
 654                         while ((d = peekChar ()) != -1){
 655                                 char e = (char)d;
 656                                 if (Char.IsDigit (e) && (e < '8')){
 657                                         valueToReturn *= 8;
 658                                         valueToReturn += (d - (int)'0');
 659                                         getChar ();
 660                                 } else
 661                                         break;
 662                         }
 663
 664                         return valueToReturn;
 665                 }
 666
 667                 int handle_integer_literal_in_other_bases(int peek)
 668                 {
 669                         if (peek == 'h' || peek == 'H'){
 670                                 getChar ();
 671                                 val = hex_digits ();
 672                                 return integer_type_suffix (peekChar ());
 673                         }
 674
 675                         if (peek == 'o' || peek == 'O'){
 676                                 getChar ();
 677                                 val = octal_digits ();
 678                                 return integer_type_suffix (peekChar ());
 679                         }
 680
 681                         return Token.NONE;
 682                 }
 683
 684                 //
 685                 // Invoked if we know we have .digits or digits
 686                 //
 687                 int is_number (int c)
 688                 {
 689                         bool is_real = false;
 690                         number = new StringBuilder ();
 691                         int type;
 692
 693                         number.Length = 0;
 694
 695                         if (Char.IsDigit ((char)c)){
 696                                 decimal_digits (c);
 697                                 c = peekChar ();
 698                         }
 699
 700                         //
 701                         // We need to handle the case of
 702                         // "1.1" vs "1.ToString()" (LITERAL_SINGLE vs NUMBER DOT IDENTIFIER)
 703                         //
 704                         if (c == '.'){
 705                                 if (decimal_digits (getChar())){
 706                                         is_real = true;
 707                                         c = peekChar ();
 708                                 } else {
 709                                         putback ('.');
 710                                         number.Length -= 1;
 711                                         val = System.Int64.Parse(number.ToString());
 712                                         return integer_type_suffix('.');
 713                                 }
 714                         }
 715
 716                         if (c == 'e' || c == 'E'){
 717                                 is_real = true;
 718                                 number.Append ("e");
 719                                 getChar ();
 720
 721                                 c = peekChar ();
 722                                 if (c == '+'){
 723                                         number.Append ((char) c);
 724                                         getChar ();
 725                                         c = peekChar ();
 726                                 } else if (c == '-'){
 727                                         number.Append ((char) c);
 728                                         getChar ();
 729                                         c = peekChar ();
 730                                 }
 731                                 decimal_digits (-1);
 732                                 c = peekChar ();
 733                         }
 734
 735                         type = real_type_suffix (c);
 736                         if (type == Token.NONE && !is_real){
 737                                 val = System.Int64.Parse(number.ToString());
 738                                 return integer_type_suffix(c);
 739                         }
 740
 741                         return adjust_real (type);
 742                 }
 743
 744                 int getChar ()
 745                 {
 746                         if (putback_char != -1){
 747                                 int x = putback_char;
 748                                 putback_char = -1;
 749
 750                                 return x;
 751                         }
 752                         return reader.Read ();
 753                 }
 754
 755                 int peekChar ()
 756                 {
 757                         if (putback_char != -1)
 758                                 return putback_char;
 759                         return reader.Peek ();
 760                 }
 761
 762                 void putback (int c)
 763                 {
 764                         if (putback_char != -1)
 765                                 throw new Exception ("This should not happen putback on putback");
 766                         putback_char = c;
 767                 }
 768
 769                 public bool advance ()
 770                 {
 771                         return current_token != Token.EOF ;
 772                 }
 773
 774                 public Object Value {
 775                         get {
 776                                 return val;
 777                         }
 778                 }
 779
 780                 public Object value ()
 781                 {
 782                         return val;
 783                 }
 784
 785                 private bool IsEOL(int currentChar)
 786                 {
 787                         bool retVal;
 788
 789                         if (currentChar ==  0x0D) {
 790                                 if (peekChar() ==  0x0A) // if it is a CR-LF pair consume LF also
 791                                         getChar();
 792
 793                                 retVal = true;
 794                         }
 795                         else {
 796                                 retVal = (currentChar ==  -1 || currentChar ==  0x0A || currentChar ==  0x2028 || currentChar ==  0x2029);
 797                         }
 798
 799                         if(retVal) {
 800                                 nextLine();
 801                         }
 802
 803                         return retVal;
 804                 }
 805
 806                 private int DropComments()
 807                 {
 808                         int d;
 809                         while (!IsEOL(d = getChar ()))
 810                                 col++;
 811
 812                         return Token.EOL;
 813                 }
 814
 815                 public int token ()
 816                 {
 817                         int lastToken = current_token;
 818                         do
 819                         {
 820                                 current_token = xtoken ();
 821                                 if (current_token == 0)
 822                                         return Token.EOF;
 823                                 if (current_token == Token.REM)
 824                                         current_token = DropComments();
 825                         } while (lastToken == Token.EOL && current_token == Token.EOL);
 826
 827                         return current_token;
 828                 }
 829
 830                 private string GetIdentifier()
 831                 {
 832                         int c = getChar();
 833                         if (is_identifier_start_character ((char) c))
 834                                 return GetIdentifier(c);
 835                         else
 836                                 return null;
 837                 }
 838
 839                 private string GetIdentifier(int c)
 840                 {
 841                         StringBuilder id = new StringBuilder ();
 842
 843                         id.Append ((char) c);
 844
 845                         while ((c = peekChar ()) != -1)
 846                         {
 847                                 if (is_identifier_part_character ((char) c))
 848                                 {
 849                                         id.Append ((char)getChar ());
 850                                         col++;
 851                                 }
 852                                 else
 853                                         break;
 854                         }
 855
 856                         cant_have_a_type_character = false;
 857
 858                         return id.ToString();
 859                 }
 860
 861                 private bool is_doublequote(int currentChar)
 862                 {
 863                         return (currentChar == '"' ||
 864                                         currentChar == 0x201C || // unicode left double-quote character
 865                                         currentChar == 0x201D);  // unicode right double-quote character
 866                 }
 867
 868                 private bool is_whitespace(int c)
 869                 {
 870                         return (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0);
 871                 }
 872
 873                 private bool tokens_seen = false;
 874
 875                 private void nextLine()
 876                 {
 877                         cant_have_a_type_character = true;
 878                         line++;
 879                         ref_line++;
 880                         col = 0;
 881                         tokens_seen = false;
 882                 }
 883
 884                 public int xtoken ()
 885                 {
 886                         int t;
 887                         bool doread = false;
 888                         int c;
 889
 890                         val = null;
 891                         for (;(c = getChar ()) != -1; col++) {
 892
 893                                 // Handle line continuation character
 894                                 if (c == '_')
 895                                 {
 896                                         int d = peekChar();
 897                                         if (!is_identifier_part_character((char)d)) {
 898                                                 while ((c = getChar ()) != -1 && !IsEOL(c)) {}
 899                                                 c = getChar ();
 900                                         }
 901                                 }
 902
 903                                 // white space
 904                                 if (is_whitespace(c)) {
 905                                         // expand tabs for location
 906                                         if (c == '\t')
 907                                                 col = (((col + ExpandedTabsSize) / ExpandedTabsSize) * ExpandedTabsSize) - 1;
 908                                         cant_have_a_type_character = true;
 909                                         continue;
 910                                 }
 911
 912                                 // Handle line comments.
 913                                 if (c == '\'')
 914                                         return Token.REM;
 915
 916                                 // Handle EOL.
 917                                 if (IsEOL(c))
 918                                 {
 919                                         if (current_token == Token.EOL) // if last token was also EOL keep skipping
 920                                                 continue;
 921                                         return Token.EOL;
 922                                 }
 923
 924                                 // Handle escaped identifiers
 925                                 if (c == '[')
 926                                 {
 927                                         if ((val = GetIdentifier()) == null)
 928                                                 break;
 929                                         if ((c = getChar()) != ']')
 930                                                 break;
 931                                         tokens_seen = true;
 932                                         return Token.IDENTIFIER;
 933                                 }
 934
 935                                 // Handle unescaped identifiers
 936                                 if (is_identifier_start_character ((char) c))
 937                                 {
 938                                         string id;
 939                                         if ((id = GetIdentifier(c)) == null)
 940                                                 break;
 941                                         val = id;
 942                                         tokens_seen = true;
 943                                         if (is_keyword(id) && (current_token != Token.DOT))
 944                                                 return getKeyword(id);
 945                                         return Token.IDENTIFIER;
 946                                 }
 947
 948                                 // Treat string literals
 949                                 if (is_doublequote(c)) {
 950                                         cant_have_a_type_character = true;
 951                                         return ExtractStringOrCharLiteral(c);
 952                                 }
 953
 954                                 // handle numeric literals
 955                                 if (c == '.')
 956                                 {
 957                                         cant_have_a_type_character = true;
 958                                         tokens_seen = true;
 959                                         if (Char.IsDigit ((char) peekChar ()))
 960                                                 return is_number (c);
 961                                         return Token.DOT;
 962                                 }
 963
 964                                 if (Char.IsDigit ((char) c))
 965                                 {
 966                                         cant_have_a_type_character = true;
 967                                         tokens_seen = true;
 968                                         return is_number (c);
 969                                 }
 970
 971                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR) {
 972                                         cant_have_a_type_character = true;
 973
 974                                         if (t == Token.NONE)
 975                                                 continue;
 976
 977                                         if (doread){
 978                                                 getChar ();
 979                                                 col++;
 980                                         }
 981                                         tokens_seen = true;
 982                                         return t;
 983                                 }
 984
 985                                 error_details = ((char)c).ToString ();
 986                                 return Token.ERROR;
 987                         }
 988
 989                         if (current_token != Token.EOL) // if last token wasn't EOL send it before EOF
 990                                 return Token.EOL;
 991
 992                         return Token.EOF;
 993                 }
 994
 995                 private int ExtractDateTimeLiteral()
 996                 {
 997                         int c;
 998
 999                         StringBuilder sb = new StringBuilder();
1000                         for (;(c = getChar ()) != -1; col++)
1001                         {
1002                                 if (c == '#') {
1003                                         val = ParseDateLiteral(sb);
1004                                         return Token.LITERAL_DATE;
1005                                 }
1006                                 if (IsEOL(c)) {
1007                                         break;
1008                                 }
1009                                 if (c == '-')
1010                                         c = '/';
1011                                 sb.Append((char)c);
1012                         }
1013                         return Token.ERROR;
1014                 }
1015
1016                 private int ExtractStringOrCharLiteral(int c)
1017                 {
1018                         StringBuilder s = new StringBuilder ();
1019
1020                         tokens_seen = true;
1021
1022                         while ((c = getChar ()) != -1){
1023                                 if (is_doublequote(c)){
1024                                         if (is_doublequote(peekChar()))
1025                                                 getChar();
1026                                         else {
1027                                                 //handle Char Literals
1028                                                 if (peekChar() == 'C' || peekChar() == 'c') {
1029                                                         getChar();
1030                                                         if (s.Length == 1) {
1031                                                                 val = s[0];
1032                                                                 return Token.LITERAL_CHARACTER;
1033                                                         } else {
1034                                                                 val = "Incorrect length for a character literal";
1035                                                                 return Token.ERROR;
1036                                                         }
1037                                                 } else {
1038                                                         val = s.ToString ();
1039                                                         return Token.LITERAL_STRING;
1040                                                 }
1041                                         }
1042                                 }
1043
1044                                 if (IsEOL(c)) {
1045                                         return Token.ERROR;
1046                                 }
1047
1048                                 s.Append ((char) c);
1049                         }
1050
1051                         return Token.ERROR;
1052                 }
1053
1054                 static IFormatProvider enUSculture = new CultureInfo("en-US", true);
1055
1056                 private DateTime ParseDateLiteral(StringBuilder value)
1057                 {
1058                         try
1059                         {
1060                                 return DateTime.Parse(value.ToString(),
1061                                                   enUSculture,
1062                                                   DateTimeStyles.NoCurrentDateDefault | DateTimeStyles.AllowWhiteSpaces);
1063                         }
1064                         catch (FormatException ex)
1065                         {
1066                                 //TODO: What is the correct error number and message?
1067                                 Report.Error (1, Location, string.Format("Invalid date literal '{0}'", value.ToString())
1068                                         + Environment.NewLine + ex.ToString());
1069                         }
1070                         catch (Exception)
1071                         {
1072                                 Report.Error (1, Location, "Error parsing date literal");       //TODO: What is the correct error number and message?
1073                         }
1074                         return new DateTime();
1075                 }
1076
1077                 public void PositionCursorAtNextPreProcessorDirective()
1078                 {
1079                         int t;
1080
1081                         for(t = token(); t != Token.HASH && t != Token.EOF; t = token());
1082
1083                         if(t == Token.EOF)
1084                                 throw new ApplicationException("Unexpected EOF while looking for a pre-processor directive");
1085
1086                         if(t == Token.HASH) {
1087                                 tokens_seen = false;
1088                                 putback('#');
1089                         }
1090                 }
1091
1092         }
1093 }