mcs/mbas/mb-tokenizer.cs

   1 //
   2 // Mono.MonoBASIC.Tokenizer.cs: The Tokenizer for the MonoBASIC compiler
   3 //
   4 // Author: A Rafael D Teixeira (rafaelteixeirabr@hotmail.com)
   5 //
   6 // Based on cs-tokenizer.cs by Miguel de Icaza (miguel@gnu.org)
   7 //
   8 // Licensed under the terms of the GNU GPL
   9 //
  10 // Copyright (C) 2001 A Rafael D Teixeira
  11 //
  12
  13 namespace Mono.MonoBASIC
  14 {
  15         using System;
  16         using System.Text;
  17         using System.Collections;
  18         using System.IO;
  19         using System.Globalization;
  20         using Mono.Languages;
  21         using Mono.CSharp;
  22
  23         /// <summary>
  24         ///    Tokenizer for MonoBASIC source code.
  25         /// </summary>
  26
  27         public class Tokenizer : yyParser.yyInput
  28         {
  29                 TextReader reader;
  30                 public string ref_name;
  31                 public int ref_line = 1;
  32                 public int line = 1;
  33                 public int col = 1;
  34                 public int current_token;
  35                 bool handle_get_set = false;
  36
  37                 public int ExpandedTabsSize = 4;
  38
  39                 public string location {
  40                         get {
  41                                 string det;
  42
  43                                 if (current_token == Token.ERROR)
  44                                         det = "detail: " + error_details;
  45                                 else
  46                                         det = "";
  47
  48                                 return "Line:     "+line+" Col: "+col + "\n" +
  49                                        "VirtLine: "+ref_line +
  50                                        " Token: "+current_token + " " + det;
  51                         }
  52                 }
  53
  54                 public bool properties {
  55                         get {
  56                                 return handle_get_set;
  57                         }
  58
  59                         set {
  60                                 handle_get_set = value;
  61                         }
  62                 }
  63
  64                 //
  65                 // Class variables
  66                 //
  67                 static Hashtable keywords;
  68                 static NumberStyles styles;
  69                 static NumberFormatInfo csharp_format_info;
  70
  71                 //
  72                 // Values for the associated token returned
  73                 //
  74                 System.Text.StringBuilder number;
  75                 int putback_char;
  76                 Object val;
  77
  78                 //
  79                 // Details about the error encoutered by the tokenizer
  80                 //
  81                 string error_details;
  82
  83                 public string error {
  84                         get {
  85                                 return error_details;
  86                         }
  87                 }
  88
  89                 public int Line {
  90                         get {
  91                                 return line;
  92                         }
  93                 }
  94
  95                 public int Col {
  96                         get {
  97                                 return col;
  98                         }
  99                 }
 100
 101                 static void initTokens ()
 102                 {
 103                         keywords = new Hashtable ();
 104
 105                         keywords.Add ("addhandler", Token.ADDHANDLER);
 106                         keywords.Add ("addressof", Token.ADDRESSOF);
 107                         keywords.Add ("alias", Token.ALIAS);
 108                         keywords.Add ("and", Token.AND);
 109                         keywords.Add ("andalso", Token.ANDALSO);
 110                         keywords.Add ("ansi", Token.ANSI);
 111                         keywords.Add ("as", Token.AS);
 112                         keywords.Add ("assembly", Token.ASSEMBLY);
 113                         keywords.Add ("auto", Token.AUTO);
 114                         keywords.Add ("boolean", Token.BOOLEAN);
 115                         keywords.Add ("byref", Token.BYREF);
 116                         keywords.Add ("byte", Token.BYTE);
 117                         keywords.Add ("byval", Token.BYVAL);
 118                         keywords.Add ("call", Token.CALL);
 119                         keywords.Add ("case", Token.CASE);
 120                         keywords.Add ("catch", Token.CATCH);
 121                         keywords.Add ("cbool", Token.CBOOL);
 122                         keywords.Add ("cbyte", Token.CBYTE);
 123                         keywords.Add ("cchar", Token.CCHAR);
 124                         keywords.Add ("cdate", Token.CDATE);
 125                         keywords.Add ("cdec", Token.CDEC);
 126                         keywords.Add ("cdbl", Token.CDBL);
 127                         keywords.Add ("char", Token.CHAR);
 128                         keywords.Add ("cint", Token.CINT);
 129                         keywords.Add ("class", Token.CLASS);
 130                         keywords.Add ("clng", Token.CLNG);
 131                         keywords.Add ("cobj", Token.COBJ);
 132                         //keywords.Add ("compare", Token.COMPARE);
 133                         keywords.Add ("const", Token.CONST);
 134                         keywords.Add ("cshort", Token.CSHORT);
 135                         keywords.Add ("csng", Token.CSNG);
 136                         keywords.Add ("cstr", Token.CSTR);
 137                         keywords.Add ("ctype", Token.CTYPE);
 138                         keywords.Add ("date", Token.DATE);
 139                         keywords.Add ("decimal", Token.DECIMAL);
 140                         keywords.Add ("declare", Token.DECLARE);
 141                         keywords.Add ("default", Token.DEFAULT);
 142                         keywords.Add ("delegate", Token.DELEGATE);
 143                         keywords.Add ("dim", Token.DIM);
 144                         keywords.Add ("do", Token.DO);
 145                         keywords.Add ("double", Token.DOUBLE);
 146                         keywords.Add ("each", Token.EACH);
 147                         keywords.Add ("else", Token.ELSE);
 148                         keywords.Add ("elseif", Token.ELSEIF);
 149                         keywords.Add ("end", Token.END);
 150                         keywords.Add ("enum", Token.ENUM);
 151                         keywords.Add ("erase", Token.ERASE);
 152                         keywords.Add ("error", Token.ERROR);
 153                         keywords.Add ("event", Token.EVENT);
 154                         keywords.Add ("exit", Token.EXIT);
 155                         //keywords.Add ("explicit", Token.EXPLICIT);
 156                         keywords.Add ("false", Token.FALSE);
 157                         keywords.Add ("finally", Token.FINALLY);
 158                         keywords.Add ("for", Token.FOR);
 159                         keywords.Add ("friend", Token.FRIEND);
 160                         keywords.Add ("function", Token.FUNCTION);
 161                         keywords.Add ("get", Token.GET);
 162                         keywords.Add ("gettype", Token.GETTYPE);
 163                         keywords.Add ("goto", Token.GOTO);
 164                         keywords.Add ("handles", Token.HANDLES);
 165                         keywords.Add ("if", Token.IF);
 166                         keywords.Add ("implements", Token.IMPLEMENTS);
 167                         keywords.Add ("imports", Token.IMPORTS);
 168                         keywords.Add ("in", Token.IN);
 169                         keywords.Add ("inherits", Token.INHERITS);
 170                         keywords.Add ("integer", Token.INTEGER);
 171                         keywords.Add ("interface", Token.INTERFACE);
 172                         keywords.Add ("is", Token.IS);
 173                         keywords.Add ("let ", Token.LET );
 174                         keywords.Add ("lib ", Token.LIB );
 175                         keywords.Add ("like ", Token.LIKE );
 176                         keywords.Add ("long", Token.LONG);
 177                         keywords.Add ("loop", Token.LOOP);
 178                         keywords.Add ("me", Token.ME);
 179                         keywords.Add ("mod", Token.MOD);
 180                         keywords.Add ("module", Token.MODULE);
 181                         keywords.Add ("mustinherit", Token.MUSTINHERIT);
 182                         keywords.Add ("mustoverride", Token.MUSTOVERRIDE);
 183                         keywords.Add ("mybase", Token.MYBASE);
 184                         keywords.Add ("myclass", Token.MYCLASS);
 185                         keywords.Add ("namespace", Token.NAMESPACE);
 186                         keywords.Add ("new", Token.NEW);
 187                         keywords.Add ("next", Token.NEXT);
 188                         keywords.Add ("not", Token.NOT);
 189                         keywords.Add ("nothing", Token.NOTHING);
 190                         keywords.Add ("notinheritable", Token.NOTINHERITABLE);
 191                         keywords.Add ("notoverridable", Token.NOTOVERRIDABLE);
 192                         keywords.Add ("object", Token.OBJECT);
 193                         keywords.Add ("on", Token.ON);
 194                         keywords.Add ("option", Token.OPTION);
 195                         keywords.Add ("optional", Token.OPTIONAL);
 196                         keywords.Add ("or", Token.OR);
 197                         keywords.Add ("orelse", Token.ORELSE);
 198                         keywords.Add ("overloads", Token.OVERLOADS);
 199                         keywords.Add ("overridable", Token.OVERRIDABLE);
 200                         keywords.Add ("overrides", Token.OVERRIDES);
 201                         keywords.Add ("paramarray", Token.PARAM_ARRAY);
 202                         keywords.Add ("preserve", Token.PRESERVE);
 203                         keywords.Add ("private", Token.PRIVATE);
 204                         keywords.Add ("property", Token.PROPERTY);
 205                         keywords.Add ("protected", Token.PROTECTED);
 206                         keywords.Add ("public", Token.PUBLIC);
 207                         keywords.Add ("raiseevent", Token.RAISEEVENT);
 208                         keywords.Add ("readonly", Token.READONLY);
 209                         keywords.Add ("redim", Token.REDIM);
 210                         keywords.Add ("rem", Token.REM);
 211                         keywords.Add ("removehandler", Token.REMOVEHANDLER);
 212                         keywords.Add ("resume", Token.RESUME);
 213                         keywords.Add ("return", Token.RETURN);
 214                         keywords.Add ("select", Token.SELECT);
 215                         keywords.Add ("set", Token.SET);
 216                         keywords.Add ("shadows", Token.SHADOWS);
 217                         keywords.Add ("shared", Token.SHARED);
 218                         keywords.Add ("short", Token.SHORT);
 219                         keywords.Add ("single", Token.SINGLE);
 220                         keywords.Add ("sizeof", Token.SIZEOF);
 221                         keywords.Add ("static", Token.STATIC);
 222                         keywords.Add ("step", Token.STEP);
 223                         keywords.Add ("stop", Token.STOP);
 224                         keywords.Add ("string", Token.STRING);
 225                         keywords.Add ("structure", Token.STRUCTURE);
 226                         keywords.Add ("sub", Token.SUB);
 227                         keywords.Add ("synclock", Token.SYNCLOCK);
 228                         keywords.Add ("then", Token.THEN);
 229                         keywords.Add ("throw", Token.THROW);
 230                         keywords.Add ("to", Token.TO);
 231                         keywords.Add ("true", Token.TRUE);
 232                         keywords.Add ("try", Token.TRY);
 233                         keywords.Add ("typeof", Token.TYPEOF);
 234                         keywords.Add ("unicode", Token.UNICODE);
 235                         keywords.Add ("until", Token.UNTIL);
 236                         keywords.Add ("variant", Token.VARIANT);
 237                         keywords.Add ("when", Token.WHEN);
 238                         keywords.Add ("while", Token.WHILE);
 239                         keywords.Add ("with", Token.WITH);
 240                         keywords.Add ("withevents", Token.WITHEVENTS);
 241                         keywords.Add ("writeonly", Token.WRITEONLY);
 242                         keywords.Add ("xor", Token.XOR);
 243                 }
 244
 245                 //
 246                 // Class initializer
 247                 //
 248                 static Tokenizer ()
 249                 {
 250                         initTokens ();
 251                         csharp_format_info = new NumberFormatInfo ();
 252                         csharp_format_info.CurrencyDecimalSeparator = ".";
 253                         styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint;
 254                 }
 255
 256                 bool is_keyword (string name)
 257                 {
 258                         bool res;
 259
 260                         res = keywords.Contains(name.ToLower());
 261                         if ((name == "get" || name == "set") && handle_get_set == false)
 262                                 return false;
 263                         return res;
 264                 }
 265
 266                 int getKeyword (string name)
 267                 {
 268                         return (int) (keywords [name.ToLower()]);
 269                 }
 270
 271                 public Location Location {
 272                         get {
 273                                 return new Location (ref_line);
 274                         }
 275                 }
 276
 277                 public bool PropertyParsing {
 278                         get {
 279                                 return handle_get_set;
 280                         }
 281
 282                         set {
 283                                 handle_get_set = value;
 284                         }
 285                 }
 286
 287                 bool is_identifier_start_character (char c)
 288                 {
 289                         return Char.IsLetter (c) || c == '_' ;
 290                 }
 291
 292                 bool is_identifier_part_character (char c)
 293                 {
 294                         return (Char.IsLetter (c) || Char.IsDigit (c) || c == '_');
 295                 }
 296
 297                 int is_punct (char c, ref bool doread)
 298                 {
 299                         int idx = "{}[](),:;~+-*/%&|^!=<>?".IndexOf (c);
 300                         int d;
 301                         int t;
 302
 303                         doread = false;
 304
 305                         switch (c){
 306 //                      case '[':
 307 //                              return Token.OPEN_BRACKET;
 308 //                      case ']':
 309 //                              return Token.CLOSE_BRACKET;
 310                         case '(':
 311                                 return Token.OPEN_PARENS;
 312                         case ')':
 313                                 return Token.CLOSE_PARENS;
 314                         case ',':
 315                                 return Token.COMMA;
 316                         case ':':
 317                                 return Token.COLON;
 318                         case '?':
 319                                 return Token.INTERR;
 320                         }
 321
 322                         d = peekChar ();
 323                         if (c == '+'){
 324
 325                                 if (d == '+')
 326                                         t = Token.OP_INC;
 327                                 else if (d == '=')
 328                                         t = Token.OP_ADD_ASSIGN;
 329                                 else
 330                                         return Token.PLUS;
 331                                 doread = true;
 332                                 return t;
 333                         }
 334                         if (c == '-'){
 335                                 if (d == '=')
 336                                         t = Token.OP_SUB_ASSIGN;
 337                                 else
 338                                         return Token.MINUS;
 339                                 doread = true;
 340                                 return t;
 341                         }
 342
 343                         if (c == '='){
 344                                 /*if (d == '='){
 345                                         doread = true;
 346                                         return Token.OP_EQ;
 347                                 }*/
 348                                 return Token.ASSIGN;
 349                         }
 350
 351                         if (c == '*'){
 352                                 if (d == '='){
 353                                         doread = true;
 354                                         return Token.OP_MULT_ASSIGN;
 355                                 }
 356                                 return Token.STAR;
 357                         }
 358
 359                         if (c == '/'){
 360                                 if (d == '='){
 361                                         doread = true;
 362                                         return Token.OP_DIV_ASSIGN;
 363                                 }
 364                                 return Token.DIV;
 365                         }
 366
 367                         if (c == '\\'){
 368                                 if (d == '='){
 369                                         doread = true;
 370                                         return Token.OP_IDIV_ASSIGN;
 371                                 }
 372                                 return Token.OP_IDIV;
 373                         }
 374
 375                         if (c == '^'){
 376                                 if (d == '='){
 377                                         doread = true;
 378                                         return Token.OP_EXP_ASSIGN;
 379                                 }
 380                                 return Token.OP_EXP;
 381                         }
 382
 383                         if (c == '<'){
 384                                 if (d == '>')
 385                                 {
 386                                         doread = true;
 387                                         return Token.OP_NE;
 388                                 }
 389                                 if (d == '='){
 390                                         doread = true;
 391                                         return Token.OP_LE;
 392                                 }
 393                                 return Token.OP_LT;
 394                         }
 395
 396                         if (c == '>'){
 397                                 if (d == '='){
 398                                         doread = true;
 399                                         return Token.OP_GE;
 400                                 }
 401                                 return Token.OP_GT;
 402                         }
 403                         return Token.ERROR;
 404                 }
 405
 406                 bool decimal_digits (int c)
 407                 {
 408                         int d;
 409                         bool seen_digits = false;
 410
 411                         if (c != -1)
 412                                 number.Append ((char) c);
 413
 414                         while ((d = peekChar ()) != -1){
 415                                 if (Char.IsDigit ((char)d)){
 416                                         number.Append ((char) d);
 417                                         getChar ();
 418                                         seen_digits = true;
 419                                 } else
 420                                         break;
 421                         }
 422                         return seen_digits;
 423                 }
 424
 425                 void hex_digits (int c)
 426                 {
 427                         int d;
 428
 429                         if (c != -1)
 430                                 number.Append ((char) c);
 431                         while ((d = peekChar ()) != -1){
 432                                 char e = Char.ToUpper ((char) d);
 433
 434                                 if (Char.IsDigit (e) ||
 435                                     (e >= 'A' && e <= 'F')){
 436                                         number.Append ((char) e);
 437                                         getChar ();
 438                                 } else
 439                                         break;
 440                         }
 441                 }
 442
 443                 int real_type_suffix (int c)
 444                 {
 445                         int t;
 446
 447                         switch (c){
 448                         case 'F': case 'f':
 449                                 t =  Token.LITERAL_SINGLE;
 450                                 break;
 451                         case 'D': case 'd':
 452                                 t = Token.LITERAL_DOUBLE;
 453                                 break;
 454                         case 'M': case 'm':
 455                                  t= Token.LITERAL_DECIMAL;
 456                                 break;
 457                         default:
 458                                 return Token.NONE;
 459                         }
 460                         getChar ();
 461                         return t;
 462                 }
 463
 464                 int integer_type_suffix (int c)
 465                 {
 466                         // FIXME: Handle U and L suffixes.
 467                         // We also need to see in which kind of
 468                         // Int the thing fits better according to the spec.
 469                         return Token.LITERAL_INTEGER;
 470                 }
 471
 472                 void adjust_int (int t)
 473                 {
 474                         val = new System.Int32();
 475                         val = System.Int32.Parse (number.ToString (), 0);
 476                 }
 477
 478                 int adjust_real (int t)
 479                 {
 480                         string s = number.ToString ();
 481
 482                         Console.WriteLine (s);
 483                         switch (t){
 484                         case Token.LITERAL_DECIMAL:
 485                                 val = new System.Decimal ();
 486                                 val = System.Decimal.Parse (
 487                                         s, styles, csharp_format_info);
 488                                 break;
 489                         case Token.LITERAL_DOUBLE:
 490                                 val = new System.Double ();
 491                                 val = System.Double.Parse (
 492                                         s, styles, csharp_format_info);
 493                                 break;
 494                         case Token.LITERAL_SINGLE:
 495                                 val = new System.Double ();
 496                                 val = (float) System.Double.Parse (
 497                                         s, styles, csharp_format_info);
 498                                 break;
 499
 500                         case Token.NONE:
 501                                 val = new System.Double ();
 502                                 val = System.Double.Parse (
 503                                         s, styles, csharp_format_info);
 504                                 t = Token.LITERAL_DOUBLE;
 505                                 break;
 506                         }
 507                         return t;
 508                 }
 509
 510                 //
 511                 // Invoked if we know we have .digits or digits
 512                 //
 513                 int is_number (int c)
 514                 {
 515                         bool is_real = false;
 516                         number = new System.Text.StringBuilder ();
 517                         int type;
 518
 519                         number.Length = 0;
 520
 521                         if (Char.IsDigit ((char)c)){
 522                                 if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){
 523                                         getChar ();
 524                                         hex_digits (-1);
 525                                         val = new System.Int32 ();
 526                                         val = System.Int32.Parse (number.ToString (), NumberStyles.HexNumber);
 527                                         return integer_type_suffix (peekChar ());
 528                                 }
 529                                 decimal_digits (c);
 530                                 c = getChar ();
 531                         }
 532
 533                         //
 534                         // We need to handle the case of
 535                         // "1.1" vs "1.string" (LITERAL_SINGLE vs NUMBER DOT IDENTIFIER)
 536                         //
 537                         if (c == '.'){
 538                                 if (decimal_digits ('.')){
 539                                         is_real = true;
 540                                         c = peekChar ();
 541                                 } else {
 542                                         putback ('.');
 543                                         number.Length -= 1;
 544                                         adjust_int (Token.LITERAL_INTEGER);
 545                                         return Token.LITERAL_INTEGER;
 546                                 }
 547                         }
 548
 549                         if (c == 'e' || c == 'E'){
 550                                 is_real = true;
 551                                 number.Append ("e");
 552                                 getChar ();
 553
 554                                 c = peekChar ();
 555                                 if (c == '+'){
 556                                         number.Append ((char) c);
 557                                         getChar ();
 558                                         c = peekChar ();
 559                                 } else if (c == '-'){
 560                                         number.Append ((char) c);
 561                                         getChar ();
 562                                         c = peekChar ();
 563                                 }
 564                                 decimal_digits (-1);
 565                                 c = peekChar ();
 566                         }
 567
 568                         type = real_type_suffix (c);
 569                         if (type == Token.NONE && !is_real){
 570                                 type = integer_type_suffix (c);
 571                                 adjust_int (type);
 572                                 putback (c);
 573                                 return type;
 574                         } else
 575                                 is_real = true;
 576
 577                         if (is_real)
 578                                 return adjust_real (type);
 579
 580                         Console.WriteLine ("This should not be reached");
 581                         throw new Exception ("Is Number should never reach this point");
 582                 }
 583
 584                 int escape (int c)
 585                 {
 586                         int d;
 587                         int v;
 588
 589                         d = peekChar ();
 590                         if (c != '\\')
 591                                 return c;
 592
 593                         switch (d){
 594                         case 'a':
 595                                 v = '\a'; break;
 596                         case 'b':
 597                                 v = '\b'; break;
 598                         case 'n':
 599                                 v = '\n'; break;
 600                         case 't':
 601                                 v = '\t'; break;
 602                         case 'v':
 603                                 v = '\v'; break;
 604                         case 'r':
 605                                 v = 'c'; break;
 606                         case '\\':
 607                                 v = '\\'; break;
 608                         case 'f':
 609                                 v = '\f'; break;
 610                         case '0':
 611                                 v = 0; break;
 612                         case '"':
 613                                 v = '"'; break;
 614                         case '\'':
 615                                 v = '\''; break;
 616                         default:
 617                                 error_details = "cs1009: Unrecognized escape sequence " + (char)d;
 618                                 return -1;
 619                         }
 620                         getChar ();
 621                         return v;
 622                 }
 623
 624                 int getChar ()
 625                 {
 626                         if (putback_char != -1){
 627                                 int x = putback_char;
 628                                 putback_char = -1;
 629
 630                                 return x;
 631                         }
 632                         return reader.Read ();
 633                 }
 634
 635                 int peekChar ()
 636                 {
 637                         if (putback_char != -1)
 638                                 return putback_char;
 639                         return reader.Peek ();
 640                 }
 641
 642                 void putback (int c)
 643                 {
 644                         if (putback_char != -1)
 645                                 throw new Exception ("This should not happen putback on putback");
 646                         putback_char = c;
 647                 }
 648
 649                 public bool advance ()
 650                 {
 651                         return current_token != Token.EOF ;
 652                 }
 653
 654                 public Object Value {
 655                         get {
 656                                 return val;
 657                         }
 658                 }
 659
 660                 public Object value ()
 661                 {
 662                         return val;
 663                 }
 664
 665                 private bool IsEOL(int currentChar)
 666                 {
 667                         if (currentChar ==  0x0D)
 668                         {
 669                                 if (peekChar() ==  0x0A) // if it is a CR-LF pair consume LF also
 670                                         getChar();
 671
 672                                 return true;
 673                         }
 674                         return (currentChar ==  -1 || currentChar ==  0x0A || currentChar ==  0x2028 || currentChar ==  0x2029);
 675                 }
 676
 677                 private int DropComments()
 678                 {
 679                         int d;
 680                         while (!IsEOL(d = getChar ()))
 681                                 col++;
 682                         line++;
 683                         ref_line++;
 684                         col = 0;
 685
 686                         return Token.EOL;
 687                 }
 688
 689                 public int token ()
 690                 {
 691                         int lastToken = current_token;
 692                         do
 693                         {
 694                                 current_token = xtoken ();
 695                                 if (current_token == 0)
 696                                         return Token.EOF;
 697                                 if (current_token == Token.REM)
 698                                         current_token = DropComments();
 699                         } while (lastToken == Token.EOL && current_token == Token.EOL);
 700
 701                         return current_token;
 702                 }
 703
 704                 private string GetIdentifier()
 705                 {
 706                         int c = getChar();
 707                         if (is_identifier_start_character ((char) c))
 708                                 return GetIdentifier(c);
 709                         else
 710                                 return null;
 711                 }
 712
 713                 private string GetIdentifier(int c)
 714                 {
 715                         System.Text.StringBuilder id = new System.Text.StringBuilder ();
 716
 717                         id.Append ((char) c);
 718
 719                         while ((c = peekChar ()) != -1)
 720                         {
 721                                 if (is_identifier_part_character ((char) c))
 722                                 {
 723                                         id.Append ((char)getChar ());
 724                                         col++;
 725                                 }
 726                                 else
 727                                         break;
 728                         }
 729
 730                         return id.ToString ();
 731                 }
 732
 733                 public int xtoken ()
 734                 {
 735                         int t;
 736                         bool doread = false;
 737                         int c;
 738
 739                         val = null;
 740                         for (;(c = getChar ()) != -1; col++) {
 741
 742                                 // Handle line comments.
 743                                 if (c == '\'')
 744                                         return Token.REM;
 745
 746                                 // Handle EOL.
 747                                 if (IsEOL(c))
 748                                 {
 749                                         line++;
 750                                         ref_line++;
 751                                         col = 0;
 752                                         if (current_token == Token.EOL) // if last token was also EOL keep skipping
 753                                                 continue;
 754                                         return Token.EOL;
 755                                 }
 756
 757                                 // Handle escaped identifiers
 758                                 if (c == '[')
 759                                 {
 760                                         if ((val = GetIdentifier()) == null)
 761                                                 break;
 762                                         if ((c = getChar()) != ']')
 763                                                 break;
 764                                         return Token.IDENTIFIER;
 765                                 }
 766
 767                                 // Handle unescaped identifiers
 768                                 if (is_identifier_start_character ((char) c))
 769                                 {
 770                                         string id;
 771                                         if ((id = GetIdentifier(c)) == null)
 772                                                 break;
 773                                         if (is_keyword(id))
 774                                                 return getKeyword(id);
 775                                         val = id;
 776                                         return Token.IDENTIFIER;
 777                                 }
 778
 779                                 // handle numeric literals
 780                                 if (c == '.'){
 781                                         if (Char.IsDigit ((char) peekChar ()))
 782                                                 return is_number (c);
 783                                         return Token.DOT;
 784                                 }
 785
 786                                 if (Char.IsDigit ((char) c))
 787                                         return is_number (c);
 788
 789                                 /* For now, limited support for pre-processor commands */
 790                                 if (col == 1 && c == '#'){
 791                                         System.Text.StringBuilder s = new System.Text.StringBuilder ();
 792
 793                                         while ((c = getChar ()) != -1 && (c != '\n')){
 794                                                 s.Append ((char) c);
 795                                         }
 796                                         if (String.Compare (s.ToString (), 0, "line", 0, 4) == 0){
 797                                                 string arg = s.ToString ().Substring (5);
 798                                                 int pos;
 799
 800                                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
 801                                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
 802                                                         pos++;
 803
 804                                                         char [] quotes = { '\"' };
 805
 806                                                         ref_name = arg.Substring (pos);
 807                                                         ref_name.TrimStart (quotes);
 808                                                         ref_name.TrimEnd (quotes);
 809                                                 } else
 810                                                         ref_line = System.Int32.Parse (arg);
 811                                         }
 812                                         line++;
 813                                         ref_line++;
 814                                         continue;
 815                                 }
 816
 817                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
 818                                         if (doread){
 819                                                 getChar ();
 820                                                 col++;
 821                                         }
 822                                         return t;
 823                                 }
 824
 825                                 // Treat string literals
 826                                 if (c == '"'){
 827                                         System.Text.StringBuilder s = new System.Text.StringBuilder ();
 828
 829                                         while ((c = getChar ()) != -1){
 830                                                 if (c == '"'){ // TODO: treat double-doublequotes
 831                                                         val = s.ToString ();
 832                                                         return Token.LITERAL_STRING;
 833                                                 }
 834
 835                                                 c = escape (c);
 836                                                 if (c == -1)
 837                                                         return Token.ERROR;
 838                                                 s.Append ((char) c);
 839                                         }
 840                                 }
 841
 842                                 // expand tabs for location and ignore it as whitespace
 843                                 if (c == '\t')
 844                                 {
 845                                         col = (((col + ExpandedTabsSize) / ExpandedTabsSize) * ExpandedTabsSize) - 1;
 846                                         continue;
 847                                 }
 848
 849                                 // white space
 850                                 if (c == ' ' || c == '\f' || c == '\v')
 851                                         continue;
 852
 853                                 error_details = ((char)c).ToString ();
 854
 855                                 return Token.ERROR;
 856                         }
 857
 858                         if (current_token != Token.EOL) // if last token wasn´t EOL send it before EOF
 859                                 return Token.EOL;
 860
 861                         return Token.EOF;
 862                 }
 863
 864                 public void cleanup ()\r
 865                 {
 866 /* borrowed from mcs - have to work it to have preprocessing in mbas
 867 \r
 868                         if (ifstack != null && ifstack.Count >= 1) {\r
 869                                 int state = (int) ifstack.Pop ();\r
 870                                 if ((state & REGION) != 0)\r
 871                                         Report.Error (1038, "#endregion directive expected");\r
 872                                 else \r
 873                                         Report.Error (1027, "#endif directive expected");\r
 874                         }\r
 875 */                              \r
 876                 }\r
 877 \r
 878                 public Tokenizer (System.IO.TextReader input, string fname, ArrayList defines)
 879                 {
 880                         this.ref_name = fname;
 881                         reader = input;
 882                         putback_char = -1;
 883
 884                         Location.Push (fname);
 885                 }
 886
 887         }
 888 }