mcs/mbas/mb-tokenizer.cs

   1 //
   2 // Mono.MonoBASIC.Tokenizer.cs: The Tokenizer for the MonoBASIC compiler
   3 //
   4 // Author: A Rafael D Teixeira (rafaelteixeirabr@hotmail.com)
   5 //
   6 // Based on cs-tokenizer.cs by Miguel de Icaza (miguel@gnu.org)
   7 //
   8 // Licensed under the terms of the GNU GPL
   9 //
  10 // Copyright (C) 2001 A Rafael D Teixeira
  11 //
  12
  13 namespace Mono.MonoBASIC
  14 {
  15         using System;
  16         using System.Text;
  17         using System.Collections;
  18         using System.IO;
  19         using System.Globalization;
  20         using Mono.Languages;
  21         using Mono.CSharp;
  22
  23         /// <summary>
  24         ///    Tokenizer for MonoBASIC source code.
  25         /// </summary>
  26
  27         public class Tokenizer : yyParser.yyInput
  28         {
  29                 TextReader reader;
  30                 // TODO: public SourceFile file_name;
  31                 public string file_name;
  32                 public string ref_name;
  33                 public int ref_line = 1;
  34                 public int line = 1;
  35                 public int col = 1;
  36                 public int current_token = Token.EOL;
  37                 bool handle_get_set = false;
  38
  39                 public int ExpandedTabsSize = 4;
  40
  41                 public string location {
  42                         get {
  43                                 string det;
  44
  45                                 if (current_token == Token.ERROR)
  46                                         det = "detail: " + error_details;
  47                                 else
  48                                         det = "";
  49
  50                                 return "Line:     "+line+" Col: "+col + "\n" +
  51                                        "VirtLine: "+ref_line +
  52                                        " Token: "+current_token + " " + det;
  53                         }
  54                 }
  55
  56                 public bool properties {
  57                         get {
  58                                 return handle_get_set;
  59                         }
  60
  61                         set {
  62                                 handle_get_set = value;
  63                         }
  64                 }
  65
  66                 //
  67                 // Class variables
  68                 //
  69                 static Hashtable keywords;
  70                 static NumberStyles styles;
  71                 static NumberFormatInfo csharp_format_info;
  72
  73                 //
  74                 // Pre-processor
  75                 //
  76                 Hashtable defines;
  77
  78                 const int TAKING        = 1;
  79                 const int TAKEN_BEFORE  = 2;
  80                 const int ELSE_SEEN     = 4;
  81                 const int PARENT_TAKING = 8;
  82                 const int REGION        = 16;
  83
  84                 //
  85                 // pre-processor if stack state:
  86                 //
  87                 Stack ifstack;
  88
  89                 //
  90                 // Values for the associated token returned
  91                 //
  92                 System.Text.StringBuilder number;
  93                 int putback_char;
  94                 Object val;
  95
  96                 //
  97                 // Details about the error encoutered by the tokenizer
  98                 //
  99                 string error_details;
 100
 101                 public string error {
 102                         get {
 103                                 return error_details;
 104                         }
 105                 }
 106
 107                 public int Line {
 108                         get {
 109                                 return line;
 110                         }
 111                 }
 112
 113                 public int Col {
 114                         get {
 115                                 return col;
 116                         }
 117                 }
 118
 119                 static void initTokens ()
 120                 {
 121                         keywords = new Hashtable ();
 122
 123                         keywords.Add ("addhandler", Token.ADDHANDLER);
 124                         keywords.Add ("addressof", Token.ADDRESSOF);
 125                         keywords.Add ("alias", Token.ALIAS);
 126                         keywords.Add ("and", Token.AND);
 127                         keywords.Add ("andalso", Token.ANDALSO);
 128                         keywords.Add ("ansi", Token.ANSI);
 129                         keywords.Add ("as", Token.AS);
 130                         keywords.Add ("assembly", Token.ASSEMBLY);
 131                         keywords.Add ("auto", Token.AUTO);
 132                         keywords.Add ("binary", Token.BINARY);
 133                         keywords.Add ("boolean", Token.BOOLEAN);
 134                         keywords.Add ("byref", Token.BYREF);
 135                         keywords.Add ("byte", Token.BYTE);
 136                         keywords.Add ("byval", Token.BYVAL);
 137                         keywords.Add ("call", Token.CALL);
 138                         keywords.Add ("case", Token.CASE);
 139                         keywords.Add ("catch", Token.CATCH);
 140                         keywords.Add ("cbool", Token.CBOOL);
 141                         keywords.Add ("cbyte", Token.CBYTE);
 142                         keywords.Add ("cchar", Token.CCHAR);
 143                         keywords.Add ("cdate", Token.CDATE);
 144                         keywords.Add ("cdec", Token.CDEC);
 145                         keywords.Add ("cdbl", Token.CDBL);
 146                         keywords.Add ("char", Token.CHAR);
 147                         keywords.Add ("cint", Token.CINT);
 148                         keywords.Add ("class", Token.CLASS);
 149                         keywords.Add ("clng", Token.CLNG);
 150                         keywords.Add ("cobj", Token.COBJ);
 151                         keywords.Add ("compare", Token.COMPARE);
 152                         keywords.Add ("const", Token.CONST);
 153                         keywords.Add ("cshort", Token.CSHORT);
 154                         keywords.Add ("csng", Token.CSNG);
 155                         keywords.Add ("cstr", Token.CSTR);
 156                         keywords.Add ("ctype", Token.CTYPE);
 157                         //keywords.Add ("date", Token.DATE);
 158                         keywords.Add ("decimal", Token.DECIMAL);
 159                         keywords.Add ("declare", Token.DECLARE);
 160                         keywords.Add ("default", Token.DEFAULT);
 161                         keywords.Add ("delegate", Token.DELEGATE);
 162                         keywords.Add ("dim", Token.DIM);
 163                         keywords.Add ("do", Token.DO);
 164                         keywords.Add ("double", Token.DOUBLE);
 165                         keywords.Add ("each", Token.EACH);
 166                         keywords.Add ("else", Token.ELSE);
 167                         keywords.Add ("elseif", Token.ELSEIF);
 168                         keywords.Add ("end", Token.END);
 169                         keywords.Add ("enum", Token.ENUM);
 170                         keywords.Add ("erase", Token.ERASE);
 171                         keywords.Add ("error", Token.ERROR);
 172                         keywords.Add ("event", Token.EVENT);
 173                         keywords.Add ("exit", Token.EXIT);
 174                         keywords.Add ("explicit", Token.EXPLICIT);
 175                         keywords.Add ("false", Token.FALSE);
 176                         keywords.Add ("finally", Token.FINALLY);
 177                         keywords.Add ("for", Token.FOR);
 178                         keywords.Add ("friend", Token.FRIEND);
 179                         keywords.Add ("function", Token.FUNCTION);
 180                         keywords.Add ("get", Token.GET);
 181                         //keywords.Add ("gettype", Token.GETTYPE);
 182                         keywords.Add ("goto", Token.GOTO);
 183                         keywords.Add ("handles", Token.HANDLES);
 184                         keywords.Add ("if", Token.IF);
 185                         keywords.Add ("implements", Token.IMPLEMENTS);
 186                         keywords.Add ("imports", Token.IMPORTS);
 187                         keywords.Add ("in", Token.IN);
 188                         keywords.Add ("inherits", Token.INHERITS);
 189                         keywords.Add ("integer", Token.INTEGER);
 190                         keywords.Add ("interface", Token.INTERFACE);
 191                         keywords.Add ("is", Token.IS);
 192                         keywords.Add ("let ", Token.LET );
 193                         keywords.Add ("lib ", Token.LIB );
 194                         keywords.Add ("like ", Token.LIKE );
 195                         keywords.Add ("long", Token.LONG);
 196                         keywords.Add ("loop", Token.LOOP);
 197                         keywords.Add ("me", Token.ME);
 198                         keywords.Add ("mod", Token.MOD);
 199                         keywords.Add ("module", Token.MODULE);
 200                         keywords.Add ("mustinherit", Token.MUSTINHERIT);
 201                         keywords.Add ("mustoverride", Token.MUSTOVERRIDE);
 202                         keywords.Add ("mybase", Token.MYBASE);
 203                         keywords.Add ("myclass", Token.MYCLASS);
 204                         keywords.Add ("namespace", Token.NAMESPACE);
 205                         keywords.Add ("new", Token.NEW);
 206                         keywords.Add ("next", Token.NEXT);
 207                         keywords.Add ("not", Token.NOT);
 208                         keywords.Add ("nothing", Token.NOTHING);
 209                         keywords.Add ("notinheritable", Token.NOTINHERITABLE);
 210                         keywords.Add ("notoverridable", Token.NOTOVERRIDABLE);
 211                         keywords.Add ("object", Token.OBJECT);
 212                         keywords.Add ("off", Token.OFF);
 213                         keywords.Add ("on", Token.ON);
 214                         keywords.Add ("option", Token.OPTION);
 215                         keywords.Add ("optional", Token.OPTIONAL);
 216                         keywords.Add ("or", Token.OR);
 217                         keywords.Add ("orelse", Token.ORELSE);
 218                         keywords.Add ("overloads", Token.OVERLOADS);
 219                         keywords.Add ("overridable", Token.OVERRIDABLE);
 220                         keywords.Add ("overrides", Token.OVERRIDES);
 221                         keywords.Add ("paramarray", Token.PARAM_ARRAY);
 222                         keywords.Add ("preserve", Token.PRESERVE);
 223                         keywords.Add ("private", Token.PRIVATE);
 224                         keywords.Add ("property", Token.PROPERTY);
 225                         keywords.Add ("protected", Token.PROTECTED);
 226                         keywords.Add ("public", Token.PUBLIC);
 227                         keywords.Add ("raiseevent", Token.RAISEEVENT);
 228                         keywords.Add ("readonly", Token.READONLY);
 229                         keywords.Add ("redim", Token.REDIM);
 230                         keywords.Add ("rem", Token.REM);
 231                         keywords.Add ("removehandler", Token.REMOVEHANDLER);
 232                         keywords.Add ("resume", Token.RESUME);
 233                         keywords.Add ("return", Token.RETURN);
 234                         keywords.Add ("select", Token.SELECT);
 235                         keywords.Add ("set", Token.SET);
 236                         keywords.Add ("shadows", Token.SHADOWS);
 237                         keywords.Add ("shared", Token.SHARED);
 238                         keywords.Add ("short", Token.SHORT);
 239                         keywords.Add ("single", Token.SINGLE);
 240                         keywords.Add ("sizeof", Token.SIZEOF);
 241                         keywords.Add ("static", Token.STATIC);
 242                         keywords.Add ("step", Token.STEP);
 243                         keywords.Add ("stop", Token.STOP);
 244                         keywords.Add ("strict", Token.STRICT);
 245                         keywords.Add ("string", Token.STRING);
 246                         keywords.Add ("structure", Token.STRUCTURE);
 247                         keywords.Add ("sub", Token.SUB);
 248                         keywords.Add ("synclock", Token.SYNCLOCK);
 249                         keywords.Add ("text", Token.TEXT);
 250                         keywords.Add ("then", Token.THEN);
 251                         keywords.Add ("throw", Token.THROW);
 252                         keywords.Add ("to", Token.TO);
 253                         keywords.Add ("true", Token.TRUE);
 254                         keywords.Add ("try", Token.TRY);
 255                         keywords.Add ("typeof", Token.TYPEOF);
 256                         keywords.Add ("unicode", Token.UNICODE);
 257                         keywords.Add ("until", Token.UNTIL);
 258                         keywords.Add ("variant", Token.VARIANT);
 259                         keywords.Add ("when", Token.WHEN);
 260                         keywords.Add ("while", Token.WHILE);
 261                         keywords.Add ("with", Token.WITH);
 262                         keywords.Add ("withevents", Token.WITHEVENTS);
 263                         keywords.Add ("writeonly", Token.WRITEONLY);
 264                         keywords.Add ("xor", Token.XOR);
 265
 266                         if (Parser.UseExtendedSyntax){
 267                                 keywords.Add ("yield", Token.YIELD);
 268                         }
 269
 270                 }
 271
 272                 //
 273                 // Class initializer
 274                 //
 275                 static Tokenizer ()
 276                 {
 277                         initTokens ();
 278                         csharp_format_info = new NumberFormatInfo ();
 279                         csharp_format_info.CurrencyDecimalSeparator = ".";
 280                         styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint;
 281                 }
 282
 283                 bool is_keyword (string name)
 284                 {
 285                         bool res;
 286
 287                         res = keywords.Contains(name.ToLower());
 288                         if ((name == "get" || name == "set") && handle_get_set == false)
 289                                 return false;
 290                         return res;
 291                 }
 292
 293                 int getKeyword (string name)
 294                 {
 295                         return (int) (keywords [name.ToLower()]);
 296                 }
 297
 298                 public Location Location {
 299                         get {
 300                                 return new Location (ref_line);
 301                         }
 302                 }
 303
 304                 void define (string def)
 305                 {
 306                         if (!RootContext.AllDefines.Contains(def)){
 307                                 RootContext.AllDefines [def] = true;
 308                         }
 309                         if (defines.Contains (def))
 310                                 return;
 311                         defines [def] = true;
 312                 }
 313
 314                 public bool PropertyParsing {
 315                         get {
 316                                 return handle_get_set;
 317                         }
 318
 319                         set {
 320                                 handle_get_set = value;
 321                         }
 322                 }
 323
 324                 bool is_identifier_start_character (char c)
 325                 {
 326                         return Char.IsLetter (c) || c == '_' ;
 327                 }
 328
 329                 bool is_identifier_part_character (char c)
 330                 {
 331                         return (Char.IsLetter (c) || Char.IsDigit (c) || c == '_');
 332                 }
 333
 334                 int is_punct (char c, ref bool doread)
 335                 {
 336                         int idx = "{}[](),:;~+-*/%&|^!=<>?".IndexOf (c);
 337                         int d;
 338                         int t;
 339
 340                         doread = false;
 341
 342                         switch (c){
 343                         case '[':
 344                                 return Token.OPEN_BRACKET;
 345                         case ']':
 346                                 return Token.CLOSE_BRACKET;
 347                         case '{':
 348                                 return Token.OPEN_BRACE;
 349                         case '}':
 350                                 return Token.CLOSE_BRACE;
 351                         case '(':
 352                                 return Token.OPEN_PARENS;
 353                         case ')':
 354                                 return Token.CLOSE_PARENS;
 355                         case ',':
 356                                 return Token.COMMA;
 357                         //case ':':
 358                         //      return Token.COLON;
 359                         case '?':
 360                                 return Token.INTERR;
 361                         //case '&':
 362                         //      return Token.OP_CONCAT;
 363                         }
 364
 365                         d = peekChar ();
 366                         if (c == '+'){
 367
 368                                 if (d == '+')
 369                                         t = Token.OP_INC;
 370                                 else if (d == '=')
 371                                         t = Token.OP_ADD_ASSIGN;
 372                                 else
 373                                         return Token.PLUS;
 374                                 doread = true;
 375                                 return t;
 376                         }
 377                         if (c == '&'){
 378                                 if (d == '=')
 379                                         t = Token.OP_CONCAT_ASSIGN;
 380                                 else
 381                                         return Token.OP_CONCAT;
 382                                 doread = true;
 383                                 return t;
 384                         }
 385                         if (c == '-'){
 386                                 if (d == '=')
 387                                         t = Token.OP_SUB_ASSIGN;
 388                                 else
 389                                         return Token.MINUS;
 390                                 doread = true;
 391                                 return t;
 392                         }
 393
 394                         if (c == '='){
 395                                 /*if (d == '='){
 396                                         doread = true;
 397                                         return Token.OP_EQ;
 398                                 }*/
 399                                 return Token.ASSIGN;
 400                         }
 401
 402                         if (c == '*'){
 403                                 if (d == '='){
 404                                         doread = true;
 405                                         return Token.OP_MULT_ASSIGN;
 406                                 }
 407                                 return Token.STAR;
 408                         }
 409
 410                         if (c == '/'){
 411                                 if (d == '='){
 412                                         doread = true;
 413                                         return Token.OP_DIV_ASSIGN;
 414                                 }
 415                                 return Token.DIV;
 416                         }
 417
 418                         if (c == '\\'){
 419                                 if (d == '='){
 420                                         doread = true;
 421                                         return Token.OP_IDIV_ASSIGN;
 422                                 }
 423                                 return Token.OP_IDIV;
 424                         }
 425
 426                         if (c == '^'){
 427                                 if (d == '='){
 428                                         doread = true;
 429                                         return Token.OP_EXP_ASSIGN;
 430                                 }
 431                                 return Token.OP_EXP;
 432                         }
 433
 434                         if (c == '<'){
 435                                 if (d == '>')
 436                                 {
 437                                         doread = true;
 438                                         return Token.OP_NE;
 439                                 }
 440                                 if (d == '='){
 441                                         doread = true;
 442                                         return Token.OP_LE;
 443                                 }
 444                                 return Token.OP_LT;
 445                         }
 446
 447                         if (c == '>'){
 448                                 if (d == '='){
 449                                         doread = true;
 450                                         return Token.OP_GE;
 451                                 }
 452                                 return Token.OP_GT;
 453                         }
 454                         if (c == ':'){
 455                                 if (d == '='){
 456                                         doread = true;
 457                                         return Token.ATTR_ASSIGN;
 458                                 }
 459                                 return Token.COLON;
 460                         }
 461                         return Token.ERROR;
 462                 }
 463
 464                 bool decimal_digits (int c)
 465                 {
 466                         int d;
 467                         bool seen_digits = false;
 468
 469                         if (c != -1)
 470                                 number.Append ((char) c);
 471
 472                         while ((d = peekChar ()) != -1){
 473                                 if (Char.IsDigit ((char)d)){
 474                                         number.Append ((char) d);
 475                                         getChar ();
 476                                         seen_digits = true;
 477                                 } else
 478                                         break;
 479                         }
 480                         return seen_digits;
 481                 }
 482
 483                 void hex_digits (int c)
 484                 {
 485                         int d;
 486
 487                         if (c != -1)
 488                                 number.Append ((char) c);
 489                         while ((d = peekChar ()) != -1){
 490                                 char e = Char.ToUpper ((char) d);
 491
 492                                 if (Char.IsDigit (e) ||
 493                                     (e >= 'A' && e <= 'F')){
 494                                         number.Append ((char) e);
 495                                         getChar ();
 496                                 } else
 497                                         break;
 498                         }
 499                 }
 500
 501                 int real_type_suffix (int c)
 502                 {
 503                         int t;
 504
 505                         switch (c){
 506                         case 'F': case 'f':
 507                                 t =  Token.LITERAL_SINGLE;
 508                                 break;
 509                         case 'D': case 'd':
 510                                 t = Token.LITERAL_DOUBLE;
 511                                 break;
 512                         case 'M': case 'm':
 513                                  t= Token.LITERAL_DECIMAL;
 514                                 break;
 515                         default:
 516                                 return Token.NONE;
 517                         }
 518                         getChar ();
 519                         return t;
 520                 }
 521
 522                 int integer_type_suffix (int c)
 523                 {
 524                         // FIXME: Handle U and L suffixes.
 525                         // We also need to see in which kind of
 526                         // Int the thing fits better according to the spec.
 527                         return Token.LITERAL_INTEGER;
 528                 }
 529
 530                 void adjust_int (int t)
 531                 {
 532                         val = new System.Int32();
 533                         val = System.Int32.Parse (number.ToString (), 0);
 534                 }
 535
 536                 int adjust_real (int t)
 537                 {
 538                         string s = number.ToString ();
 539
 540                         Console.WriteLine (s);
 541                         switch (t){
 542                         case Token.LITERAL_DECIMAL:
 543                                 val = new System.Decimal ();
 544                                 val = System.Decimal.Parse (
 545                                         s, styles, csharp_format_info);
 546                                 break;
 547                         case Token.LITERAL_DOUBLE:
 548                                 val = new System.Double ();
 549                                 val = System.Double.Parse (
 550                                         s, styles, csharp_format_info);
 551                                 break;
 552                         case Token.LITERAL_SINGLE:
 553                                 val = new System.Double ();
 554                                 val = (float) System.Double.Parse (
 555                                         s, styles, csharp_format_info);
 556                                 break;
 557
 558                         case Token.NONE:
 559                                 val = new System.Double ();
 560                                 val = System.Double.Parse (
 561                                         s, styles, csharp_format_info);
 562                                 t = Token.LITERAL_DOUBLE;
 563                                 break;
 564                         }
 565                         return t;
 566                 }
 567
 568                 //
 569                 // Invoked if we know we have .digits or digits
 570                 //
 571                 int is_number (int c)
 572                 {
 573                         bool is_real = false;
 574                         number = new System.Text.StringBuilder ();
 575                         int type;
 576
 577                         number.Length = 0;
 578
 579                         if (Char.IsDigit ((char)c)){
 580                                 if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){
 581                                         getChar ();
 582                                         hex_digits (-1);
 583                                         val = new System.Int32 ();
 584                                         val = System.Int32.Parse (number.ToString (), NumberStyles.HexNumber);
 585                                         return integer_type_suffix (peekChar ());
 586                                 }
 587                                 decimal_digits (c);
 588                                 c = getChar ();
 589                         }
 590
 591                         //
 592                         // We need to handle the case of
 593                         // "1.1" vs "1.string" (LITERAL_SINGLE vs NUMBER DOT IDENTIFIER)
 594                         //
 595                         if (c == '.'){
 596                                 if (decimal_digits ('.')){
 597                                         is_real = true;
 598                                         c = peekChar ();
 599                                 } else {
 600                                         putback ('.');
 601                                         number.Length -= 1;
 602                                         adjust_int (Token.LITERAL_INTEGER);
 603                                         return Token.LITERAL_INTEGER;
 604                                 }
 605                         }
 606
 607                         if (c == 'e' || c == 'E'){
 608                                 is_real = true;
 609                                 number.Append ("e");
 610                                 getChar ();
 611
 612                                 c = peekChar ();
 613                                 if (c == '+'){
 614                                         number.Append ((char) c);
 615                                         getChar ();
 616                                         c = peekChar ();
 617                                 } else if (c == '-'){
 618                                         number.Append ((char) c);
 619                                         getChar ();
 620                                         c = peekChar ();
 621                                 }
 622                                 decimal_digits (-1);
 623                                 c = peekChar ();
 624                         }
 625
 626                         type = real_type_suffix (c);
 627                         if (type == Token.NONE && !is_real){
 628                                 type = integer_type_suffix (c);
 629                                 adjust_int (type);
 630                                 putback (c);
 631                                 return type;
 632                         } else
 633                                 is_real = true;
 634
 635                         if (is_real)
 636                                 return adjust_real (type);
 637
 638                         Console.WriteLine ("This should not be reached");
 639                         throw new Exception ("Is Number should never reach this point");
 640                 }
 641
 642                 int escape (int c)
 643                 {
 644                         return peekChar ();
 645                 }
 646
 647                 int getChar ()
 648                 {
 649                         if (putback_char != -1){
 650                                 int x = putback_char;
 651                                 putback_char = -1;
 652
 653                                 return x;
 654                         }
 655                         return reader.Read ();
 656                 }
 657
 658                 int peekChar ()
 659                 {
 660                         if (putback_char != -1)
 661                                 return putback_char;
 662                         return reader.Peek ();
 663                 }
 664
 665                 void putback (int c)
 666                 {
 667                         if (putback_char != -1)
 668                                 throw new Exception ("This should not happen putback on putback");
 669                         putback_char = c;
 670                 }
 671
 672                 public bool advance ()
 673                 {
 674                         return current_token != Token.EOF ;
 675                 }
 676
 677                 public Object Value {
 678                         get {
 679                                 return val;
 680                         }
 681                 }
 682
 683                 public Object value ()
 684                 {
 685                         return val;
 686                 }
 687
 688                 private bool IsEOL(int currentChar)
 689                 {
 690                         if (currentChar ==  0x0D)
 691                         {
 692                                 if (peekChar() ==  0x0A) // if it is a CR-LF pair consume LF also
 693                                         getChar();
 694
 695                                 return true;
 696                         }
 697                         return (currentChar ==  -1 || currentChar ==  0x0A || currentChar ==  0x2028 || currentChar ==  0x2029);
 698                 }
 699
 700                 private int DropComments()
 701                 {
 702                         int d;
 703                         while (!IsEOL(d = getChar ()))
 704                                 col++;
 705                         line++;
 706                         ref_line++;
 707                         col = 0;
 708
 709                         return Token.EOL;
 710                 }
 711
 712                 public int token ()
 713                 {
 714                         int lastToken = current_token;
 715                         do
 716                         {
 717                                 current_token = xtoken ();
 718                                 if (current_token == 0)
 719                                         return Token.EOF;
 720                                 if (current_token == Token.REM)
 721                                         current_token = DropComments();
 722                         } while (lastToken == Token.EOL && current_token == Token.EOL);
 723
 724                         return current_token;
 725                 }
 726
 727                 private string GetIdentifier()
 728                 {
 729                         int c = getChar();
 730                         if (is_identifier_start_character ((char) c))
 731                                 return GetIdentifier(c);
 732                         else
 733                                 return null;
 734                 }
 735
 736                 private string GetIdentifier(int c)
 737                 {
 738                         System.Text.StringBuilder id = new System.Text.StringBuilder ();
 739
 740                         id.Append ((char) c);
 741
 742                         while ((c = peekChar ()) != -1)
 743                         {
 744                                 if (is_identifier_part_character ((char) c))
 745                                 {
 746                                         id.Append ((char)getChar ());
 747                                         col++;
 748                                 }
 749                                 else
 750                                         break;
 751                         }
 752
 753                         return id.ToString ();
 754                 }
 755
 756                 private bool tokens_seen = false;
 757
 758                 public int xtoken ()
 759                 {
 760                         int t;
 761                         bool doread = false;
 762                         int c;
 763
 764                         val = null;
 765                         for (;(c = getChar ()) != -1; col++) {
 766
 767                                 // Handle line comments.
 768                                 if (c == '\'')
 769                                         return Token.REM;
 770
 771                                 // Handle line continuation character
 772                                 if (c == '_')
 773                                 {
 774                                         int d = getChar();
 775                                         putback(d);
 776                                         if (!is_identifier_part_character((char)d)) {
 777                                                 while ((c = getChar ()) != -1 && !IsEOL(c)) {}
 778                                                 c = getChar ();
 779                                         }
 780                                 }
 781                                 // Handle EOL.
 782                                 if (IsEOL(c))
 783                                 {
 784                                         line++;
 785                                         ref_line++;
 786                                         col = 0;
 787                                         tokens_seen = false;
 788                                         if (current_token == Token.EOL) // if last token was also EOL keep skipping
 789                                                 continue;
 790                                         return Token.EOL;
 791                                 }
 792
 793                                 // Handle escaped identifiers
 794                                 if (c == '[')
 795                                 {
 796                                         if ((val = GetIdentifier()) == null)
 797                                                 break;
 798                                         if ((c = getChar()) != ']')
 799                                                 break;
 800                                         tokens_seen = true;
 801                                         return Token.IDENTIFIER;
 802                                 }
 803
 804                                 // Handle unescaped identifiers
 805                                 if (is_identifier_start_character ((char) c))
 806                                 {
 807                                         string id;
 808                                         if ((id = GetIdentifier(c)) == null)
 809                                                 break;
 810                                         val = id;
 811                                         tokens_seen = true;
 812                                         if (is_keyword(id) && (current_token != Token.DOT))
 813                                                 return getKeyword(id);
 814                                         return Token.IDENTIFIER;
 815                                 }
 816
 817                                 // handle numeric literals
 818                                 if (c == '.')
 819                                 {
 820                                         tokens_seen = true;
 821                                         if (Char.IsDigit ((char) peekChar ()))
 822                                                 return is_number (c);
 823                                         return Token.DOT;
 824                                 }
 825
 826                                 if (Char.IsDigit ((char) c))
 827                                 {
 828                                         tokens_seen = true;
 829                                         return is_number (c);
 830                                 }
 831
 832                                 if (c == '#' && !tokens_seen)
 833                                 {
 834                                         bool cont = true;
 835
 836                                 start_again:
 837
 838                                         cont = handle_preprocessing_directive (cont);
 839
 840                                         if (cont)
 841                                         {
 842                                                 col = 0;
 843                                                 continue;
 844                                         }
 845                                         col = 1;
 846
 847                                         bool skipping = false;
 848                                         for (;(c = getChar ()) != -1; col++)
 849                                         {
 850                                                 if (IsEOL(c))
 851                                                 {
 852                                                         col = 0;
 853                                                         line++;
 854                                                         ref_line++;
 855                                                         skipping = false;
 856                                                 }
 857                                                 else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
 858                                                         continue;
 859                                                 else if (c != '#')
 860                                                 {
 861                                                         skipping = true;
 862                                                         continue;
 863                                                 }
 864                                                 if (c == '#' && !skipping)
 865                                                         goto start_again;
 866                                         }
 867                                         tokens_seen = false;
 868                                         if (c == -1)
 869                                                 Report.Error (1027, Location, "#endif/#endregion expected");
 870                                         continue;
 871                                 }
 872
 873                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
 874                                         if (doread){
 875                                                 getChar ();
 876                                                 col++;
 877                                         }
 878                                         tokens_seen = true;
 879                                         return t;
 880                                 }
 881
 882                                 // Treat string literals
 883                                 if (c == '"'){
 884                                         System.Text.StringBuilder s = new System.Text.StringBuilder ();
 885
 886                                         tokens_seen = true;
 887
 888                                         while ((c = getChar ()) != -1){
 889                                                 if (c == '"'){
 890                                                         if (peekChar() == '"')
 891                                                                 getChar();
 892                                                         else {
 893                                                                 val = s.ToString ();
 894                                                                 return Token.LITERAL_STRING;
 895                                                         }
 896                                                 }
 897
 898                                                 if (IsEOL(c))
 899                                                         return Token.ERROR;
 900
 901                                                 s.Append ((char) c);
 902                                         }
 903                                 }
 904
 905                                 // expand tabs for location and ignore it as whitespace
 906                                 if (c == '\t')
 907                                 {
 908                                         col = (((col + ExpandedTabsSize) / ExpandedTabsSize) * ExpandedTabsSize) - 1;
 909                                         continue;
 910                                 }
 911
 912                                 // white space
 913                                 if (c == ' ' || c == '\f' || c == '\v')
 914                                         continue;
 915
 916                                 error_details = ((char)c).ToString ();
 917
 918                                 return Token.ERROR;
 919                         }
 920
 921                         if (current_token != Token.EOL) // if last token wasn't EOL send it before EOF
 922                                 return Token.EOL;
 923
 924                         return Token.EOF;
 925                 }
 926
 927                 public void cleanup ()
 928                 {
 929 /* borrowed from mcs - have to work it to have preprocessing in mbas
 930
 931                         if (ifstack != null && ifstack.Count >= 1) {
 932                                 int state = (int) ifstack.Pop ();
 933                                 if ((state & REGION) != 0)
 934                                         Report.Error (1038, "#endregion directive expected");
 935                                 else
 936                                         Report.Error (1027, "#endif directive expected");
 937                         }
 938 */
 939                 }
 940
 941                 public Tokenizer (System.IO.TextReader input, string fname, ArrayList defines)
 942                 {
 943                         this.ref_name = fname;
 944                         reader = input;
 945                         putback_char = -1;
 946
 947                         Location.Push (fname);
 948                 }
 949
 950                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
 951
 952                 void get_cmd_arg (out string cmd, out string arg)
 953                 {
 954                         int c;
 955
 956                         tokens_seen = false;
 957                         arg = "";
 958                         static_cmd_arg.Length = 0;
 959
 960                         while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
 961                                 static_cmd_arg.Append ((char) c);
 962                         }
 963
 964                         cmd = static_cmd_arg.ToString ();
 965
 966                         if (c == '\n'){
 967                                 line++;
 968                                 ref_line++;
 969                                 return;
 970                         } else if (c == '\r')
 971                                 col = 0;
 972
 973                         // skip over white space
 974                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
 975                                 ;
 976
 977                         if (c == '\n'){
 978                                 line++;
 979                                 ref_line++;
 980                                 return;
 981                         } else if (c == '\r'){
 982                                 col = 0;
 983                                 return;
 984                         }
 985
 986                         static_cmd_arg.Length = 0;
 987                         static_cmd_arg.Append ((char) c);
 988
 989                         while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
 990                                 static_cmd_arg.Append ((char) c);
 991                         }
 992
 993                         if (c == '\n'){
 994                                 line++;
 995                                 ref_line++;
 996                         } else if (c == '\r')
 997                                 col = 0;
 998                         arg = static_cmd_arg.ToString ().Trim ();
 999
1000                         if (cmd == "End" && arg == "Region") {
1001                                 cmd = "End Region";
1002                                 arg = "";
1003                         }
1004                         if (cmd == "End" && arg == "If") {
1005                                 cmd = "End If";
1006                                 arg = "";
1007                         }
1008
1009                 }
1010
1011                 //
1012                 // Handles the #line directive
1013                 //
1014                 bool PreProcessLine (string arg)
1015                 {
1016                         if (arg == "")
1017                                 return false;
1018
1019                         if (arg == "default"){
1020                                 ref_line = line;
1021                                 ref_name = file_name;
1022                                 Location.Push (ref_name);
1023                                 return true;
1024                         }
1025
1026                         try {
1027                                 int pos;
1028
1029                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1030                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));
1031                                         pos++;
1032
1033                                         char [] quotes = { '\"' };
1034
1035                                         string name = arg.Substring (pos). Trim (quotes);
1036                                         ref_name = name; // TODO: Synchronize with mcs: Location.LookupFile (name);
1037                                         Location.Push (ref_name);
1038                                 } else {
1039                                         ref_line = System.Int32.Parse (arg);
1040                                 }
1041                         } catch {
1042                                 return false;
1043                         }
1044
1045                         return true;
1046                 }
1047
1048                 //
1049                 // Handles #define and #undef
1050                 //
1051                 void PreProcessDefinition (bool is_define, string arg)
1052                 {
1053                         if (arg == "" || arg == "true" || arg == "false"){
1054                                 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
1055                                 return;
1056                         }
1057
1058                         char[] whitespace = { ' ', '\t' };
1059                         if (arg.IndexOfAny (whitespace) != -1){
1060                                 Report.Error (1025, Location, "Single-line comment or end-of-line expected");
1061                                 return;
1062                         }
1063
1064                         foreach (char c in arg){
1065                                 if (!Char.IsLetter (c) && (c != '_')){
1066                                         Report.Error (1001, Location, "Identifier expected");
1067                                         return;
1068                                 }
1069                         }
1070
1071                         if (is_define){
1072                                 if (defines == null)
1073                                         defines = new Hashtable ();
1074                                 define (arg);
1075                         } else {
1076                                 if (defines == null)
1077                                         return;
1078                                 if (defines.Contains (arg))
1079                                         defines.Remove (arg);
1080                         }
1081                 }
1082
1083                 bool eval_val (string s)
1084                 {
1085                         if (s == "true")
1086                                 return true;
1087                         if (s == "false")
1088                                 return false;
1089
1090                         if (defines == null)
1091                                 return false;
1092                         if (defines.Contains (s))
1093                                 return true;
1094
1095                         return false;
1096                 }
1097
1098                 bool pp_primary (ref string s)
1099                 {
1100                         s = s.Trim ();
1101                         int len = s.Length;
1102
1103                         if (len > 0){
1104                                 char c = s [0];
1105
1106                                 if (c == '('){
1107                                         s = s.Substring (1);
1108                                         bool val = pp_expr (ref s);
1109                                         if (s.Length > 0 && s [0] == ')'){
1110                                                 s = s.Substring (1);
1111                                                 return val;
1112                                         }
1113                                         Error_InvalidDirective ();
1114                                         return false;
1115                                 }
1116
1117                                 if (is_identifier_start_character (c)){
1118                                         int j = 1;
1119
1120                                         while (j < len){
1121                                                 c = s [j];
1122
1123                                                 if (is_identifier_part_character (c)){
1124                                                         j++;
1125                                                         continue;
1126                                                 }
1127                                                 bool v = eval_val (s.Substring (0, j));
1128                                                 s = s.Substring (j);
1129                                                 return v;
1130                                         }
1131                                         bool vv = eval_val (s);
1132                                         s = "";
1133                                         return vv;
1134                                 }
1135                         }
1136                         Error_InvalidDirective ();
1137                         return false;
1138                 }
1139
1140                 bool pp_unary (ref string s)
1141                 {
1142                         s = s.Trim ();
1143                         int len = s.Length;
1144
1145                         if (len > 0){
1146                                 if (s [0] == '!'){
1147                                         if (len > 1 && s [1] == '='){
1148                                                 Error_InvalidDirective ();
1149                                                 return false;
1150                                         }
1151                                         s = s.Substring (1);
1152                                         return ! pp_primary (ref s);
1153                                 } else
1154                                         return pp_primary (ref s);
1155                         } else {
1156                                 Error_InvalidDirective ();
1157                                 return false;
1158                         }
1159                 }
1160
1161                 bool pp_eq (ref string s)
1162                 {
1163                         bool va = pp_unary (ref s);
1164
1165                         s = s.Trim ();
1166                         int len = s.Length;
1167                         if (len > 0){
1168                                 if (s [0] == '='){
1169                                         if (len > 2 && s [1] == '='){
1170                                                 s = s.Substring (2);
1171                                                 return va == pp_unary (ref s);
1172                                         } else {
1173                                                 Error_InvalidDirective ();
1174                                                 return false;
1175                                         }
1176                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
1177                                         s = s.Substring (2);
1178
1179                                         return va != pp_unary (ref s);
1180
1181                                 }
1182                         }
1183
1184                         return va;
1185
1186                 }
1187
1188                 bool pp_and (ref string s)
1189                 {
1190                         bool va = pp_eq (ref s);
1191
1192                         s = s.Trim ();
1193                         int len = s.Length;
1194                         if (len > 0){
1195                                 if (s [0] == '&'){
1196                                         if (len > 2 && s [1] == '&'){
1197                                                 s = s.Substring (2);
1198                                                 return (va & pp_eq (ref s));
1199                                         } else {
1200                                                 Error_InvalidDirective ();
1201                                                 return false;
1202                                         }
1203                                 }
1204                         }
1205                         return va;
1206                 }
1207
1208                 //
1209                 // Evaluates an expression for `#if' or `#elif'
1210                 //
1211                 bool pp_expr (ref string s)
1212                 {
1213                         bool va = pp_and (ref s);
1214                         s = s.Trim ();
1215                         int len = s.Length;
1216                         if (len > 0){
1217                                 char c = s [0];
1218
1219                                 if (c == '|'){
1220                                         if (len > 2 && s [1] == '|'){
1221                                                 s = s.Substring (2);
1222                                                 return va | pp_expr (ref s);
1223                                         } else {
1224                                                 Error_InvalidDirective ();
1225                                                 return false;
1226                                         }
1227                                 }
1228                         }
1229
1230                         return va;
1231                 }
1232
1233                 bool eval (string s)
1234                 {
1235                         bool v = pp_expr (ref s);
1236                         s = s.Trim ();
1237                         if (s.Length != 0){
1238                                 Error_InvalidDirective ();
1239                                 return false;
1240                         }
1241
1242                         return v;
1243                 }
1244
1245                 void Error_InvalidDirective ()
1246                 {
1247                         Report.Error (1517, Location, "Invalid pre-processor directive");
1248                 }
1249
1250                 void Error_UnexpectedDirective (string extra)
1251                 {
1252                         Report.Error (
1253                                 1028, Location,
1254                                 "Unexpected processor directive (" + extra + ")");
1255                 }
1256
1257                 void Error_TokensSeen ()
1258                 {
1259                         Report.Error (
1260                                 1032, Location,
1261                                 "Cannot define or undefine pre-processor symbols after a token in the file");
1262                 }
1263
1264                 //
1265                 // if true, then the code continues processing the code
1266                 // if false, the code stays in a loop until another directive is
1267                 // reached.
1268                 //
1269                 bool handle_preprocessing_directive (bool caller_is_taking)
1270                 {
1271                         char [] blank = { ' ', '\t' };
1272                         string cmd, arg;
1273                         bool region_directive = false;
1274
1275                         get_cmd_arg (out cmd, out arg);
1276                         // Eat any trailing whitespaces and single-line comments
1277                         if (arg.IndexOf ("//") != -1)
1278                                 arg = arg.Substring (0, arg.IndexOf ("//"));
1279                         arg = arg.TrimEnd (' ', '\t');
1280
1281                         //
1282                         // The first group of pre-processing instructions is always processed
1283                         //
1284                         switch (cmd){
1285                         case "line":
1286                                 if (!PreProcessLine (arg))
1287                                         Report.Error (
1288                                                 1576, Location,
1289                                                 "Argument to #line directive is missing or invalid");
1290                                 return true;
1291
1292                         case "Region":
1293                                 region_directive = true;
1294                                 arg = "true";
1295                                 goto case "If";
1296
1297                         case "End Region":
1298                                 region_directive = true;
1299                                 goto case "End If";
1300
1301                         case "If":
1302                                 if (arg == ""){
1303                                         Error_InvalidDirective ();
1304                                         return true;
1305                                 }
1306                                 bool taking = false;
1307                                 if (ifstack == null)
1308                                         ifstack = new Stack ();
1309
1310                                 if (ifstack.Count == 0){
1311                                         taking = true;
1312                                 } else {
1313                                         int state = (int) ifstack.Peek ();
1314                                         if ((state & TAKING) != 0)
1315                                                 taking = true;
1316                                 }
1317
1318                                 if (eval (arg) && taking){
1319                                         int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;
1320                                         if (region_directive)
1321                                                 push |= REGION;
1322                                         ifstack.Push (push);
1323                                         return true;
1324                                 } else {
1325                                         int push = (taking ? PARENT_TAKING : 0);
1326                                         if (region_directive)
1327                                                 push |= REGION;
1328                                         ifstack.Push (push);
1329                                         return false;
1330                                 }
1331
1332                         case "End If":
1333                                 if (ifstack == null || ifstack.Count == 0){
1334                                         Error_UnexpectedDirective ("no #if for this #endif");
1335                                         return true;
1336                                 } else {
1337                                         int pop = (int) ifstack.Pop ();
1338
1339                                         if (region_directive && ((pop & REGION) == 0))
1340                                                 Report.Error (1027, Location, "#endif directive expected");
1341                                         else if (!region_directive && ((pop & REGION) != 0))
1342                                                 Report.Error (1038, Location, "#endregion directive expected");
1343
1344                                         if (ifstack.Count == 0)
1345                                                 return true;
1346                                         else {
1347                                                 int state = (int) ifstack.Peek ();
1348
1349                                                 if ((state & TAKING) != 0)
1350                                                         return true;
1351                                                 else
1352                                                         return false;
1353                                         }
1354                                 }
1355
1356                         case "ElseIf":
1357                                 if (ifstack == null || ifstack.Count == 0){
1358                                         Error_UnexpectedDirective ("no #if for this #elif");
1359                                         return true;
1360                                 } else {
1361                                         int state = (int) ifstack.Peek ();
1362
1363                                         if ((state & REGION) != 0) {
1364                                                 Report.Error (1038, Location, "#endregion directive expected");
1365                                                 return true;
1366                                         }
1367
1368                                         if ((state & ELSE_SEEN) != 0){
1369                                                 Error_UnexpectedDirective ("#elif not valid after #else");
1370                                                 return true;
1371                                         }
1372
1373                                         if ((state & (TAKEN_BEFORE | TAKING)) != 0)
1374                                                 return false;
1375
1376                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
1377                                                 state = (int) ifstack.Pop ();
1378                                                 ifstack.Push (state | TAKING | TAKEN_BEFORE);
1379                                                 return true;
1380                                         } else
1381                                                 return false;
1382                                 }
1383
1384                         case "Else":
1385                                 if (ifstack == null || ifstack.Count == 0){
1386                                         Report.Error (
1387                                                 1028, Location,
1388                                                 "Unexpected processor directive (no #if for this #else)");
1389                                         return true;
1390                                 } else {
1391                                         int state = (int) ifstack.Peek ();
1392
1393                                         if ((state & REGION) != 0) {
1394                                                 Report.Error (1038, Location, "#endregion directive expected");
1395                                                 return true;
1396                                         }
1397
1398                                         if ((state & ELSE_SEEN) != 0){
1399                                                 Error_UnexpectedDirective ("#else within #else");
1400                                                 return true;
1401                                         }
1402
1403                                         ifstack.Pop ();
1404
1405                                         bool ret;
1406                                         if ((state & TAKEN_BEFORE) == 0){
1407                                                 ret = ((state & PARENT_TAKING) != 0);
1408                                         } else
1409                                                 ret = false;
1410
1411                                         if (ret)
1412                                                 state |= TAKING;
1413                                         else
1414                                                 state &= ~TAKING;
1415
1416                                         ifstack.Push (state | ELSE_SEEN);
1417
1418                                         return ret;
1419                                 }
1420                         }
1421
1422                         //
1423                         // These are only processed if we are in a `taking' block
1424                         //
1425                         if (!caller_is_taking)
1426                                 return false;
1427
1428                         switch (cmd){
1429                         case "define":
1430                                 /* if (any_token_seen){
1431                                         Error_TokensSeen ();
1432                                         return true;
1433                                 } */
1434                                 PreProcessDefinition (true, arg);
1435                                 return true;
1436
1437                         case "undef":
1438                                 /* if (any_token_seen){
1439                                         Error_TokensSeen ();
1440                                         return true;
1441                                 } */
1442                                 PreProcessDefinition (false, arg);
1443                                 return true;
1444
1445                         case "error":
1446                                 Report.Error (1029, Location, "#error: '" + arg + "'");
1447                                 return true;
1448
1449                         case "warning":
1450                                 Report.Warning (1030, Location, "#warning: '" + arg + "'");
1451                                 return true;
1452                         }
1453
1454                         Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");
1455                         return true;
1456
1457                 }
1458
1459         }
1460 }