2005-07-13 Maverson Eduardo Schulze Rosa <maverson@gmail.com>

[mono.git] / mcs / mbas / mb-tokenizer.cs

diff --git a/mcs/mbas/mb-tokenizer.cs b/mcs/mbas/mb-tokenizer.cs
index 612f8a04f3c422e3de5c6a0ad8dc6b51408aa478..788a8d0ab3a560a457fcc2cff9d845aadac4b23d 100644 (file)
--- a/mcs/mbas/mb-tokenizer.cs
+++ b/mcs/mbas/mb-tokenizer.cs
@@ -2,7 +2,7 @@
 // Mono.MonoBASIC.Tokenizer.cs: The Tokenizer for the MonoBASIC compiler
 //
 // Author: A Rafael D Teixeira (rafaelteixeirabr@hotmail.com)
-//        
+//      : Manjula GHM (mmanjula@novell.com)  
 // Based on cs-tokenizer.cs by Miguel de Icaza (miguel@gnu.org)
 //
 // Licensed under the terms of the GNU GPL
@@ -76,6 +76,7 @@ namespace Mono.MonoBASIC
                StringBuilder number;
                int putback_char = -1;
                Object val;
+               long lon = 0;
                
                //
                // Details about the error encoutered by the tokenizer
@@ -210,7 +211,7 @@ namespace Mono.MonoBASIC
                        keywords.Add ("is", Token.IS);
                        keywords.Add ("let ", Token.LET ); // An unused VB.NET keyword
                        keywords.Add ("lib ", Token.LIB );
-                       keywords.Add ("like ", Token.LIKE );
+                       keywords.Add ("like", Token.LIKE );
                        keywords.Add ("long", Token.LONG);
                        keywords.Add ("loop", Token.LOOP);
                        keywords.Add ("me", Token.ME);
@@ -498,7 +499,6 @@ namespace Mono.MonoBASIC
                        
                        if (c != -1)
                                number.Append ((char) c);
-                       
                        while ((d = peekChar ()) != -1){
                                if (Char.IsDigit ((char)d)){
                                        number.Append ((char) d);
@@ -541,7 +541,16 @@ namespace Mono.MonoBASIC
                                switch (c){
                                case 'S': case 's':
                                        t =  Token.LITERAL_INTEGER; // SHORT ?
-                                       val = ((IConvertible)val).ToInt16(null);
+                       
+                               // hexadecimal literals - like &H8000S is "-32768" 
+                               // and not an overflow exception 
+                               // Check for other literals ???
+
+                                       if(lon == 32768) {
+                                                val = (short) lon;
+                                       }
+                                       else 
+                                               val = ((IConvertible)val).ToInt16(null);
                                        break;
                                case 'I': case 'i':
                                        t = Token.LITERAL_INTEGER;
@@ -615,7 +624,8 @@ namespace Mono.MonoBASIC
                                } else
                                        break;
                        }
-                       return System.Int64.Parse (hexNumber.ToString(), NumberStyles.HexNumber);
+                       lon = System.Int64.Parse (hexNumber.ToString(), NumberStyles.HexNumber);
+                       return lon;
                }
 
                long octal_digits ()
@@ -662,12 +672,14 @@ namespace Mono.MonoBASIC
                        bool is_real = false;
                        number = new StringBuilder ();
                        int type;
+                       bool non_prefixdecimal = false; //To capture decimals like .50
 
                        number.Length = 0;
 
                        if (Char.IsDigit ((char)c)){
                                decimal_digits (c);
-                               c = peekChar ();
+                               c = peekChar ();        
+                               non_prefixdecimal = true;
                        }
 
                        //
@@ -675,6 +687,8 @@ namespace Mono.MonoBASIC
                        // "1.1" vs "1.ToString()" (LITERAL_SINGLE vs NUMBER DOT IDENTIFIER)
                        //
                        if (c == '.'){
+                               if (non_prefixdecimal == false)
+                                        putback ('.');
                                if (decimal_digits (getChar())){
                                        is_real = true;
                                        c = peekChar ();
@@ -731,6 +745,7 @@ namespace Mono.MonoBASIC
                                return putback_char;
                        return reader.Peek ();
                }
+               
 
                void putback (int c)
                {
@@ -778,12 +793,16 @@ namespace Mono.MonoBASIC
 
                private int DropComments()              
                {
-                       int d;
-                       while (!IsEOL(d = getChar ()))
+                       //int d;
+                       while (!IsEOL(/*d =*/ getChar ()))
                                col++;
 
                        return Token.EOL;
                }       
+               
+               public bool putbacktoken = false;
+               public bool flag = false;               
+               int next_token;
                        
                public int token ()
                {
@@ -791,6 +810,14 @@ namespace Mono.MonoBASIC
                        do
                        {
                                current_token = xtoken ();
+                               if(current_token == Token.END) {
+                                       next_token = xtoken();
+                                       putbacktoken = true;
+                                       if (next_token == Token.EOL) 
+                                               return Token.END_EOL;
+                                        else 
+                                               return Token.END;
+                               }       
                                if (current_token == 0) 
                                        return Token.EOF;
                                if (current_token == Token.REM)
@@ -860,6 +887,11 @@ namespace Mono.MonoBASIC
                        bool doread = false;
                        int c;
 
+                       if (putbacktoken == true) {
+                               putbacktoken = false;
+                               return next_token;
+                       }
+       
                        val = null;
                        for (;(c = getChar ()) != -1; col++) {
                        
@@ -872,7 +904,8 @@ namespace Mono.MonoBASIC
                                                c = getChar ();                 
                                        }               
                                }
-
+                                       
+                               
                                // white space
                                if (is_whitespace(c)) {
                                        // expand tabs for location
@@ -925,6 +958,14 @@ namespace Mono.MonoBASIC
                                }
                        
                                // handle numeric literals
+
+                               if (Char.IsDigit ((char) c))
+                                {
+                                        cant_have_a_type_character = true;
+                                        tokens_seen = true;
+                                        return is_number (c);
+                                }
+
                                if (c == '.')
                                {
                                        cant_have_a_type_character = true;
@@ -933,14 +974,6 @@ namespace Mono.MonoBASIC
                                                return is_number (c);
                                        return Token.DOT;
                                }
-                               
-                               if (Char.IsDigit ((char) c))
-                               {
-                                       cant_have_a_type_character = true;
-                                       tokens_seen = true;
-                                       return is_number (c);
-                               }
-
                                if ((t = is_punct ((char)c, ref doread)) != Token.ERROR) {
                                        cant_have_a_type_character = true;
 
@@ -1051,7 +1084,7 @@ namespace Mono.MonoBASIC
                {
                        int t;
                        
-                       for(t = token(); t != Token.HASH && t != Token.EOF; t = token());
+                       for(t = token(); t != Token.HASH && t != Token.EOF ; t = token()); 
 
                        if(t == Token.EOF)
                                throw new ApplicationException("Unexpected EOF while looking for a pre-processor directive");