// Mono.MonoBASIC.Tokenizer.cs: The Tokenizer for the MonoBASIC compiler
//
// Author: A Rafael D Teixeira (rafaelteixeirabr@hotmail.com)
-//
+// : Manjula GHM (mmanjula@novell.com)
// Based on cs-tokenizer.cs by Miguel de Icaza (miguel@gnu.org)
//
// Licensed under the terms of the GNU GPL
int line = 0;
int col = 1;
public int current_token = Token.ERROR;
+ public int last_token = Token.ERROR;
bool handle_get_set = false;
bool cant_have_a_type_character = false;
StringBuilder number;
int putback_char = -1;
Object val;
+ long lon = 0;
//
// Details about the error encoutered by the tokenizer
keywords.Add ("is", Token.IS);
keywords.Add ("let ", Token.LET ); // An unused VB.NET keyword
keywords.Add ("lib ", Token.LIB );
- keywords.Add ("like ", Token.LIKE );
+ keywords.Add ("like", Token.LIKE );
keywords.Add ("long", Token.LONG);
keywords.Add ("loop", Token.LOOP);
keywords.Add ("me", Token.ME);
if (c != -1)
number.Append ((char) c);
-
while ((d = peekChar ()) != -1){
if (Char.IsDigit ((char)d)){
number.Append ((char) d);
switch (c){
case 'S': case 's':
t = Token.LITERAL_INTEGER; // SHORT ?
- val = ((IConvertible)val).ToInt16(null);
+
+ // hexadecimal literals - like &H8000S is "-32768"
+ // and not an overflow exception
+ // Check for other literals ???
+
+ if(lon == 32768) {
+ val = (short) lon;
+ }
+ else
+ val = ((IConvertible)val).ToInt16(null);
break;
case 'I': case 'i':
t = Token.LITERAL_INTEGER;
} else
break;
}
- return System.Int64.Parse (hexNumber.ToString(), NumberStyles.HexNumber);
+ lon = System.Int64.Parse (hexNumber.ToString(), NumberStyles.HexNumber);
+ return lon;
}
long octal_digits ()
bool is_real = false;
number = new StringBuilder ();
int type;
+ bool non_prefixdecimal = false; //To capture decimals like .50
number.Length = 0;
if (Char.IsDigit ((char)c)){
decimal_digits (c);
- c = peekChar ();
+ c = peekChar ();
+ non_prefixdecimal = true;
}
//
// "1.1" vs "1.ToString()" (LITERAL_SINGLE vs NUMBER DOT IDENTIFIER)
//
if (c == '.'){
+ if (non_prefixdecimal == false)
+ putback ('.');
if (decimal_digits (getChar())){
is_real = true;
c = peekChar ();
return putback_char;
return reader.Peek ();
}
+
void putback (int c)
{
private int DropComments()
{
- int d;
- while (!IsEOL(d = getChar ()))
+ //int d;
+ while (!IsEOL(/*d =*/ getChar ()))
col++;
return Token.EOL;
}
+
+ public bool putbacktoken = false;
+ public bool flag = false;
+ int next_token;
public int token ()
{
- int lastToken = current_token;
+ int before_last_token = last_token;
+ last_token = current_token;
do
{
current_token = xtoken ();
+ if(current_token == Token.END) {
+ next_token = xtoken();
+ putbacktoken = true;
+ if (next_token == Token.EOL)
+ return Token.END_EOL;
+ else
+ return Token.END;
+ }
+ if (current_token == Token.COLON) {
+ next_token = xtoken();
+ putbacktoken = true;
+ if (next_token == Token.EOL) {
+ if (last_token != Token.LABELNAME && last_token != Token.LITERAL_INTEGER) {
+ current_token = Token.EOL;
+ putbacktoken = false;
+ }
+ else if (before_last_token == Token.GOTO) {
+ current_token = Token.EOL;
+ putbacktoken = false;
+ }
+ }
+ }
if (current_token == 0)
return Token.EOF;
if (current_token == Token.REM)
current_token = DropComments();
- } while (lastToken == Token.EOL && current_token == Token.EOL);
+ } while (last_token == Token.EOL && current_token == Token.EOL);
return current_token;
}
return null;
}
+ private bool IsLabel ()
+ {
+ char c = (char) peekChar();
+ //putback (c);
+ return (c == ':');
+ }
+
private string GetIdentifier(int c)
{
StringBuilder id = new StringBuilder ();
bool doread = false;
int c;
+ if (putbacktoken == true) {
+ putbacktoken = false;
+ return next_token;
+ }
+
val = null;
for (;(c = getChar ()) != -1; col++) {
int d = peekChar();
if (!is_identifier_part_character((char)d)) {
while ((c = getChar ()) != -1 && !IsEOL(c)) {}
- c = getChar ();
- }
+ c = getChar ();
+ tokens_seen = true;
+ }
}
-
+
+
// white space
if (is_whitespace(c)) {
// expand tabs for location
// Handle escaped identifiers
if (c == '[')
{
+ bool is_first_token_in_line = !tokens_seen;
if ((val = GetIdentifier()) == null)
break;
if ((c = getChar()) != ']')
break;
tokens_seen = true;
+ if (IsLabel() && is_first_token_in_line)
+ return Token.LABELNAME;
+
+ if (last_token == Token.GOTO)
+ return Token.LABELNAME;
return Token.IDENTIFIER;
}
if (is_identifier_start_character ((char) c))
{
string id;
+ bool is_first_token_in_line = !tokens_seen;
if ((id = GetIdentifier(c)) == null)
break;
val = id;
tokens_seen = true;
if (is_keyword(id) && (current_token != Token.DOT))
return getKeyword(id);
+
+ if (IsLabel() && is_first_token_in_line)
+ return Token.LABELNAME;
+
+ if (last_token == Token.GOTO)
+ return Token.LABELNAME;
return Token.IDENTIFIER;
}
}
// handle numeric literals
+
+ if (Char.IsDigit ((char) c))
+ {
+ cant_have_a_type_character = false;
+ tokens_seen = true;
+ return is_number (c);
+ }
+
if (c == '.')
{
cant_have_a_type_character = true;
return is_number (c);
return Token.DOT;
}
-
- if (Char.IsDigit ((char) c))
- {
- cant_have_a_type_character = true;
- tokens_seen = true;
- return is_number (c);
- }
-
if ((t = is_punct ((char)c, ref doread)) != Token.ERROR) {
cant_have_a_type_character = true;
{
int t;
- for(t = token(); t != Token.HASH && t != Token.EOF; t = token());
+ for(t = token(); t != Token.HASH && t != Token.EOF ; t = token());
if(t == Token.EOF)
throw new ApplicationException("Unexpected EOF while looking for a pre-processor directive");
projects / mono.git / blobdiff