// Mono.MonoBASIC.Tokenizer.cs: The Tokenizer for the MonoBASIC compiler
//
// Author: A Rafael D Teixeira (rafaelteixeirabr@hotmail.com)
-//
+// : Manjula GHM (mmanjula@novell.com)
// Based on cs-tokenizer.cs by Miguel de Icaza (miguel@gnu.org)
//
// Licensed under the terms of the GNU GPL
using System.IO;
using System.Globalization;
using Mono.Languages;
- using Mono.CSharp;
+ using Mono.MonoBASIC;
/// <summary>
/// Tokenizer for MonoBASIC source code.
public class Tokenizer : yyParser.yyInput
{
TextReader reader;
- // TODO: public SourceFile file_name;\r
- public string file_name;\r
- public string ref_name;
- public int ref_line = 1;
- public int line = 1;
- public int col = 1;
- public int current_token = Token.EOL;
+ string file_name;
+ string ref_name;
+ int ref_line = 0;
+ int line = 0;
+ int col = 1;
+ public int current_token = Token.ERROR;
+ public int last_token = Token.ERROR;
bool handle_get_set = false;
+ bool cant_have_a_type_character = false;
public int ExpandedTabsSize = 4;
static NumberStyles styles;
static NumberFormatInfo csharp_format_info;
- //\r
- // Pre-processor\r
- //\r
- Hashtable defines;\r
-\r
- const int TAKING = 1;\r
- const int TAKEN_BEFORE = 2;\r
- const int ELSE_SEEN = 4;\r
- const int PARENT_TAKING = 8;\r
- const int REGION = 16; \r
-\r
- //\r
- // pre-processor if stack state:\r
- //\r
- Stack ifstack;\r
-\r
//
// Values for the associated token returned
//
- System.Text.StringBuilder number;
- int putback_char;
+ StringBuilder number;
+ int putback_char = -1;
Object val;
+ long lon = 0;
//
// Details about the error encoutered by the tokenizer
return error_details;
}
}
+
+ public string Source {
+ get {
+ return file_name;
+ }
+
+ set {
+ file_name = value;
+ ref_name = value;
+ Location.SetCurrentSource(file_name);
+ }
+ }
+
+ public string EffectiveSource {
+ get {
+ return ref_name;
+ }
+ set {
+ ref_name = value;
+ Location.SetCurrentSource(ref_name);
+ }
+ }
+
public int Line {
get {
return line;
}
}
+ public int EffectiveLine {
+ get {
+ return ref_line;
+ }
+ set {
+ ref_line = value;
+ }
+ }
+
public int Col {
get {
return col;
keywords.Add ("as", Token.AS);
keywords.Add ("assembly", Token.ASSEMBLY);
keywords.Add ("auto", Token.AUTO);
- keywords.Add ("binary", Token.BINARY);
+ keywords.Add ("binary", Token.BINARY); // Not a VB.NET Keyword
keywords.Add ("boolean", Token.BOOLEAN);
keywords.Add ("byref", Token.BYREF);
keywords.Add ("byte", Token.BYTE);
keywords.Add ("class", Token.CLASS);
keywords.Add ("clng", Token.CLNG);
keywords.Add ("cobj", Token.COBJ);
- keywords.Add ("compare", Token.COMPARE);
+ keywords.Add ("compare", Token.COMPARE); // Not a VB.NET Keyword
keywords.Add ("const", Token.CONST);
keywords.Add ("cshort", Token.CSHORT);
keywords.Add ("csng", Token.CSNG);
keywords.Add ("default", Token.DEFAULT);
keywords.Add ("delegate", Token.DELEGATE);
keywords.Add ("dim", Token.DIM);
+ keywords.Add ("directcast", Token.DIRECTCAST);
keywords.Add ("do", Token.DO);
keywords.Add ("double", Token.DOUBLE);
keywords.Add ("each", Token.EACH);
keywords.Add ("else", Token.ELSE);
keywords.Add ("elseif", Token.ELSEIF);
keywords.Add ("end", Token.END);
+ keywords.Add ("endif", Token.ENDIF); // An unused VB.NET keyword
keywords.Add ("enum", Token.ENUM);
keywords.Add ("erase", Token.ERASE);
keywords.Add ("error", Token.ERROR);
keywords.Add ("event", Token.EVENT);
keywords.Add ("exit", Token.EXIT);
- keywords.Add ("explicit", Token.EXPLICIT);
+ keywords.Add ("explicit", Token.EXPLICIT); // Not a VB.NET keyword
keywords.Add ("false", Token.FALSE);
keywords.Add ("finally", Token.FINALLY);
keywords.Add ("for", Token.FOR);
keywords.Add ("friend", Token.FRIEND);
keywords.Add ("function", Token.FUNCTION);
keywords.Add ("get", Token.GET);
- //keywords.Add ("gettype", Token.GETTYPE);
+ keywords.Add ("gettype", Token.GETTYPE);
+ keywords.Add ("gosub", Token.GOSUB); // An unused VB.NET keyword
keywords.Add ("goto", Token.GOTO);
keywords.Add ("handles", Token.HANDLES);
keywords.Add ("if", Token.IF);
keywords.Add ("integer", Token.INTEGER);
keywords.Add ("interface", Token.INTERFACE);
keywords.Add ("is", Token.IS);
- keywords.Add ("let ", Token.LET );
+ keywords.Add ("let ", Token.LET ); // An unused VB.NET keyword
keywords.Add ("lib ", Token.LIB );
- keywords.Add ("like ", Token.LIKE );
+ keywords.Add ("like", Token.LIKE );
keywords.Add ("long", Token.LONG);
keywords.Add ("loop", Token.LOOP);
keywords.Add ("me", Token.ME);
keywords.Add ("notinheritable", Token.NOTINHERITABLE);
keywords.Add ("notoverridable", Token.NOTOVERRIDABLE);
keywords.Add ("object", Token.OBJECT);
- keywords.Add ("off", Token.OFF);
+ keywords.Add ("off", Token.OFF); // Not a VB.NET Keyword
keywords.Add ("on", Token.ON);
keywords.Add ("option", Token.OPTION);
keywords.Add ("optional", Token.OPTIONAL);
keywords.Add ("shared", Token.SHARED);
keywords.Add ("short", Token.SHORT);
keywords.Add ("single", Token.SINGLE);
- keywords.Add ("sizeof", Token.SIZEOF);
+ keywords.Add ("sizeof", Token.SIZEOF); // Not a VB.NET Keyword
keywords.Add ("static", Token.STATIC);
keywords.Add ("step", Token.STEP);
keywords.Add ("stop", Token.STOP);
- keywords.Add ("strict", Token.STRICT);
+ keywords.Add ("strict", Token.STRICT); // Not a VB.NET Keyword
keywords.Add ("string", Token.STRING);
keywords.Add ("structure", Token.STRUCTURE);
keywords.Add ("sub", Token.SUB);
keywords.Add ("synclock", Token.SYNCLOCK);
- keywords.Add ("text", Token.TEXT);
+ keywords.Add ("text", Token.TEXT); // Not a VB.NET Keyword
keywords.Add ("then", Token.THEN);
keywords.Add ("throw", Token.THROW);
keywords.Add ("to", Token.TO);
keywords.Add ("typeof", Token.TYPEOF);
keywords.Add ("unicode", Token.UNICODE);
keywords.Add ("until", Token.UNTIL);
- keywords.Add ("variant", Token.VARIANT);
+ keywords.Add ("variant", Token.VARIANT); // An unused VB.NET keyword
+ keywords.Add ("wend", Token.WEND); // An unused VB.NET keyword
keywords.Add ("when", Token.WHEN);
keywords.Add ("while", Token.WHILE);
keywords.Add ("with", Token.WITH);
keywords.Add ("writeonly", Token.WRITEONLY);
keywords.Add ("xor", Token.XOR);
- if (Parser.UseExtendedSyntax){\r
- keywords.Add ("yield", Token.YIELD);\r
- }\r
+ if (Parser.UseExtendedSyntax){
+ keywords.Add ("yield", Token.YIELD);
+ }
}
- //
- // Class initializer
- //
static Tokenizer ()
{
initTokens ();
styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint;
}
+ public Tokenizer (System.IO.TextReader input, string fname, ArrayList defines)
+ {
+ this.Source = fname;
+
+ reader = input;
+
+ // putback an EOL at the beginning of a stream. This is a convenience that
+ // allows pre-processor directives to be added to the beginning of a vb file.
+ putback('\n');
+ }
+
bool is_keyword (string name)
{
bool res;
+ name = name.ToLower();
- res = keywords.Contains(name.ToLower());
- if ((name == "get" || name == "set") && handle_get_set == false)
+ res = keywords.Contains(name);
+ if ((name == "GET" || name == "SET") && handle_get_set == false)
return false;
return res;
}
public Location Location {
get {
- return new Location (ref_line);
+ return new Location (ref_line, col);
}
}
- void define (string def)\r
- {\r
- if (!RootContext.AllDefines.Contains(def)){\r
- RootContext.AllDefines [def] = true;\r
- }\r
- if (defines.Contains (def))\r
- return;\r
- defines [def] = true;\r
- }\r
-
public bool PropertyParsing {
get {
return handle_get_set;
int is_punct (char c, ref bool doread)
{
- int idx = "{}[](),:;~+-*/%&|^!=<>?".IndexOf (c);
int d;
int t;
doread = false;
-
+
+ error_details = c.ToString();
+
+ d = peekChar ();
+
switch (c){
case '[':
return Token.OPEN_BRACKET;
return Token.CLOSE_PARENS;
case ',':
return Token.COMMA;
- //case ':':
- // return Token.COLON;
case '?':
return Token.INTERR;
+ case '!':
+ if (is_identifier_start_character((char)d) || cant_have_a_type_character)
+ return Token.EXCLAMATION;
+ return Token.SINGLETYPECHAR;
+ case '$':
+ if (cant_have_a_type_character)
+ return Token.ERROR;
+ return Token.DOLAR_SIGN;
+ case '@':
+ if (cant_have_a_type_character)
+ return Token.ERROR;
+ return Token.AT_SIGN;
+ case '%':
+ if (cant_have_a_type_character)
+ return Token.ERROR;
+ return Token.PERCENT;
+ case '#':
+ if(tokens_seen)
+ {
+ if (cant_have_a_type_character)
+ return ExtractDateTimeLiteral();
+ else
+ return Token.NUMBER_SIGN;
+ }
+ else
+ {
+ tokens_seen = true;
+ return Token.HASH;
+ }
case '&':
- return Token.OP_CONCAT;
+ if (!cant_have_a_type_character)
+ return Token.LONGTYPECHAR;
+ t = handle_integer_literal_in_other_bases(d);
+ if (t == Token.NONE) {
+ t = Token.OP_CONCAT;
+ }
+ return t;
}
- d = peekChar ();
if (c == '+'){
-
if (d == '+')
t = Token.OP_INC;
- else if (d == '=')
- t = Token.OP_ADD_ASSIGN;
- else
+ else
return Token.PLUS;
doread = true;
return t;
}
if (c == '-'){
- if (d == '=')
- t = Token.OP_SUB_ASSIGN;
- else
- return Token.MINUS;
- doread = true;
- return t;
+ return Token.MINUS;
}
if (c == '='){
- /*if (d == '='){
- doread = true;
- return Token.OP_EQ;
- }*/
return Token.ASSIGN;
}
if (c == '*'){
- if (d == '='){
- doread = true;
- return Token.OP_MULT_ASSIGN;
- }
return Token.STAR;
}
if (c == '/'){
- if (d == '='){
- doread = true;
- return Token.OP_DIV_ASSIGN;
- }
return Token.DIV;
}
if (c == '\\'){
- if (d == '='){
- doread = true;
- return Token.OP_IDIV_ASSIGN;
- }
return Token.OP_IDIV;
}
if (c == '^'){
- if (d == '='){
- doread = true;
- return Token.OP_EXP_ASSIGN;
- }
return Token.OP_EXP;
}
doread = true;
return Token.OP_LE;
}
+ if (d == '<')
+ {
+ doread = true;
+ return Token.OP_SHIFT_LEFT;
+ }
return Token.OP_LT;
}
doread = true;
return Token.OP_GE;
}
+ if (d == '>')
+ {
+ doread = true;
+ return Token.OP_SHIFT_RIGHT;
+ }
return Token.OP_GT;
}
+
if (c == ':'){
if (d == '='){
doread = true;
}
return Token.COLON;
}
+
return Token.ERROR;
}
if (c != -1)
number.Append ((char) c);
-
while ((d = peekChar ()) != -1){
if (Char.IsDigit ((char)d)){
number.Append ((char) d);
return seen_digits;
}
- void hex_digits (int c)
- {
- int d;
-
- if (c != -1)
- number.Append ((char) c);
- while ((d = peekChar ()) != -1){
- char e = Char.ToUpper ((char) d);
-
- if (Char.IsDigit (e) ||
- (e >= 'A' && e <= 'F')){
- number.Append ((char) e);
- getChar ();
- } else
- break;
- }
- }
int real_type_suffix (int c)
{
case 'F': case 'f':
t = Token.LITERAL_SINGLE;
break;
- case 'D': case 'd':
+ case 'R': case 'r':
t = Token.LITERAL_DOUBLE;
break;
- case 'M': case 'm':
+ case 'D': case 'd':
t= Token.LITERAL_DECIMAL;
break;
default:
int integer_type_suffix (int c)
{
- // FIXME: Handle U and L suffixes.
- // We also need to see in which kind of
- // Int the thing fits better according to the spec.
- return Token.LITERAL_INTEGER;
+ int t;
+
+ try {
+
+ switch (c){
+ case 'S': case 's':
+ t = Token.LITERAL_INTEGER; // SHORT ?
+
+ // hexadecimal literals - like &H8000S is "-32768"
+ // and not an overflow exception
+ // Check for other literals ???
+
+ if(lon == 32768) {
+ val = (short) lon;
+ }
+ else
+ val = ((IConvertible)val).ToInt16(null);
+ break;
+ case 'I': case 'i':
+ t = Token.LITERAL_INTEGER;
+ val = ((IConvertible)val).ToInt32(null);
+ break;
+ case 'L': case 'l':
+ t= Token.LITERAL_INTEGER; // LONG ?
+ val = ((IConvertible)val).ToInt64(null);
+ break;
+ default:
+ if ((long)val <= System.Int32.MaxValue &&
+ (long)val >= System.Int32.MinValue) {
+ val = ((IConvertible)val).ToInt32(null);
+ return Token.LITERAL_INTEGER;
+ } else {
+ val = ((IConvertible)val).ToInt64(null);
+ return Token.LITERAL_INTEGER; // LONG ?
+ }
+ }
+ getChar ();
+ return t;
+ } catch (Exception e) {
+ val = e.ToString();
+ return Token.ERROR;
+ }
}
- void adjust_int (int t)
- {
- val = new System.Int32();
- val = System.Int32.Parse (number.ToString (), 0);
- }
-
int adjust_real (int t)
{
string s = number.ToString ();
- Console.WriteLine (s);
switch (t){
case Token.LITERAL_DECIMAL:
val = new System.Decimal ();
return t;
}
+ long hex_digits ()
+ {
+ StringBuilder hexNumber = new StringBuilder ();
+
+ int d;
+
+ while ((d = peekChar ()) != -1){
+ char e = Char.ToUpper ((char) d);
+
+ if (Char.IsDigit (e) || (e >= 'A' && e <= 'F')){
+ hexNumber.Append (e);
+ getChar ();
+ } else
+ break;
+ }
+ lon = System.Int64.Parse (hexNumber.ToString(), NumberStyles.HexNumber);
+ return lon;
+ }
+
+ long octal_digits ()
+ {
+ long valueToReturn = 0;
+
+ int d;
+
+ while ((d = peekChar ()) != -1){
+ char e = (char)d;
+ if (Char.IsDigit (e) && (e < '8')){
+ valueToReturn *= 8;
+ valueToReturn += (d - (int)'0');
+ getChar ();
+ } else
+ break;
+ }
+
+ return valueToReturn;
+ }
+
+ int handle_integer_literal_in_other_bases(int peek)
+ {
+ if (peek == 'h' || peek == 'H'){
+ getChar ();
+ val = hex_digits ();
+ return integer_type_suffix (peekChar ());
+ }
+
+ if (peek == 'o' || peek == 'O'){
+ getChar ();
+ val = octal_digits ();
+ return integer_type_suffix (peekChar ());
+ }
+
+ return Token.NONE;
+ }
+
//
// Invoked if we know we have .digits or digits
//
int is_number (int c)
{
bool is_real = false;
- number = new System.Text.StringBuilder ();
+ number = new StringBuilder ();
int type;
+ bool non_prefixdecimal = false; //To capture decimals like .50
number.Length = 0;
if (Char.IsDigit ((char)c)){
- if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){
- getChar ();
- hex_digits (-1);
- val = new System.Int32 ();
- val = System.Int32.Parse (number.ToString (), NumberStyles.HexNumber);
- return integer_type_suffix (peekChar ());
- }
decimal_digits (c);
- c = getChar ();
+ c = peekChar ();
+ non_prefixdecimal = true;
}
//
// We need to handle the case of
- // "1.1" vs "1.string" (LITERAL_SINGLE vs NUMBER DOT IDENTIFIER)
+ // "1.1" vs "1.ToString()" (LITERAL_SINGLE vs NUMBER DOT IDENTIFIER)
//
if (c == '.'){
- if (decimal_digits ('.')){
+ if (non_prefixdecimal == false)
+ putback ('.');
+ if (decimal_digits (getChar())){
is_real = true;
c = peekChar ();
} else {
putback ('.');
number.Length -= 1;
- adjust_int (Token.LITERAL_INTEGER);
- return Token.LITERAL_INTEGER;
+ val = System.Int64.Parse(number.ToString());
+ return integer_type_suffix('.');
}
}
type = real_type_suffix (c);
if (type == Token.NONE && !is_real){
- type = integer_type_suffix (c);
- adjust_int (type);
- putback (c);
- return type;
- } else
- is_real = true;
-
- if (is_real)
- return adjust_real (type);
-
- Console.WriteLine ("This should not be reached");
- throw new Exception ("Is Number should never reach this point");
- }
+ val = System.Int64.Parse(number.ToString());
+ return integer_type_suffix(c);
+ }
- int escape (int c)
- {
- return peekChar ();
+ return adjust_real (type);
}
-
+
int getChar ()
{
if (putback_char != -1){
return putback_char;
return reader.Peek ();
}
+
void putback (int c)
{
private bool IsEOL(int currentChar)
{
- if (currentChar == 0x0D)
- {
+ bool retVal;
+
+ if (currentChar == 0x0D) {
if (peekChar() == 0x0A) // if it is a CR-LF pair consume LF also
getChar();
- return true;
+ retVal = true;
+ }
+ else {
+ retVal = (currentChar == -1 || currentChar == 0x0A || currentChar == 0x2028 || currentChar == 0x2029);
+ }
+
+ if(retVal) {
+ nextLine();
}
- return (currentChar == -1 || currentChar == 0x0A || currentChar == 0x2028 || currentChar == 0x2029);
+
+ return retVal;
}
private int DropComments()
{
- int d;
- while (!IsEOL(d = getChar ()))
+ //int d;
+ while (!IsEOL(/*d =*/ getChar ()))
col++;
- line++;
- ref_line++;
- col = 0;
return Token.EOL;
}
+
+ public bool putbacktoken = false;
+ public bool flag = false;
+ int next_token;
public int token ()
{
- int lastToken = current_token;
+ int before_last_token = last_token;
+ last_token = current_token;
do
{
current_token = xtoken ();
+ if(current_token == Token.END) {
+ next_token = xtoken();
+ putbacktoken = true;
+ if (next_token == Token.EOL)
+ return Token.END_EOL;
+ else
+ return Token.END;
+ }
+ if (current_token == Token.COLON) {
+ next_token = xtoken();
+ putbacktoken = true;
+ if (next_token == Token.EOL) {
+ if (last_token != Token.LABELNAME && last_token != Token.LITERAL_INTEGER) {
+ current_token = Token.EOL;
+ putbacktoken = false;
+ }
+ else if (before_last_token == Token.GOTO) {
+ current_token = Token.EOL;
+ putbacktoken = false;
+ }
+ }
+ }
if (current_token == 0)
return Token.EOF;
if (current_token == Token.REM)
current_token = DropComments();
- } while (lastToken == Token.EOL && current_token == Token.EOL);
+ } while (last_token == Token.EOL && current_token == Token.EOL);
return current_token;
}
return null;
}
+ private bool IsLabel ()
+ {
+ char c = (char) peekChar();
+ //putback (c);
+ return (c == ':');
+ }
+
private string GetIdentifier(int c)
{
- System.Text.StringBuilder id = new System.Text.StringBuilder ();
+ StringBuilder id = new StringBuilder ();
id.Append ((char) c);
else
break;
}
-
- return id.ToString ();
+
+ cant_have_a_type_character = false;
+
+ return id.ToString();
}
+ private bool is_doublequote(int currentChar)
+ {
+ return (currentChar == '"' ||
+ currentChar == 0x201C || // unicode left double-quote character
+ currentChar == 0x201D); // unicode right double-quote character
+ }
+
+ private bool is_whitespace(int c)
+ {
+ return (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0);
+ }
+
private bool tokens_seen = false;
+
+ private void nextLine()
+ {
+ cant_have_a_type_character = true;
+ line++;
+ ref_line++;
+ col = 0;
+ tokens_seen = false;
+ }
public int xtoken ()
{
bool doread = false;
int c;
+ if (putbacktoken == true) {
+ putbacktoken = false;
+ return next_token;
+ }
+
val = null;
for (;(c = getChar ()) != -1; col++) {
- // Handle line comments.
- if (c == '\'')
- return Token.REM;
-
// Handle line continuation character
if (c == '_')
{
- while ((c = getChar ()) != -1 && !IsEOL(c)) {}
- c = getChar ();
+ int d = peekChar();
+ if (!is_identifier_part_character((char)d)) {
+ while ((c = getChar ()) != -1 && !IsEOL(c)) {}
+ c = getChar ();
+ tokens_seen = true;
+ }
+ }
+
+
+ // white space
+ if (is_whitespace(c)) {
+ // expand tabs for location
+ if (c == '\t')
+ col = (((col + ExpandedTabsSize) / ExpandedTabsSize) * ExpandedTabsSize) - 1;
+ cant_have_a_type_character = true;
+ continue;
}
+
+ // Handle line comments.
+ if (c == '\'')
+ return Token.REM;
+
// Handle EOL.
if (IsEOL(c))
{
- line++;
- ref_line++;
- col = 0;
- tokens_seen = false;
if (current_token == Token.EOL) // if last token was also EOL keep skipping
continue;
return Token.EOL;
// Handle escaped identifiers
if (c == '[')
{
+ bool is_first_token_in_line = !tokens_seen;
if ((val = GetIdentifier()) == null)
break;
if ((c = getChar()) != ']')
break;
tokens_seen = true;
+ if (IsLabel() && is_first_token_in_line)
+ return Token.LABELNAME;
+
+ if (last_token == Token.GOTO)
+ return Token.LABELNAME;
return Token.IDENTIFIER;
}
if (is_identifier_start_character ((char) c))
{
string id;
+ bool is_first_token_in_line = !tokens_seen;
if ((id = GetIdentifier(c)) == null)
break;
val = id;
tokens_seen = true;
- if (is_keyword(id))
+ if (is_keyword(id) && (current_token != Token.DOT))
return getKeyword(id);
+
+ if (IsLabel() && is_first_token_in_line)
+ return Token.LABELNAME;
+
+ if (last_token == Token.GOTO)
+ return Token.LABELNAME;
return Token.IDENTIFIER;
}
+ // Treat string literals
+ if (is_doublequote(c)) {
+ cant_have_a_type_character = true;
+ return ExtractStringOrCharLiteral(c);
+ }
+
// handle numeric literals
+
+ if (Char.IsDigit ((char) c))
+ {
+ cant_have_a_type_character = false;
+ tokens_seen = true;
+ return is_number (c);
+ }
+
if (c == '.')
{
+ cant_have_a_type_character = true;
tokens_seen = true;
if (Char.IsDigit ((char) peekChar ()))
return is_number (c);
return Token.DOT;
}
-
- if (Char.IsDigit ((char) c))
- {
- tokens_seen = true;
- return is_number (c);
- }
+ if ((t = is_punct ((char)c, ref doread)) != Token.ERROR) {
+ cant_have_a_type_character = true;
- if (c == '#' && !tokens_seen)
- {\r
- bool cont = true;\r
- \r
- start_again:\r
- \r
- cont = handle_preprocessing_directive (cont);\r
-\r
- if (cont)
- {\r
- col = 0;\r
- continue;\r
- }\r
- col = 1;\r
-\r
- bool skipping = false;\r
- for (;(c = getChar ()) != -1; col++)
- {\r
- if (IsEOL(c))
- {\r
- col = 0;\r
- line++;\r
- ref_line++;\r
- skipping = false;\r
- }
- else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)\r
- continue;\r
- else if (c != '#')
- {\r
- skipping = true;
- continue;\r
- } \r
- if (c == '#' && !skipping)\r
- goto start_again;\r
- }\r
- tokens_seen = false;\r
- if (c == -1)\r
- Report.Error (1027, Location, "#endif/#endregion expected");\r
- continue;\r
- }\r
-
- if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
+ if (t == Token.NONE)
+ continue;
+
if (doread){
getChar ();
col++;
return t;
}
- // Treat string literals
- if (c == '"'){
- System.Text.StringBuilder s = new System.Text.StringBuilder ();
-
- tokens_seen = true;
-
- while ((c = getChar ()) != -1){
- if (c == '"'){
- if (peekChar() == '"')
- getChar();
- else {
- val = s.ToString ();
- return Token.LITERAL_STRING;
- }
- }
-
- if (IsEOL(c))
- return Token.ERROR;
-
- s.Append ((char) c);
- }
- }
-
- // expand tabs for location and ignore it as whitespace
- if (c == '\t')
- {
- col = (((col + ExpandedTabsSize) / ExpandedTabsSize) * ExpandedTabsSize) - 1;
- continue;
- }
-
- // white space
- if (c == ' ' || c == '\f' || c == '\v')
- continue;
-
error_details = ((char)c).ToString ();
-
return Token.ERROR;
}
return Token.EOF;
}
- public void cleanup ()
+ private int ExtractDateTimeLiteral()
+ {
+ int c;
+
+ StringBuilder sb = new StringBuilder();
+ for (;(c = getChar ()) != -1; col++)
+ {
+ if (c == '#') {
+ val = ParseDateLiteral(sb);
+ return Token.LITERAL_DATE;
+ }
+ if (IsEOL(c)) {
+ break;
+ }
+ if (c == '-')
+ c = '/';
+ sb.Append((char)c);
+ }
+ return Token.ERROR;
+ }
+
+ private int ExtractStringOrCharLiteral(int c)
{
-/* borrowed from mcs - have to work it to have preprocessing in mbas
+ StringBuilder s = new StringBuilder ();
+
+ tokens_seen = true;
+
+ while ((c = getChar ()) != -1){
+ if (is_doublequote(c)){
+ if (is_doublequote(peekChar()))
+ getChar();
+ else {
+ //handle Char Literals
+ if (peekChar() == 'C' || peekChar() == 'c') {
+ getChar();
+ if (s.Length == 1) {
+ val = s[0];
+ return Token.LITERAL_CHARACTER;
+ } else {
+ val = "Incorrect length for a character literal";
+ return Token.ERROR;
+ }
+ } else {
+ val = s.ToString ();
+ return Token.LITERAL_STRING;
+ }
+ }
+ }
- if (ifstack != null && ifstack.Count >= 1) {
- int state = (int) ifstack.Pop ();
- if ((state & REGION) != 0)
- Report.Error (1038, "#endregion directive expected");
- else
- Report.Error (1027, "#endif directive expected");
+ if (IsEOL(c)) {
+ return Token.ERROR;
+ }
+
+ s.Append ((char) c);
}
-*/
+
+ return Token.ERROR;
}
- public Tokenizer (System.IO.TextReader input, string fname, ArrayList defines)
+ static IFormatProvider enUSculture = new CultureInfo("en-US", true);
+
+ private DateTime ParseDateLiteral(StringBuilder value)
{
- this.ref_name = fname;
- reader = input;
- putback_char = -1;
-
- Location.Push (fname);
+ try
+ {
+ return DateTime.Parse(value.ToString(),
+ enUSculture,
+ DateTimeStyles.NoCurrentDateDefault | DateTimeStyles.AllowWhiteSpaces);
+ }
+ catch (FormatException ex)
+ {
+ //TODO: What is the correct error number and message?
+ Report.Error (1, Location, string.Format("Invalid date literal '{0}'", value.ToString())
+ + Environment.NewLine + ex.ToString());
+ }
+ catch (Exception)
+ {
+ Report.Error (1, Location, "Error parsing date literal"); //TODO: What is the correct error number and message?
+ }
+ return new DateTime();
}
+
+ public void PositionCursorAtNextPreProcessorDirective()
+ {
+ int t;
+
+ for(t = token(); t != Token.HASH && t != Token.EOF ; t = token());
- static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();\r
- \r
- void get_cmd_arg (out string cmd, out string arg)\r
- {\r
- int c;\r
- \r
- tokens_seen = false;\r
- arg = "";\r
- static_cmd_arg.Length = 0;\r
- \r
- while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){\r
- static_cmd_arg.Append ((char) c);\r
- }\r
-\r
- cmd = static_cmd_arg.ToString ();\r
-\r
- if (c == '\n'){\r
- line++;\r
- ref_line++;\r
- return;\r
- } else if (c == '\r')\r
- col = 0;\r
-\r
- // skip over white space\r
- while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))\r
- ;\r
-\r
- if (c == '\n'){\r
- line++;\r
- ref_line++;\r
- return;\r
- } else if (c == '\r'){\r
- col = 0;\r
- return;\r
- }\r
- \r
- static_cmd_arg.Length = 0;\r
- static_cmd_arg.Append ((char) c);\r
- \r
- while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){\r
- static_cmd_arg.Append ((char) c);\r
- }\r
-\r
- if (c == '\n'){\r
- line++;\r
- ref_line++;\r
- } else if (c == '\r')\r
- col = 0;\r
- arg = static_cmd_arg.ToString ().Trim ();\r
- }\r
-\r
- //\r
- // Handles the #line directive\r
- //\r
- bool PreProcessLine (string arg)\r
- {\r
- if (arg == "")\r
- return false;\r
-\r
- if (arg == "default"){\r
- ref_line = line;\r
- ref_name = file_name;\r
- Location.Push (ref_name);\r
- return true;\r
- }\r
- \r
- try {\r
- int pos;\r
-\r
- if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){\r
- ref_line = System.Int32.Parse (arg.Substring (0, pos));\r
- pos++;\r
- \r
- char [] quotes = { '\"' };\r
- \r
- string name = arg.Substring (pos). Trim (quotes);\r
- ref_name = name; // TODO: Synchronize with mcs: Location.LookupFile (name);\r
- Location.Push (ref_name);\r
- } else {\r
- ref_line = System.Int32.Parse (arg);\r
- }\r
- } catch {\r
- return false;\r
- }\r
- \r
- return true;\r
- }\r
-\r
- //\r
- // Handles #define and #undef\r
- //\r
- void PreProcessDefinition (bool is_define, string arg)\r
- {\r
- if (arg == "" || arg == "true" || arg == "false"){\r
- Report.Error (1001, Location, "Missing identifer to pre-processor directive");\r
- return;\r
- }\r
-\r
- char[] whitespace = { ' ', '\t' };\r
- if (arg.IndexOfAny (whitespace) != -1){\r
- Report.Error (1025, Location, "Single-line comment or end-of-line expected");\r
- return;\r
- }\r
-\r
- foreach (char c in arg){\r
- if (!Char.IsLetter (c) && (c != '_')){\r
- Report.Error (1001, Location, "Identifier expected");\r
- return;\r
- }\r
- }\r
-\r
- if (is_define){\r
- if (defines == null)\r
- defines = new Hashtable ();\r
- define (arg);\r
- } else {\r
- if (defines == null)\r
- return;\r
- if (defines.Contains (arg))\r
- defines.Remove (arg);\r
- }\r
- }\r
-\r
- bool eval_val (string s)\r
- {\r
- if (s == "true")\r
- return true;\r
- if (s == "false")\r
- return false;\r
- \r
- if (defines == null)\r
- return false;\r
- if (defines.Contains (s))\r
- return true;\r
-\r
- return false;\r
- }\r
-\r
- bool pp_primary (ref string s)\r
- {\r
- s = s.Trim ();\r
- int len = s.Length;\r
-\r
- if (len > 0){\r
- char c = s [0];\r
- \r
- if (c == '('){\r
- s = s.Substring (1);\r
- bool val = pp_expr (ref s);\r
- if (s.Length > 0 && s [0] == ')'){\r
- s = s.Substring (1);\r
- return val;\r
- }\r
- Error_InvalidDirective ();\r
- return false;\r
- }\r
- \r
- if (is_identifier_start_character (c)){\r
- int j = 1;\r
-\r
- while (j < len){\r
- c = s [j];\r
- \r
- if (is_identifier_part_character (c)){\r
- j++;\r
- continue;\r
- }\r
- bool v = eval_val (s.Substring (0, j));\r
- s = s.Substring (j);\r
- return v;\r
- }\r
- bool vv = eval_val (s);\r
- s = "";\r
- return vv;\r
- }\r
- }\r
- Error_InvalidDirective ();\r
- return false;\r
- }\r
- \r
- bool pp_unary (ref string s)\r
- {\r
- s = s.Trim ();\r
- int len = s.Length;\r
-\r
- if (len > 0){\r
- if (s [0] == '!'){\r
- if (len > 1 && s [1] == '='){\r
- Error_InvalidDirective ();\r
- return false;\r
- }\r
- s = s.Substring (1);\r
- return ! pp_primary (ref s);\r
- } else\r
- return pp_primary (ref s);\r
- } else {\r
- Error_InvalidDirective ();\r
- return false;\r
- }\r
- }\r
- \r
- bool pp_eq (ref string s)\r
- {\r
- bool va = pp_unary (ref s);\r
-\r
- s = s.Trim ();\r
- int len = s.Length;\r
- if (len > 0){\r
- if (s [0] == '='){\r
- if (len > 2 && s [1] == '='){\r
- s = s.Substring (2);\r
- return va == pp_unary (ref s);\r
- } else {\r
- Error_InvalidDirective ();\r
- return false;\r
- }\r
- } else if (s [0] == '!' && len > 1 && s [1] == '='){\r
- s = s.Substring (2);\r
-\r
- return va != pp_unary (ref s);\r
-\r
- } \r
- }\r
-\r
- return va;\r
- \r
- }\r
- \r
- bool pp_and (ref string s)\r
- {\r
- bool va = pp_eq (ref s);\r
-\r
- s = s.Trim ();\r
- int len = s.Length;\r
- if (len > 0){\r
- if (s [0] == '&'){\r
- if (len > 2 && s [1] == '&'){\r
- s = s.Substring (2);\r
- return (va & pp_eq (ref s));\r
- } else {\r
- Error_InvalidDirective ();\r
- return false;\r
- }\r
- } \r
- }\r
- return va;\r
- }\r
- \r
- //\r
- // Evaluates an expression for `#if' or `#elif'\r
- //\r
- bool pp_expr (ref string s)\r
- {\r
- bool va = pp_and (ref s);\r
- s = s.Trim ();\r
- int len = s.Length;\r
- if (len > 0){\r
- char c = s [0];\r
- \r
- if (c == '|'){\r
- if (len > 2 && s [1] == '|'){\r
- s = s.Substring (2);\r
- return va | pp_expr (ref s);\r
- } else {\r
- Error_InvalidDirective ();\r
- return false;\r
- }\r
- } \r
- }\r
- \r
- return va;\r
- }\r
-\r
- bool eval (string s)\r
- {\r
- bool v = pp_expr (ref s);\r
- s = s.Trim ();\r
- if (s.Length != 0){\r
- Error_InvalidDirective ();\r
- return false;\r
- }\r
-\r
- return v;\r
- }\r
- \r
- void Error_InvalidDirective ()\r
- {\r
- Report.Error (1517, Location, "Invalid pre-processor directive");\r
- }\r
-\r
- void Error_UnexpectedDirective (string extra)\r
- {\r
- Report.Error (\r
- 1028, Location,\r
- "Unexpected processor directive (" + extra + ")");\r
- }\r
-\r
- void Error_TokensSeen ()\r
- {\r
- Report.Error (\r
- 1032, Location,\r
- "Cannot define or undefine pre-processor symbols after a token in the file");\r
- }\r
- \r
- //\r
- // if true, then the code continues processing the code\r
- // if false, the code stays in a loop until another directive is\r
- // reached.\r
- //\r
- bool handle_preprocessing_directive (bool caller_is_taking)\r
- {\r
- char [] blank = { ' ', '\t' };\r
- string cmd, arg;\r
- bool region_directive = false;\r
-\r
- get_cmd_arg (out cmd, out arg);\r
-\r
- // Eat any trailing whitespaces and single-line comments\r
- if (arg.IndexOf ("//") != -1)\r
- arg = arg.Substring (0, arg.IndexOf ("//"));\r
- arg = arg.TrimEnd (' ', '\t');\r
-\r
- //\r
- // The first group of pre-processing instructions is always processed\r
- //\r
- switch (cmd){\r
- case "line":\r
- if (!PreProcessLine (arg))\r
- Report.Error (\r
- 1576, Location,\r
- "Argument to #line directive is missing or invalid");\r
- return true;\r
-\r
- case "region":\r
- region_directive = true;\r
- arg = "true";\r
- goto case "if";\r
-\r
- case "endregion":\r
- region_directive = true;\r
- goto case "endif";\r
- \r
- case "if":\r
- if (arg == ""){\r
- Error_InvalidDirective ();\r
- return true;\r
- }\r
- bool taking = false;\r
- if (ifstack == null)\r
- ifstack = new Stack ();\r
-\r
- if (ifstack.Count == 0){\r
- taking = true;\r
- } else {\r
- int state = (int) ifstack.Peek ();\r
- if ((state & TAKING) != 0)\r
- taking = true;\r
- }\r
-\r
- if (eval (arg) && taking){\r
- int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;\r
- if (region_directive)\r
- push |= REGION;\r
- ifstack.Push (push);\r
- return true;\r
- } else {\r
- int push = (taking ? PARENT_TAKING : 0);\r
- if (region_directive)\r
- push |= REGION;\r
- ifstack.Push (push);\r
- return false;\r
- }\r
- \r
- case "endif":\r
- if (ifstack == null || ifstack.Count == 0){\r
- Error_UnexpectedDirective ("no #if for this #endif");\r
- return true;\r
- } else {\r
- int pop = (int) ifstack.Pop ();\r
- \r
- if (region_directive && ((pop & REGION) == 0))\r
- Report.Error (1027, Location, "#endif directive expected");\r
- else if (!region_directive && ((pop & REGION) != 0))\r
- Report.Error (1038, Location, "#endregion directive expected");\r
- \r
- if (ifstack.Count == 0)\r
- return true;\r
- else {\r
- int state = (int) ifstack.Peek ();\r
-\r
- if ((state & TAKING) != 0)\r
- return true;\r
- else\r
- return false;\r
- }\r
- }\r
-\r
- case "elif":\r
- if (ifstack == null || ifstack.Count == 0){\r
- Error_UnexpectedDirective ("no #if for this #elif");\r
- return true;\r
- } else {\r
- int state = (int) ifstack.Peek ();\r
-\r
- if ((state & REGION) != 0) {\r
- Report.Error (1038, Location, "#endregion directive expected");\r
- return true;\r
- }\r
-\r
- if ((state & ELSE_SEEN) != 0){\r
- Error_UnexpectedDirective ("#elif not valid after #else");\r
- return true;\r
- }\r
-\r
- if ((state & (TAKEN_BEFORE | TAKING)) != 0)\r
- return false;\r
-\r
- if (eval (arg) && ((state & PARENT_TAKING) != 0)){\r
- state = (int) ifstack.Pop ();\r
- ifstack.Push (state | TAKING | TAKEN_BEFORE);\r
- return true;\r
- } else \r
- return false;\r
- }\r
-\r
- case "else":\r
- if (ifstack == null || ifstack.Count == 0){\r
- Report.Error (\r
- 1028, Location,\r
- "Unexpected processor directive (no #if for this #else)");\r
- return true;\r
- } else {\r
- int state = (int) ifstack.Peek ();\r
-\r
- if ((state & REGION) != 0) {\r
- Report.Error (1038, Location, "#endregion directive expected");\r
- return true;\r
- }\r
-\r
- if ((state & ELSE_SEEN) != 0){\r
- Error_UnexpectedDirective ("#else within #else");\r
- return true;\r
- }\r
-\r
- ifstack.Pop ();\r
-\r
- bool ret;\r
- if ((state & TAKEN_BEFORE) == 0){\r
- ret = ((state & PARENT_TAKING) != 0);\r
- } else\r
- ret = false;\r
- \r
- if (ret)\r
- state |= TAKING;\r
- else\r
- state &= ~TAKING;\r
- \r
- ifstack.Push (state | ELSE_SEEN);\r
- \r
- return ret;\r
- }\r
- }\r
-\r
- //\r
- // These are only processed if we are in a `taking' block\r
- //\r
- if (!caller_is_taking)\r
- return false;\r
- \r
- switch (cmd){\r
- case "define":\r
- /* if (any_token_seen){\r
- Error_TokensSeen ();\r
- return true;\r
- } */\r
- PreProcessDefinition (true, arg);\r
- return true;\r
-\r
- case "undef":\r
- /* if (any_token_seen){\r
- Error_TokensSeen ();\r
- return true;\r
- } */\r
- PreProcessDefinition (false, arg);\r
- return true;\r
-\r
- case "error":\r
- Report.Error (1029, Location, "#error: '" + arg + "'");\r
- return true;\r
-\r
- case "warning":\r
- Report.Warning (1030, Location, "#warning: '" + arg + "'");\r
- return true;\r
- }\r
-\r
- Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");\r
- return true;\r
-\r
- }\r
+ if(t == Token.EOF)
+ throw new ApplicationException("Unexpected EOF while looking for a pre-processor directive");
+
+ if(t == Token.HASH) {
+ tokens_seen = false;
+ putback('#');
+ }
+ }
}
}
projects / mono.git / blobdiff