public class Tokenizer : yyParser.yyInput
{
TextReader reader;
- // TODO: public SourceFile file_name;
- public string file_name;
- public string ref_name;
- public int ref_line = 1;
- public int line = 1;
- public int col = 1;
- public int current_token = Token.EOL;
+ string file_name;
+ string ref_name;
+ int ref_line = 0;
+ int line = 0;
+ int col = 1;
+ public int current_token = Token.ERROR;
bool handle_get_set = false;
bool cant_have_a_type_character = false;
static NumberStyles styles;
static NumberFormatInfo csharp_format_info;
- //
- // Pre-processor
- //
- Hashtable defines;
-
- const int TAKING = 1;
- const int TAKEN_BEFORE = 2;
- const int ELSE_SEEN = 4;
- const int PARENT_TAKING = 8;
- const int REGION = 16;
-
- //
- // pre-processor if stack state:
- //
- Stack ifstack;
-
//
// Values for the associated token returned
//
StringBuilder number;
- int putback_char;
+ int putback_char = -1;
Object val;
//
return error_details;
}
}
+
+ public string Source {
+ get {
+ return file_name;
+ }
+
+ set {
+ file_name = value;
+ ref_name = value;
+ Location.SetCurrentSource(file_name);
+ }
+ }
+
+ public string EffectiveSource {
+ get {
+ return ref_name;
+ }
+ set {
+ ref_name = value;
+ Location.SetCurrentSource(ref_name);
+ }
+ }
+
public int Line {
get {
return line;
}
}
+ public int EffectiveLine {
+ get {
+ return ref_line;
+ }
+ set {
+ ref_line = value;
+ }
+ }
+
public int Col {
get {
return col;
keywords.Add ("as", Token.AS);
keywords.Add ("assembly", Token.ASSEMBLY);
keywords.Add ("auto", Token.AUTO);
- keywords.Add ("binary", Token.BINARY);
+ keywords.Add ("binary", Token.BINARY); // Not a VB.NET Keyword
keywords.Add ("boolean", Token.BOOLEAN);
keywords.Add ("byref", Token.BYREF);
keywords.Add ("byte", Token.BYTE);
keywords.Add ("class", Token.CLASS);
keywords.Add ("clng", Token.CLNG);
keywords.Add ("cobj", Token.COBJ);
- keywords.Add ("compare", Token.COMPARE);
+ keywords.Add ("compare", Token.COMPARE); // Not a VB.NET Keyword
keywords.Add ("const", Token.CONST);
keywords.Add ("cshort", Token.CSHORT);
keywords.Add ("csng", Token.CSNG);
keywords.Add ("default", Token.DEFAULT);
keywords.Add ("delegate", Token.DELEGATE);
keywords.Add ("dim", Token.DIM);
+ keywords.Add ("directcast", Token.DIRECTCAST);
keywords.Add ("do", Token.DO);
keywords.Add ("double", Token.DOUBLE);
keywords.Add ("each", Token.EACH);
keywords.Add ("else", Token.ELSE);
keywords.Add ("elseif", Token.ELSEIF);
keywords.Add ("end", Token.END);
+ keywords.Add ("endif", Token.ENDIF); // An unused VB.NET keyword
keywords.Add ("enum", Token.ENUM);
keywords.Add ("erase", Token.ERASE);
keywords.Add ("error", Token.ERROR);
keywords.Add ("event", Token.EVENT);
keywords.Add ("exit", Token.EXIT);
- keywords.Add ("explicit", Token.EXPLICIT);
+ keywords.Add ("explicit", Token.EXPLICIT); // Not a VB.NET keyword
keywords.Add ("false", Token.FALSE);
keywords.Add ("finally", Token.FINALLY);
keywords.Add ("for", Token.FOR);
keywords.Add ("friend", Token.FRIEND);
keywords.Add ("function", Token.FUNCTION);
keywords.Add ("get", Token.GET);
- //keywords.Add ("gettype", Token.GETTYPE);
+ keywords.Add ("gettype", Token.GETTYPE);
+ keywords.Add ("gosub", Token.GOSUB); // An unused VB.NET keyword
keywords.Add ("goto", Token.GOTO);
keywords.Add ("handles", Token.HANDLES);
keywords.Add ("if", Token.IF);
keywords.Add ("integer", Token.INTEGER);
keywords.Add ("interface", Token.INTERFACE);
keywords.Add ("is", Token.IS);
- keywords.Add ("let ", Token.LET );
+ keywords.Add ("let ", Token.LET ); // An unused VB.NET keyword
keywords.Add ("lib ", Token.LIB );
keywords.Add ("like ", Token.LIKE );
keywords.Add ("long", Token.LONG);
keywords.Add ("notinheritable", Token.NOTINHERITABLE);
keywords.Add ("notoverridable", Token.NOTOVERRIDABLE);
keywords.Add ("object", Token.OBJECT);
- keywords.Add ("off", Token.OFF);
+ keywords.Add ("off", Token.OFF); // Not a VB.NET Keyword
keywords.Add ("on", Token.ON);
keywords.Add ("option", Token.OPTION);
keywords.Add ("optional", Token.OPTIONAL);
keywords.Add ("shared", Token.SHARED);
keywords.Add ("short", Token.SHORT);
keywords.Add ("single", Token.SINGLE);
- keywords.Add ("sizeof", Token.SIZEOF);
+ keywords.Add ("sizeof", Token.SIZEOF); // Not a VB.NET Keyword
keywords.Add ("static", Token.STATIC);
keywords.Add ("step", Token.STEP);
keywords.Add ("stop", Token.STOP);
- keywords.Add ("strict", Token.STRICT);
+ keywords.Add ("strict", Token.STRICT); // Not a VB.NET Keyword
keywords.Add ("string", Token.STRING);
keywords.Add ("structure", Token.STRUCTURE);
keywords.Add ("sub", Token.SUB);
keywords.Add ("synclock", Token.SYNCLOCK);
- keywords.Add ("text", Token.TEXT);
+ keywords.Add ("text", Token.TEXT); // Not a VB.NET Keyword
keywords.Add ("then", Token.THEN);
keywords.Add ("throw", Token.THROW);
keywords.Add ("to", Token.TO);
keywords.Add ("typeof", Token.TYPEOF);
keywords.Add ("unicode", Token.UNICODE);
keywords.Add ("until", Token.UNTIL);
- keywords.Add ("variant", Token.VARIANT);
+ keywords.Add ("variant", Token.VARIANT); // An unused VB.NET keyword
+ keywords.Add ("wend", Token.WEND); // An unused VB.NET keyword
keywords.Add ("when", Token.WHEN);
keywords.Add ("while", Token.WHILE);
keywords.Add ("with", Token.WITH);
}
- //
- // Class initializer
- //
static Tokenizer ()
{
initTokens ();
public Tokenizer (System.IO.TextReader input, string fname, ArrayList defines)
{
- this.ref_name = fname;
+ this.Source = fname;
+
reader = input;
- putback_char = -1;
-
- Location.Push (fname);
+
+ // putback an EOL at the beginning of a stream. This is a convenience that
+ // allows pre-processor directives to be added to the beginning of a vb file.
+ putback('\n');
}
bool is_keyword (string name)
}
}
- void define (string def)
- {
- if (!RootContext.AllDefines.Contains(def)){
- RootContext.AllDefines [def] = true;
- }
- if (defines.Contains (def))
- return;
- defines [def] = true;
- }
-
public bool PropertyParsing {
get {
return handle_get_set;
return Token.ERROR;
return Token.PERCENT;
case '#':
- if (!tokens_seen)
+ if(tokens_seen)
{
- workout_preprocessing_directive();
- return Token.NONE;
+ if (cant_have_a_type_character)
+ return ExtractDateTimeLiteral();
+ else
+ return Token.NUMBER_SIGN;
+ }
+ else
+ {
+ tokens_seen = true;
+ return Token.HASH;
}
- if (cant_have_a_type_character)
- return ExtractDateTimeLiteral();
- return Token.NUMBER_SIGN;
case '&':
if (!cant_have_a_type_character)
return Token.LONGTYPECHAR;
t = handle_integer_literal_in_other_bases(d);
if (t == Token.NONE) {
- if (d == '=') {
- doread = true;
- t = Token.OP_CONCAT_ASSIGN;
- } else
- t = Token.OP_CONCAT;
+ t = Token.OP_CONCAT;
}
return t;
}
if (c == '+'){
if (d == '+')
t = Token.OP_INC;
- else if (d == '=')
- t = Token.OP_ADD_ASSIGN;
- else
+ else
return Token.PLUS;
doread = true;
return t;
}
if (c == '-'){
- if (d == '=')
- t = Token.OP_SUB_ASSIGN;
- else
- return Token.MINUS;
- doread = true;
- return t;
+ return Token.MINUS;
}
if (c == '='){
}
if (c == '*'){
- if (d == '='){
- doread = true;
- return Token.OP_MULT_ASSIGN;
- }
return Token.STAR;
}
if (c == '/'){
- if (d == '='){
- doread = true;
- return Token.OP_DIV_ASSIGN;
- }
return Token.DIV;
}
if (c == '\\'){
- if (d == '='){
- doread = true;
- return Token.OP_IDIV_ASSIGN;
- }
return Token.OP_IDIV;
}
if (c == '^'){
- if (d == '='){
- doread = true;
- return Token.OP_EXP_ASSIGN;
- }
return Token.OP_EXP;
}
private bool IsEOL(int currentChar)
{
- if (currentChar == 0x0D)
- {
+ bool retVal;
+
+ if (currentChar == 0x0D) {
if (peekChar() == 0x0A) // if it is a CR-LF pair consume LF also
getChar();
- return true;
+ retVal = true;
}
- return (currentChar == -1 || currentChar == 0x0A || currentChar == 0x2028 || currentChar == 0x2029);
+ else {
+ retVal = (currentChar == -1 || currentChar == 0x0A || currentChar == 0x2028 || currentChar == 0x2029);
+ }
+
+ if(retVal) {
+ nextLine();
+ }
+
+ return retVal;
}
private int DropComments()
{
- int d;
- while (!IsEOL(d = getChar ()))
+ //int d;
+ while (!IsEOL(/*d =*/ getChar ()))
col++;
- line++;
- ref_line++;
- col = 0;
return Token.EOL;
}
cant_have_a_type_character = false;
- return id.ToString ();
+ return id.ToString();
}
private bool is_doublequote(int currentChar)
// Handle EOL.
if (IsEOL(c))
{
- nextLine();
if (current_token == Token.EOL) // if last token was also EOL keep skipping
continue;
return Token.EOL;
return Token.LITERAL_DATE;
}
if (IsEOL(c)) {
- nextLine();
break;
}
if (c == '-')
}
}
- if (IsEOL(c))
- {
- nextLine();
+ if (IsEOL(c)) {
return Token.ERROR;
}
return Token.ERROR;
}
- private void workout_preprocessing_directive()
- {
- int c;
- bool cont = true;
-
- start_again:
-
- cont = handle_preprocessing_directive (cont);
-
- if (cont) {
- col = 0;
- return;
- }
- col = 1;
-
- bool skipping = false;
- for (;(c = getChar ()) != -1; col++) {
- if (is_whitespace(c))
- continue;
- if (IsEOL(c)) {
- col = 0;
- line++;
- ref_line++;
- skipping = false;
- continue;
- }
- if (c != '#') {
- skipping = true;
- continue;
- }
- if (c == '#' && !skipping)
- goto start_again;
- }
- tokens_seen = false;
- if (c == -1)
- Report.Error (1027, Location, "#endif/#endregion expected");
- }
-
static IFormatProvider enUSculture = new CultureInfo("en-US", true);
private DateTime ParseDateLiteral(StringBuilder value)
return new DateTime();
}
-
- public void cleanup ()
+ public void PositionCursorAtNextPreProcessorDirective()
{
-/* borrowed from mcs - have to work it to have preprocessing in mbas
-
- if (ifstack != null && ifstack.Count >= 1) {
- int state = (int) ifstack.Pop ();
- if ((state & REGION) != 0)
- Report.Error (1038, "#endregion directive expected");
- else
- Report.Error (1027, "#endif directive expected");
- }
-*/
- }
-
- static StringBuilder static_cmd_arg = new StringBuilder ();
-
- void get_cmd_arg (out string cmd, out string arg)
- {
- int c;
-
- tokens_seen = false;
- cmd = "";
- arg = "";
-
- // skip over white space
- while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r') && ((c == ' ') || (c == '\t')))
- ;
-
- if (c == '\n'){
- line++;
- ref_line++;
- return;
- } else if (c == '\r')
- col = 0;
-
- static_cmd_arg.Length = 0;
- static_cmd_arg.Append ((char) c);
-
-
- while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
- static_cmd_arg.Append ((char) c);
- }
-
- cmd = static_cmd_arg.ToString().ToLower();
-
- if (c == '\n'){
- line++;
- ref_line++;
- return;
- } else if (c == '\r')
- col = 0;
-
- // skip over white space
- while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r') && ((c == ' ') || (c == '\t')))
- ;
-
- if (c == '\n'){
- line++;
- ref_line++;
- return;
- } else if (c == '\r'){
- col = 0;
- return;
- }
-
- static_cmd_arg.Length = 0;
- static_cmd_arg.Append ((char) c);
-
- while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
- static_cmd_arg.Append ((char) c);
- }
-
- if (c == '\n'){
- line++;
- ref_line++;
- } else if (c == '\r')
- col = 0;
- arg = static_cmd_arg.ToString ().Trim ();
-
- if (cmd == "end" && arg.ToLower() == "region") {
- cmd = "end region";
- arg = "";
- }
- if (cmd == "end" && arg.ToLower() == "if") {
- cmd = "end if";
- arg = "";
- }
-
- }
-
- //
- // Handles the #line directive
- //
- bool PreProcessLine (string arg)
- {
- if (arg == "")
- return false;
-
- if (arg == "default"){
- ref_line = line;
- ref_name = file_name;
- Location.Push (ref_name);
- return true;
- }
-
- try {
- int pos;
-
- if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
- ref_line = System.Int32.Parse (arg.Substring (0, pos));
- pos++;
-
- char [] quotes = { '\"' };
-
- string name = arg.Substring (pos). Trim (quotes);
- ref_name = name; // TODO: Synchronize with mcs: Location.LookupFile (name);
- Location.Push (ref_name);
- } else {
- ref_line = System.Int32.Parse (arg);
- }
- } catch {
- return false;
- }
-
- return true;
- }
-
- //
- // Handles #define and #undef
- //
- void PreProcessDefinition (bool is_define, string arg)
- {
- if (arg == "" || arg == "true" || arg == "false"){
- Report.Error (1001, Location, "Missing identifer to pre-processor directive");
- return;
- }
-
- char[] whitespace = { ' ', '\t' };
- if (arg.IndexOfAny (whitespace) != -1){
- Report.Error (1025, Location, "Single-line comment or end-of-line expected");
- return;
- }
-
- foreach (char c in arg){
- if (!Char.IsLetter (c) && (c != '_')){
- Report.Error (1001, Location, "Identifier expected");
- return;
- }
- }
-
- if (is_define){
- if (defines == null)
- defines = new Hashtable ();
- define (arg);
- } else {
- if (defines == null)
- return;
- if (defines.Contains (arg))
- defines.Remove (arg);
- }
- }
-
- bool eval_val (string s)
- {
- if (s == "true")
- return true;
- if (s == "false")
- return false;
+ int t;
- if (defines == null)
- return false;
- if (defines.Contains (s))
- return true;
-
- return false;
- }
-
- bool pp_primary (ref string s)
- {
- s = s.Trim ();
- int len = s.Length;
+ for(t = token(); t != Token.HASH && t != Token.EOF; t = token());
- if (len > 0){
- char c = s [0];
-
- if (c == '('){
- s = s.Substring (1);
- bool val = pp_expr (ref s);
- if (s.Length > 0 && s [0] == ')'){
- s = s.Substring (1);
- return val;
- }
- Error_InvalidDirective ();
- return false;
- }
-
- if (is_identifier_start_character (c)){
- int j = 1;
-
- while (j < len){
- c = s [j];
-
- if (is_identifier_part_character (c)){
- j++;
- continue;
- }
- bool v = eval_val (s.Substring (0, j));
- s = s.Substring (j);
- return v;
- }
- bool vv = eval_val (s);
- s = "";
- return vv;
- }
- }
- Error_InvalidDirective ();
- return false;
- }
-
- bool pp_unary (ref string s)
- {
- s = s.Trim ();
- int len = s.Length;
-
- if (len > 0){
- if (s [0] == '!'){
- if (len > 1 && s [1] == '='){
- Error_InvalidDirective ();
- return false;
- }
- s = s.Substring (1);
- return ! pp_primary (ref s);
- } else
- return pp_primary (ref s);
- } else {
- Error_InvalidDirective ();
- return false;
- }
- }
-
- bool pp_eq (ref string s)
- {
- bool va = pp_unary (ref s);
-
- s = s.Trim ();
- int len = s.Length;
- if (len > 0){
- if (s [0] == '='){
- if (len > 2 && s [1] == '='){
- s = s.Substring (2);
- return va == pp_unary (ref s);
- } else {
- Error_InvalidDirective ();
- return false;
- }
- } else if (s [0] == '!' && len > 1 && s [1] == '='){
- s = s.Substring (2);
-
- return va != pp_unary (ref s);
-
- }
- }
-
- return va;
-
- }
-
- bool pp_and (ref string s)
- {
- bool va = pp_eq (ref s);
-
- s = s.Trim ();
- int len = s.Length;
- if (len > 0){
- if (s [0] == '&'){
- if (len > 2 && s [1] == '&'){
- s = s.Substring (2);
- return (va & pp_eq (ref s));
- } else {
- Error_InvalidDirective ();
- return false;
- }
- }
- }
- return va;
- }
-
- //
- // Evaluates an expression for `#if' or `#elif'
- //
- bool pp_expr (ref string s)
- {
- bool va = pp_and (ref s);
- s = s.Trim ();
- int len = s.Length;
- if (len > 0){
- char c = s [0];
-
- if (c == '|'){
- if (len > 2 && s [1] == '|'){
- s = s.Substring (2);
- return va | pp_expr (ref s);
- } else {
- Error_InvalidDirective ();
- return false;
- }
- }
- }
+ if(t == Token.EOF)
+ throw new ApplicationException("Unexpected EOF while looking for a pre-processor directive");
- return va;
- }
-
- bool eval (string s)
- {
- bool v = pp_expr (ref s);
- s = s.Trim ();
- if (s.Length != 0){
- Error_InvalidDirective ();
- return false;
+ if(t == Token.HASH) {
+ tokens_seen = false;
+ putback('#');
}
-
- return v;
- }
-
- void Error_InvalidDirective ()
- {
- Report.Error (1517, Location, "Invalid pre-processor directive");
- }
-
- void Error_UnexpectedDirective (string extra)
- {
- Report.Error (
- 1028, Location,
- "Unexpected processor directive (" + extra + ")");
- }
-
- void Error_TokensSeen ()
- {
- Report.Error (
- 1032, Location,
- "Cannot define or undefine pre-processor symbols after a token in the file");
- }
-
- //
- // if true, then the code continues processing the code
- // if false, the code stays in a loop until another directive is
- // reached.
- //
- bool handle_preprocessing_directive (bool caller_is_taking)
- {
- //char [] blank = { ' ', '\t' };
- string cmd, arg;
- bool region_directive = false;
-
- get_cmd_arg (out cmd, out arg);
- // Eat any trailing whitespaces and single-line comments
- if (arg.IndexOf ("'") != -1)
- arg = arg.Substring (0, arg.IndexOf ("//"));
- arg = arg.TrimEnd (' ', '\t');
-
- //
- // The first group of pre-processing instructions is always processed
- //
- switch (cmd.ToLower()){
- case "line":
- if (!PreProcessLine (arg))
- Report.Error (
- 1576, Location,
- "Argument to #line directive is missing or invalid");
- return true;
-
- case "region":
- region_directive = true;
- arg = "true";
- goto case "if";
-
- case "end region":
- region_directive = true;
- goto case "end if";
-
- case "if":
- if (arg == ""){
- Error_InvalidDirective ();
- return true;
- }
- bool taking = false;
- if (ifstack == null)
- ifstack = new Stack ();
-
- if (ifstack.Count == 0){
- taking = true;
- } else {
- int state = (int) ifstack.Peek ();
- if ((state & TAKING) != 0)
- taking = true;
- }
-
- if (eval (arg) && taking){
- int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;
- if (region_directive)
- push |= REGION;
- ifstack.Push (push);
- return true;
- } else {
- int push = (taking ? PARENT_TAKING : 0);
- if (region_directive)
- push |= REGION;
- ifstack.Push (push);
- return false;
- }
-
- case "end if":
- if (ifstack == null || ifstack.Count == 0){
- Error_UnexpectedDirective ("no #if for this #end if");
- return true;
- } else {
- int pop = (int) ifstack.Pop ();
-
- if (region_directive && ((pop & REGION) == 0))
- Report.Error (1027, Location, "#end if directive expected");
- else if (!region_directive && ((pop & REGION) != 0))
- Report.Error (1038, Location, "#end region directive expected");
-
- if (ifstack.Count == 0)
- return true;
- else {
- int state = (int) ifstack.Peek ();
-
- if ((state & TAKING) != 0)
- return true;
- else
- return false;
- }
- }
-
- case "elseif":
- if (ifstack == null || ifstack.Count == 0){
- Error_UnexpectedDirective ("no #if for this #elif");
- return true;
- } else {
- int state = (int) ifstack.Peek ();
-
- if ((state & REGION) != 0) {
- Report.Error (1038, Location, "#end region directive expected");
- return true;
- }
-
- if ((state & ELSE_SEEN) != 0){
- Error_UnexpectedDirective ("#elif not valid after #else");
- return true;
- }
-
- if ((state & (TAKEN_BEFORE | TAKING)) != 0)
- return false;
-
- if (eval (arg) && ((state & PARENT_TAKING) != 0)){
- state = (int) ifstack.Pop ();
- ifstack.Push (state | TAKING | TAKEN_BEFORE);
- return true;
- } else
- return false;
- }
-
- case "else":
- if (ifstack == null || ifstack.Count == 0){
- Report.Error (
- 1028, Location,
- "Unexpected processor directive (no #if for this #else)");
- return true;
- } else {
- int state = (int) ifstack.Peek ();
-
- if ((state & REGION) != 0) {
- Report.Error (1038, Location, "#end region directive expected");
- return true;
- }
-
- if ((state & ELSE_SEEN) != 0){
- Error_UnexpectedDirective ("#else within #else");
- return true;
- }
-
- ifstack.Pop ();
-
- bool ret;
- if ((state & TAKEN_BEFORE) == 0){
- ret = ((state & PARENT_TAKING) != 0);
- } else
- ret = false;
-
- if (ret)
- state |= TAKING;
- else
- state &= ~TAKING;
-
- ifstack.Push (state | ELSE_SEEN);
-
- return ret;
- }
- }
-
- //
- // These are only processed if we are in a `taking' block
- //
- if (!caller_is_taking)
- return false;
-
- switch (cmd.ToLower()){
- case "define":
- /* if (any_token_seen){
- Error_TokensSeen ();
- return true;
- } */
- PreProcessDefinition (true, arg);
- return true;
-
- case "undef":
- /* if (any_token_seen){
- Error_TokensSeen ();
- return true;
- } */
- PreProcessDefinition (false, arg);
- return true;
-
- case "error":
- Report.Error (1029, Location, "#error: '" + arg + "'");
- return true;
-
- case "warning":
- Report.Warning (1030, Location, "#warning: '" + arg + "'");
- return true;
- }
-
- Report.Error (1024, Location, "Preprocessor directive expected (got: '" + cmd + "')");
- return true;
}
}
projects / mono.git / blobdiff