-//\r
-// Mono.MonoBASIC.Tokenizer.cs: The Tokenizer for the MonoBASIC compiler\r
-//\r
-// Author: A Rafael D Teixeira (rafaelteixeirabr@hotmail.com)\r
-// \r
-// Based on cs-tokenizer.cs by Miguel de Icaza (miguel@gnu.org)\r
-//\r
-// Licensed under the terms of the GNU GPL\r
-//\r
-// Copyright (C) 2001 A Rafael D Teixeira\r
-//\r
-\r
-namespace Mono.MonoBASIC\r
-{\r
- using System;\r
- using System.Text;\r
- using System.Collections;\r
- using System.IO;\r
- using System.Globalization;\r
- using Mono.Languages;\r
- using Mono.CSharp;\r
- \r
- /// <summary>\r
- /// Tokenizer for MonoBASIC source code. \r
- /// </summary>\r
- \r
- public class Tokenizer : yyParser.yyInput\r
- {\r
- TextReader reader;\r
- public string ref_name;\r
- public int ref_line = 1;\r
- public int line = 1;\r
- public int col = 1;\r
- public int current_token;\r
- bool handle_get_set = false;\r
-\r
- public string location {\r
- get {\r
- string det;\r
-\r
- if (current_token == Token.ERROR)\r
- det = "detail: " + error_details;\r
- else\r
- det = "";\r
- \r
- return "Line: "+line+" Col: "+col + "\n" +\r
- "VirtLine: "+ref_line +\r
- " Token: "+current_token + " " + det;\r
- }\r
- }\r
-\r
- public bool properties {\r
- get {\r
- return handle_get_set;\r
- }\r
-\r
- set {\r
- handle_get_set = value;\r
- }\r
- }\r
- \r
- //\r
- // Class variables\r
- // \r
- static Hashtable keywords;\r
- static NumberStyles styles;\r
- static NumberFormatInfo csharp_format_info;\r
- \r
- //\r
- // Values for the associated token returned\r
- //\r
- System.Text.StringBuilder number;\r
- int putback_char;\r
- Object val;\r
- \r
- //\r
- // Details about the error encoutered by the tokenizer\r
- //\r
- string error_details;\r
- \r
- public string error {\r
- get {\r
- return error_details;\r
- }\r
- }\r
- \r
- public int Line {\r
- get {\r
- return line;\r
- }\r
- }\r
-\r
- public int Col {\r
- get {\r
- return col;\r
- }\r
- }\r
- \r
- static void initTokens ()\r
- {\r
- keywords = new Hashtable ();\r
-\r
- keywords.Add ("addhandler", Token.ADDHANDLER);\r
- keywords.Add ("addressof", Token.ADDRESSOF);\r
- keywords.Add ("alias", Token.ALIAS);\r
- keywords.Add ("and", Token.AND);\r
- keywords.Add ("andalso", Token.ANDALSO);\r
- keywords.Add ("ansi", Token.ANSI);\r
- keywords.Add ("as", Token.AS);\r
- keywords.Add ("assembly", Token.ASSEMBLY);\r
- keywords.Add ("auto", Token.AUTO);\r
- keywords.Add ("boolean", Token.BOOLEAN);\r
- keywords.Add ("byref", Token.BYREF);\r
- keywords.Add ("byte", Token.BYTE);\r
- keywords.Add ("byval", Token.BYVAL);\r
- keywords.Add ("call", Token.CALL);\r
- keywords.Add ("case", Token.CASE);\r
- keywords.Add ("catch", Token.CATCH);\r
- keywords.Add ("cbool", Token.CBOOL);\r
- keywords.Add ("cbyte", Token.CBYTE);\r
- keywords.Add ("cchar", Token.CCHAR);\r
- keywords.Add ("cdate", Token.CDATE);\r
- keywords.Add ("cdec", Token.CDEC);\r
- keywords.Add ("cdbl", Token.CDBL);\r
- keywords.Add ("char", Token.CHAR);\r
- keywords.Add ("cint", Token.CINT);\r
- keywords.Add ("class", Token.CLASS);\r
- keywords.Add ("clng", Token.CLNG);\r
- keywords.Add ("cobj", Token.COBJ);\r
- //keywords.Add ("compare", Token.COMPARE);\r
- keywords.Add ("const", Token.CONST);\r
- keywords.Add ("cshort", Token.CSHORT);\r
- keywords.Add ("csng", Token.CSNG);\r
- keywords.Add ("cstr", Token.CSTR);\r
- keywords.Add ("ctype", Token.CTYPE);\r
- keywords.Add ("date", Token.DATE);\r
- keywords.Add ("decimal", Token.DECIMAL);\r
- keywords.Add ("declare", Token.DECLARE);\r
- keywords.Add ("default", Token.DEFAULT);\r
- keywords.Add ("delegate", Token.DELEGATE);\r
- keywords.Add ("dim", Token.DIM);\r
- keywords.Add ("do", Token.DO);\r
- keywords.Add ("double", Token.DOUBLE);\r
- keywords.Add ("each", Token.EACH);\r
- keywords.Add ("else", Token.ELSE);\r
- keywords.Add ("elseif", Token.ELSEIF);\r
- keywords.Add ("end", Token.END);\r
- keywords.Add ("enum", Token.ENUM);\r
- keywords.Add ("erase", Token.ERASE);\r
- keywords.Add ("error", Token.ERROR);\r
- keywords.Add ("event", Token.EVENT);\r
- keywords.Add ("exit", Token.EXIT);\r
- //keywords.Add ("explicit", Token.EXPLICIT);\r
- keywords.Add ("false", Token.FALSE);\r
- keywords.Add ("finally", Token.FINALLY);\r
- keywords.Add ("for", Token.FOR);\r
- keywords.Add ("friend", Token.FRIEND);\r
- keywords.Add ("function", Token.FUNCTION);\r
- keywords.Add ("get", Token.GET);\r
- keywords.Add ("gettype", Token.GETTYPE);\r
- keywords.Add ("goto", Token.GOTO);\r
- keywords.Add ("handles", Token.HANDLES);\r
- keywords.Add ("if", Token.IF);\r
- keywords.Add ("implements", Token.IMPLEMENTS);\r
- keywords.Add ("imports", Token.IMPORTS);\r
- keywords.Add ("in", Token.IN);\r
- keywords.Add ("inherits", Token.INHERITS);\r
- keywords.Add ("integer", Token.INTEGER);\r
- keywords.Add ("interface", Token.INTERFACE);\r
- keywords.Add ("is", Token.IS);\r
- keywords.Add ("let ", Token.LET );\r
- keywords.Add ("lib ", Token.LIB );\r
- keywords.Add ("like ", Token.LIKE );\r
- keywords.Add ("long", Token.LONG);\r
- keywords.Add ("loop", Token.LOOP);\r
- keywords.Add ("me", Token.ME);\r
- keywords.Add ("mod", Token.MOD);\r
- keywords.Add ("module", Token.MODULE);\r
- keywords.Add ("mustinherit", Token.MUSTINHERIT);\r
- keywords.Add ("mustoverride", Token.MUSTOVERRIDE);\r
- keywords.Add ("mybase", Token.MYBASE);\r
- keywords.Add ("myclass", Token.MYCLASS);\r
- keywords.Add ("namespace", Token.NAMESPACE);\r
- keywords.Add ("new", Token.NEW);\r
- keywords.Add ("next", Token.NEXT);\r
- keywords.Add ("not", Token.NOT);\r
- keywords.Add ("nothing", Token.NOTHING);\r
- keywords.Add ("notinheritable", Token.NOTINHERITABLE);\r
- keywords.Add ("notoverridable", Token.NOTOVERRIDABLE);\r
- keywords.Add ("object", Token.OBJECT);\r
- keywords.Add ("on", Token.ON);\r
- keywords.Add ("option", Token.OPTION);\r
- keywords.Add ("optional", Token.OPTIONAL);\r
- keywords.Add ("or", Token.OR);\r
- keywords.Add ("orelse", Token.ORELSE);\r
- keywords.Add ("overloads", Token.OVERLOADS);\r
- keywords.Add ("overridable", Token.OVERRIDABLE);\r
- keywords.Add ("overrides", Token.OVERRIDES);\r
- keywords.Add ("paramarray", Token.PARAM_ARRAY);\r
- keywords.Add ("preserve", Token.PRESERVE);\r
- keywords.Add ("private", Token.PRIVATE);\r
- keywords.Add ("property", Token.PROPERTY);\r
- keywords.Add ("protected", Token.PROTECTED);\r
- keywords.Add ("public", Token.PUBLIC);\r
- keywords.Add ("raiseevent", Token.RAISEEVENT);\r
- keywords.Add ("readonly", Token.READONLY);\r
- keywords.Add ("redim", Token.REDIM);\r
- keywords.Add ("rem", Token.REM);\r
- keywords.Add ("removehandler", Token.REMOVEHANDLER);\r
- keywords.Add ("resume", Token.RESUME);\r
- keywords.Add ("return", Token.RETURN);\r
- keywords.Add ("select", Token.SELECT);\r
- keywords.Add ("set", Token.SET);\r
- keywords.Add ("shadows", Token.SHADOWS);\r
- keywords.Add ("shared", Token.SHARED);\r
- keywords.Add ("short", Token.SHORT);\r
- keywords.Add ("single", Token.SINGLE);\r
- keywords.Add ("sizeof", Token.SIZEOF);\r
- keywords.Add ("static", Token.STATIC);\r
- keywords.Add ("step", Token.STEP);\r
- keywords.Add ("stop", Token.STOP);\r
- keywords.Add ("string", Token.STRING);\r
- keywords.Add ("structure", Token.STRUCTURE);\r
- keywords.Add ("sub", Token.SUB);\r
- keywords.Add ("synclock", Token.SYNCLOCK);\r
- keywords.Add ("then", Token.THEN);\r
- keywords.Add ("throw", Token.THROW);\r
- keywords.Add ("to", Token.TO);\r
- keywords.Add ("true", Token.TRUE);\r
- keywords.Add ("try", Token.TRY);\r
- keywords.Add ("typeof", Token.TYPEOF);\r
- keywords.Add ("unicode", Token.UNICODE);\r
- keywords.Add ("until", Token.UNTIL);\r
- keywords.Add ("variant", Token.VARIANT);\r
- keywords.Add ("when", Token.WHEN);\r
- keywords.Add ("while", Token.WHILE);\r
- keywords.Add ("with", Token.WITH);\r
- keywords.Add ("withevents", Token.WITHEVENTS);\r
- keywords.Add ("writeonly", Token.WRITEONLY);\r
- keywords.Add ("xor", Token.XOR);\r
- }\r
-\r
- //\r
- // Class initializer\r
- // \r
- static Tokenizer ()\r
- {\r
- initTokens ();\r
- csharp_format_info = new NumberFormatInfo ();\r
- csharp_format_info.CurrencyDecimalSeparator = ".";\r
- styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint;\r
- }\r
-\r
- bool is_keyword (string name)\r
- {\r
- bool res;\r
-\r
- res = keywords.Contains(name.ToLower());\r
- if ((name == "get" || name == "set") && handle_get_set == false)\r
- return false;\r
- return res;\r
- }\r
-\r
- int getKeyword (string name)\r
- {\r
- return (int) (keywords [name.ToLower()]);\r
- }\r
- \r
- public Location Location {\r
- get {\r
- return new Location (ref_line);\r
- }\r
- }\r
- \r
- bool is_identifier_start_character (char c)\r
- {\r
- return Char.IsLetter (c) || c == '_' ;\r
- }\r
-\r
- bool is_identifier_part_character (char c)\r
- {\r
- return (Char.IsLetter (c) || Char.IsDigit (c) || c == '_');\r
- }\r
-\r
- int is_punct (char c, ref bool doread)\r
- {\r
- int idx = "{}[](),:;~+-*/%&|^!=<>?".IndexOf (c);\r
- int d;\r
- int t;\r
-\r
- doread = false;\r
-\r
- switch (c){\r
- case '[':\r
- return Token.OPEN_BRACKET;\r
- case ']':\r
- return Token.CLOSE_BRACKET;\r
- case '(':\r
- return Token.OPEN_PARENS;\r
- case ')':\r
- return Token.CLOSE_PARENS;\r
- case ',':\r
- return Token.COMMA;\r
- case ':':\r
- return Token.COLON;\r
- case '?':\r
- return Token.INTERR;\r
- }\r
-\r
- d = peekChar ();\r
- if (c == '+'){\r
- \r
- if (d == '+')\r
- t = Token.OP_INC;\r
- else if (d == '=')\r
- t = Token.OP_ADD_ASSIGN;\r
- else\r
- return Token.PLUS;\r
- doread = true;\r
- return t;\r
- }\r
- if (c == '-'){\r
- if (d == '=')\r
- t = Token.OP_SUB_ASSIGN;\r
- else\r
- return Token.MINUS;\r
- doread = true;\r
- return t;\r
- }\r
-\r
- if (c == '='){\r
- /*if (d == '='){\r
- doread = true;\r
- return Token.OP_EQ;\r
- }*/\r
- return Token.ASSIGN;\r
- }\r
-\r
- if (c == '*'){\r
- if (d == '='){\r
- doread = true;\r
- return Token.OP_MULT_ASSIGN;\r
- }\r
- return Token.STAR;\r
- }\r
-\r
- if (c == '/'){\r
- if (d == '='){\r
- doread = true;\r
- return Token.OP_DIV_ASSIGN;\r
- }\r
- return Token.DIV;\r
- }\r
-\r
- if (c == '\\'){\r
- if (d == '='){\r
- doread = true;\r
- return Token.OP_IDIV_ASSIGN;\r
- }\r
- return Token.OP_IDIV;\r
- }\r
-\r
- if (c == '^'){\r
- if (d == '='){\r
- doread = true;\r
- return Token.OP_EXP_ASSIGN;\r
- }\r
- return Token.OP_EXP;\r
- }\r
-\r
- if (c == '<'){\r
- if (d == '>')\r
- {\r
- doread = true;\r
- return Token.OP_NE;\r
- }\r
- if (d == '='){\r
- doread = true;\r
- return Token.OP_LE;\r
- }\r
- return Token.OP_LT;\r
- }\r
-\r
- if (c == '>'){\r
- if (d == '='){\r
- doread = true;\r
- return Token.OP_GE;\r
- }\r
- return Token.OP_GT;\r
- }\r
- return Token.ERROR;\r
- }\r
-\r
- bool decimal_digits (int c)\r
- {\r
- int d;\r
- bool seen_digits = false;\r
- \r
- if (c != -1)\r
- number.Append ((char) c);\r
- \r
- while ((d = peekChar ()) != -1){\r
- if (Char.IsDigit ((char)d)){\r
- number.Append ((char) d);\r
- getChar ();\r
- seen_digits = true;\r
- } else\r
- break;\r
- }\r
- return seen_digits;\r
- }\r
-\r
- void hex_digits (int c)\r
- {\r
- int d;\r
-\r
- if (c != -1)\r
- number.Append ((char) c);\r
- while ((d = peekChar ()) != -1){\r
- char e = Char.ToUpper ((char) d);\r
- \r
- if (Char.IsDigit (e) ||\r
- (e >= 'A' && e <= 'F')){\r
- number.Append ((char) e);\r
- getChar ();\r
- } else\r
- break;\r
- }\r
- }\r
- \r
- int real_type_suffix (int c)\r
- {\r
- int t;\r
- \r
- switch (c){\r
- case 'F': case 'f':\r
- t = Token.LITERAL_SINGLE;\r
- break;\r
- case 'D': case 'd':\r
- t = Token.LITERAL_DOUBLE;\r
- break;\r
- case 'M': case 'm':\r
- t= Token.LITERAL_DECIMAL;\r
- break;\r
- default:\r
- return Token.NONE;\r
- }\r
- getChar ();\r
- return t;\r
- }\r
-\r
- int integer_type_suffix (int c)\r
- {\r
- // FIXME: Handle U and L suffixes.\r
- // We also need to see in which kind of\r
- // Int the thing fits better according to the spec.\r
- return Token.LITERAL_INTEGER;\r
- }\r
- \r
- void adjust_int (int t)\r
- {\r
- val = new System.Int32();\r
- val = System.Int32.Parse (number.ToString (), 0);\r
- }\r
-\r
- int adjust_real (int t)\r
- {\r
- string s = number.ToString ();\r
-\r
- Console.WriteLine (s);\r
- switch (t){\r
- case Token.LITERAL_DECIMAL:\r
- val = new System.Decimal ();\r
- val = System.Decimal.Parse (\r
- s, styles, csharp_format_info);\r
- break;\r
- case Token.LITERAL_DOUBLE:\r
- val = new System.Double ();\r
- val = System.Double.Parse (\r
- s, styles, csharp_format_info);\r
- break;\r
- case Token.LITERAL_SINGLE:\r
- val = new System.Double ();\r
- val = (float) System.Double.Parse (\r
- s, styles, csharp_format_info);\r
- break;\r
-\r
- case Token.NONE:\r
- val = new System.Double ();\r
- val = System.Double.Parse (\r
- s, styles, csharp_format_info);\r
- t = Token.LITERAL_DOUBLE;\r
- break;\r
- }\r
- return t;\r
- }\r
-\r
- //\r
- // Invoked if we know we have .digits or digits\r
- //\r
- int is_number (int c)\r
- {\r
- bool is_real = false;\r
- number = new System.Text.StringBuilder ();\r
- int type;\r
-\r
- number.Length = 0;\r
-\r
- if (Char.IsDigit ((char)c)){\r
- if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){\r
- getChar ();\r
- hex_digits (-1);\r
- val = new System.Int32 ();\r
- val = System.Int32.Parse (number.ToString (), NumberStyles.HexNumber);\r
- return integer_type_suffix (peekChar ());\r
- }\r
- decimal_digits (c);\r
- c = getChar ();\r
- }\r
-\r
- //\r
- // We need to handle the case of\r
- // "1.1" vs "1.string" (LITERAL_SINGLE vs NUMBER DOT IDENTIFIER)\r
- //\r
- if (c == '.'){\r
- if (decimal_digits ('.')){\r
- is_real = true;\r
- c = peekChar ();\r
- } else {\r
- putback ('.');\r
- number.Length -= 1;\r
- adjust_int (Token.LITERAL_INTEGER);\r
- return Token.LITERAL_INTEGER;\r
- }\r
- }\r
- \r
- if (c == 'e' || c == 'E'){\r
- is_real = true;\r
- number.Append ("e");\r
- getChar ();\r
- \r
- c = peekChar ();\r
- if (c == '+'){\r
- number.Append ((char) c);\r
- getChar ();\r
- c = peekChar ();\r
- } else if (c == '-'){\r
- number.Append ((char) c);\r
- getChar ();\r
- c = peekChar ();\r
- }\r
- decimal_digits (-1);\r
- c = peekChar ();\r
- }\r
-\r
- type = real_type_suffix (c);\r
- if (type == Token.NONE && !is_real){\r
- type = integer_type_suffix (c);\r
- adjust_int (type);\r
- putback (c);\r
- return type;\r
- } else\r
- is_real = true;\r
-\r
- if (is_real)\r
- return adjust_real (type);\r
-\r
- Console.WriteLine ("This should not be reached");\r
- throw new Exception ("Is Number should never reach this point");\r
- }\r
- \r
- int escape (int c)\r
- {\r
- int d;\r
- int v;\r
-\r
- d = peekChar ();\r
- if (c != '\\')\r
- return c;\r
- \r
- switch (d){\r
- case 'a':\r
- v = '\a'; break;\r
- case 'b':\r
- v = '\b'; break;\r
- case 'n':\r
- v = '\n'; break;\r
- case 't':\r
- v = '\t'; break;\r
- case 'v':\r
- v = '\v'; break;\r
- case 'r':\r
- v = 'c'; break;\r
- case '\\':\r
- v = '\\'; break;\r
- case 'f':\r
- v = '\f'; break;\r
- case '0':\r
- v = 0; break;\r
- case '"':\r
- v = '"'; break;\r
- case '\'':\r
- v = '\''; break;\r
- default:\r
- error_details = "cs1009: Unrecognized escape sequence " + (char)d;\r
- return -1;\r
- }\r
- getChar ();\r
- return v;\r
- }\r
-\r
- int getChar ()\r
- {\r
- if (putback_char != -1){\r
- int x = putback_char;\r
- putback_char = -1;\r
-\r
- return x;\r
- }\r
- return reader.Read ();\r
- }\r
-\r
- int peekChar ()\r
- {\r
- if (putback_char != -1)\r
- return putback_char;\r
- return reader.Peek ();\r
- }\r
-\r
- void putback (int c)\r
- {\r
- if (putback_char != -1)\r
- throw new Exception ("This should not happen putback on putback");\r
- putback_char = c;\r
- }\r
-\r
- public bool advance ()\r
- {\r
- return current_token != Token.EOF ;\r
- }\r
-\r
- public Object Value {\r
- get {\r
- return val;\r
- }\r
- }\r
-\r
- public Object value ()\r
- {\r
- return val;\r
- }\r
- \r
- public int token ()\r
- {\r
- current_token = xtoken ();\r
- if (current_token == 0) \r
- return Token.EOF;\r
- return current_token;\r
- }\r
- \r
- public int xtoken ()\r
- {\r
- int t;\r
- bool allow_keyword_as_ident = false;\r
- bool doread = false;\r
- int c;\r
-\r
- val = null;\r
- for (;(c = getChar ()) != -1; col++) {\r
- \r
- // Handle line comments.\r
- if (c == '\''){\r
- int d = getChar ();\r
- while ((d = getChar ()) != -1 && (d != '\n'))\r
- col++;\r
- line++;\r
- ref_line++;\r
- col = 0;\r
- if (current_token == Token.EOL) // if last token was also EOL keep skipping\r
- continue;\r
- return Token.EOL;\r
- }\r
-\r
- // Handle EOL.\r
- if (c == '\n')\r
- {\r
- line++;\r
- ref_line++;\r
- col = 0;\r
- if (current_token == Token.EOL) // if last token was also EOL keep skipping\r
- continue;\r
- return Token.EOL;\r
- }\r
- \r
- // Handle identifiers\r
- if (is_identifier_start_character ((char) c)){\r
- System.Text.StringBuilder id = new System.Text.StringBuilder ();\r
- string ids;\r
- \r
- id.Append ((char) c);\r
- \r
- while ((c = peekChar ()) != -1) {\r
- if (is_identifier_part_character ((char) c)){\r
- id.Append ((char)getChar ());\r
- col++;\r
- } else \r
- break;\r
- }\r
- \r
- ids = id.ToString ();\r
-\r
- if (!is_keyword (ids) || allow_keyword_as_ident) {\r
- val = ids;\r
- return Token.IDENTIFIER;\r
- }\r
-\r
- // true, false and null are in the hash anyway.\r
- return getKeyword (ids);\r
-\r
- }\r
-\r
- if (c == '.'){\r
- if (Char.IsDigit ((char) peekChar ()))\r
- return is_number (c);\r
- return Token.DOT;\r
- }\r
- \r
- if (Char.IsDigit ((char) c))\r
- return is_number (c);\r
-\r
- /* For now, ignore pre-processor commands */\r
- if (col == 1 && c == '#'){\r
- System.Text.StringBuilder s = new System.Text.StringBuilder ();\r
- \r
- while ((c = getChar ()) != -1 && (c != '\n')){\r
- s.Append ((char) c);\r
- }\r
- if (String.Compare (s.ToString (), 0, "line", 0, 4) == 0){\r
- string arg = s.ToString ().Substring (5);\r
- int pos;\r
-\r
- if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){\r
- ref_line = System.Int32.Parse (arg.Substring (0, pos));\r
- pos++;\r
-\r
- char [] quotes = { '\"' };\r
-\r
- ref_name = arg.Substring (pos);\r
- ref_name.TrimStart (quotes);\r
- ref_name.TrimEnd (quotes);\r
- } else\r
- ref_line = System.Int32.Parse (arg);\r
- }\r
- line++;\r
- ref_line++;\r
- continue;\r
- }\r
- \r
- if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){\r
- if (doread){\r
- getChar ();\r
- col++;\r
- }\r
- return t;\r
- }\r
- \r
- if (c == '"'){\r
- System.Text.StringBuilder s = new System.Text.StringBuilder ();\r
-\r
- while ((c = getChar ()) != -1){\r
- if (c == '"'){\r
- val = s.ToString ();\r
- return Token.LITERAL_STRING;\r
- }\r
-\r
- c = escape (c);\r
- if (c == -1)\r
- return Token.ERROR;\r
- s.Append ((char) c);\r
- }\r
- }\r
- \r
- // white space\r
- if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r'){\r
- if (c == '\t')\r
- col = (((col + 8) / 8) * 8) - 1;\r
- \r
- continue;\r
- }\r
-\r
- if (c == '@'){\r
- allow_keyword_as_ident = true;\r
- continue;\r
- }\r
-\r
- error_details = ((char)c).ToString ();\r
- \r
- return Token.ERROR;\r
- }\r
-\r
- if (current_token != Token.EOL) // if last token wasn´t EOL send it before EOF\r
- return Token.EOL;\r
- \r
- return Token.EOF;\r
- }\r
-\r
- public Tokenizer (System.IO.TextReader input, string fname, ArrayList defines)\r
- {\r
- this.ref_name = fname;\r
- reader = input;\r
- putback_char = -1;\r
- \r
- Location.Push (fname);\r
- }\r
-\r
- }\r
-}\r
+//
+// Mono.MonoBASIC.Tokenizer.cs: The Tokenizer for the MonoBASIC compiler
+//
+// Author: A Rafael D Teixeira (rafaelteixeirabr@hotmail.com)
+// : Manjula GHM (mmanjula@novell.com)
+// Based on cs-tokenizer.cs by Miguel de Icaza (miguel@gnu.org)
+//
+// Licensed under the terms of the GNU GPL
+//
+// Copyright (C) 2001 A Rafael D Teixeira
+//
+
+namespace Mono.MonoBASIC
+{
+ using System;
+ using System.Text;
+ using System.Collections;
+ using System.IO;
+ using System.Globalization;
+ using Mono.Languages;
+ using Mono.MonoBASIC;
+
+ /// <summary>
+ /// Tokenizer for MonoBASIC source code.
+ /// </summary>
+
+ public class Tokenizer : yyParser.yyInput
+ {
+ TextReader reader;
+ string file_name;
+ string ref_name;
+ int ref_line = 0;
+ int line = 0;
+ int col = 1;
+ public int current_token = Token.ERROR;
+ public int last_token = Token.ERROR;
+ bool handle_get_set = false;
+ bool cant_have_a_type_character = false;
+
+ public int ExpandedTabsSize = 4;
+
+ public string location {
+ get {
+ string det;
+
+ if (current_token == Token.ERROR)
+ det = "detail: " + error_details;
+ else
+ det = "";
+
+ return "Line: "+line+" Col: "+col + "\n" +
+ "VirtLine: "+ref_line +
+ " Token: "+current_token + " " + det;
+ }
+ }
+
+ public bool properties {
+ get {
+ return handle_get_set;
+ }
+
+ set {
+ handle_get_set = value;
+ }
+ }
+
+ //
+ // Class variables
+ //
+ static Hashtable keywords;
+ static NumberStyles styles;
+ static NumberFormatInfo csharp_format_info;
+
+ //
+ // Values for the associated token returned
+ //
+ StringBuilder number;
+ int putback_char = -1;
+ Object val;
+ long lon = 0;
+
+ //
+ // Details about the error encoutered by the tokenizer
+ //
+ string error_details;
+
+ public string error {
+ get {
+ return error_details;
+ }
+ }
+
+
+ public string Source {
+ get {
+ return file_name;
+ }
+
+ set {
+ file_name = value;
+ ref_name = value;
+ Location.SetCurrentSource(file_name);
+ }
+ }
+
+ public string EffectiveSource {
+ get {
+ return ref_name;
+ }
+ set {
+ ref_name = value;
+ Location.SetCurrentSource(ref_name);
+ }
+ }
+
+ public int Line {
+ get {
+ return line;
+ }
+ }
+
+ public int EffectiveLine {
+ get {
+ return ref_line;
+ }
+ set {
+ ref_line = value;
+ }
+ }
+
+ public int Col {
+ get {
+ return col;
+ }
+ }
+
+ static void initTokens ()
+ {
+ keywords = new Hashtable ();
+
+ keywords.Add ("addhandler", Token.ADDHANDLER);
+ keywords.Add ("addressof", Token.ADDRESSOF);
+ keywords.Add ("alias", Token.ALIAS);
+ keywords.Add ("and", Token.AND);
+ keywords.Add ("andalso", Token.ANDALSO);
+ keywords.Add ("ansi", Token.ANSI);
+ keywords.Add ("as", Token.AS);
+ keywords.Add ("assembly", Token.ASSEMBLY);
+ keywords.Add ("auto", Token.AUTO);
+ keywords.Add ("binary", Token.BINARY); // Not a VB.NET Keyword
+ keywords.Add ("boolean", Token.BOOLEAN);
+ keywords.Add ("byref", Token.BYREF);
+ keywords.Add ("byte", Token.BYTE);
+ keywords.Add ("byval", Token.BYVAL);
+ keywords.Add ("call", Token.CALL);
+ keywords.Add ("case", Token.CASE);
+ keywords.Add ("catch", Token.CATCH);
+ keywords.Add ("cbool", Token.CBOOL);
+ keywords.Add ("cbyte", Token.CBYTE);
+ keywords.Add ("cchar", Token.CCHAR);
+ keywords.Add ("cdate", Token.CDATE);
+ keywords.Add ("cdec", Token.CDEC);
+ keywords.Add ("cdbl", Token.CDBL);
+ keywords.Add ("char", Token.CHAR);
+ keywords.Add ("cint", Token.CINT);
+ keywords.Add ("class", Token.CLASS);
+ keywords.Add ("clng", Token.CLNG);
+ keywords.Add ("cobj", Token.COBJ);
+ keywords.Add ("compare", Token.COMPARE); // Not a VB.NET Keyword
+ keywords.Add ("const", Token.CONST);
+ keywords.Add ("cshort", Token.CSHORT);
+ keywords.Add ("csng", Token.CSNG);
+ keywords.Add ("cstr", Token.CSTR);
+ keywords.Add ("ctype", Token.CTYPE);
+ keywords.Add ("date", Token.DATE);
+ keywords.Add ("decimal", Token.DECIMAL);
+ keywords.Add ("declare", Token.DECLARE);
+ keywords.Add ("default", Token.DEFAULT);
+ keywords.Add ("delegate", Token.DELEGATE);
+ keywords.Add ("dim", Token.DIM);
+ keywords.Add ("directcast", Token.DIRECTCAST);
+ keywords.Add ("do", Token.DO);
+ keywords.Add ("double", Token.DOUBLE);
+ keywords.Add ("each", Token.EACH);
+ keywords.Add ("else", Token.ELSE);
+ keywords.Add ("elseif", Token.ELSEIF);
+ keywords.Add ("end", Token.END);
+ keywords.Add ("endif", Token.ENDIF); // An unused VB.NET keyword
+ keywords.Add ("enum", Token.ENUM);
+ keywords.Add ("erase", Token.ERASE);
+ keywords.Add ("error", Token.ERROR);
+ keywords.Add ("event", Token.EVENT);
+ keywords.Add ("exit", Token.EXIT);
+ keywords.Add ("explicit", Token.EXPLICIT); // Not a VB.NET keyword
+ keywords.Add ("false", Token.FALSE);
+ keywords.Add ("finally", Token.FINALLY);
+ keywords.Add ("for", Token.FOR);
+ keywords.Add ("friend", Token.FRIEND);
+ keywords.Add ("function", Token.FUNCTION);
+ keywords.Add ("get", Token.GET);
+ keywords.Add ("gettype", Token.GETTYPE);
+ keywords.Add ("gosub", Token.GOSUB); // An unused VB.NET keyword
+ keywords.Add ("goto", Token.GOTO);
+ keywords.Add ("handles", Token.HANDLES);
+ keywords.Add ("if", Token.IF);
+ keywords.Add ("implements", Token.IMPLEMENTS);
+ keywords.Add ("imports", Token.IMPORTS);
+ keywords.Add ("in", Token.IN);
+ keywords.Add ("inherits", Token.INHERITS);
+ keywords.Add ("integer", Token.INTEGER);
+ keywords.Add ("interface", Token.INTERFACE);
+ keywords.Add ("is", Token.IS);
+ keywords.Add ("let ", Token.LET ); // An unused VB.NET keyword
+ keywords.Add ("lib ", Token.LIB );
+ keywords.Add ("like", Token.LIKE );
+ keywords.Add ("long", Token.LONG);
+ keywords.Add ("loop", Token.LOOP);
+ keywords.Add ("me", Token.ME);
+ keywords.Add ("mod", Token.MOD);
+ keywords.Add ("module", Token.MODULE);
+ keywords.Add ("mustinherit", Token.MUSTINHERIT);
+ keywords.Add ("mustoverride", Token.MUSTOVERRIDE);
+ keywords.Add ("mybase", Token.MYBASE);
+ keywords.Add ("myclass", Token.MYCLASS);
+ keywords.Add ("namespace", Token.NAMESPACE);
+ keywords.Add ("new", Token.NEW);
+ keywords.Add ("next", Token.NEXT);
+ keywords.Add ("not", Token.NOT);
+ keywords.Add ("nothing", Token.NOTHING);
+ keywords.Add ("notinheritable", Token.NOTINHERITABLE);
+ keywords.Add ("notoverridable", Token.NOTOVERRIDABLE);
+ keywords.Add ("object", Token.OBJECT);
+ keywords.Add ("off", Token.OFF); // Not a VB.NET Keyword
+ keywords.Add ("on", Token.ON);
+ keywords.Add ("option", Token.OPTION);
+ keywords.Add ("optional", Token.OPTIONAL);
+ keywords.Add ("or", Token.OR);
+ keywords.Add ("orelse", Token.ORELSE);
+ keywords.Add ("overloads", Token.OVERLOADS);
+ keywords.Add ("overridable", Token.OVERRIDABLE);
+ keywords.Add ("overrides", Token.OVERRIDES);
+ keywords.Add ("paramarray", Token.PARAM_ARRAY);
+ keywords.Add ("preserve", Token.PRESERVE);
+ keywords.Add ("private", Token.PRIVATE);
+ keywords.Add ("property", Token.PROPERTY);
+ keywords.Add ("protected", Token.PROTECTED);
+ keywords.Add ("public", Token.PUBLIC);
+ keywords.Add ("raiseevent", Token.RAISEEVENT);
+ keywords.Add ("readonly", Token.READONLY);
+ keywords.Add ("redim", Token.REDIM);
+ keywords.Add ("rem", Token.REM);
+ keywords.Add ("removehandler", Token.REMOVEHANDLER);
+ keywords.Add ("resume", Token.RESUME);
+ keywords.Add ("return", Token.RETURN);
+ keywords.Add ("select", Token.SELECT);
+ keywords.Add ("set", Token.SET);
+ keywords.Add ("shadows", Token.SHADOWS);
+ keywords.Add ("shared", Token.SHARED);
+ keywords.Add ("short", Token.SHORT);
+ keywords.Add ("single", Token.SINGLE);
+ keywords.Add ("sizeof", Token.SIZEOF); // Not a VB.NET Keyword
+ keywords.Add ("static", Token.STATIC);
+ keywords.Add ("step", Token.STEP);
+ keywords.Add ("stop", Token.STOP);
+ keywords.Add ("strict", Token.STRICT); // Not a VB.NET Keyword
+ keywords.Add ("string", Token.STRING);
+ keywords.Add ("structure", Token.STRUCTURE);
+ keywords.Add ("sub", Token.SUB);
+ keywords.Add ("synclock", Token.SYNCLOCK);
+ keywords.Add ("text", Token.TEXT); // Not a VB.NET Keyword
+ keywords.Add ("then", Token.THEN);
+ keywords.Add ("throw", Token.THROW);
+ keywords.Add ("to", Token.TO);
+ keywords.Add ("true", Token.TRUE);
+ keywords.Add ("try", Token.TRY);
+ keywords.Add ("typeof", Token.TYPEOF);
+ keywords.Add ("unicode", Token.UNICODE);
+ keywords.Add ("until", Token.UNTIL);
+ keywords.Add ("variant", Token.VARIANT); // An unused VB.NET keyword
+ keywords.Add ("wend", Token.WEND); // An unused VB.NET keyword
+ keywords.Add ("when", Token.WHEN);
+ keywords.Add ("while", Token.WHILE);
+ keywords.Add ("with", Token.WITH);
+ keywords.Add ("withevents", Token.WITHEVENTS);
+ keywords.Add ("writeonly", Token.WRITEONLY);
+ keywords.Add ("xor", Token.XOR);
+
+ if (Parser.UseExtendedSyntax){
+ keywords.Add ("yield", Token.YIELD);
+ }
+
+ }
+
+ static Tokenizer ()
+ {
+ initTokens ();
+ csharp_format_info = new NumberFormatInfo ();
+ csharp_format_info.CurrencyDecimalSeparator = ".";
+ styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint;
+ }
+
+ public Tokenizer (System.IO.TextReader input, string fname, ArrayList defines)
+ {
+ this.Source = fname;
+
+ reader = input;
+
+ // putback an EOL at the beginning of a stream. This is a convenience that
+ // allows pre-processor directives to be added to the beginning of a vb file.
+ putback('\n');
+ }
+
+ bool is_keyword (string name)
+ {
+ bool res;
+ name = name.ToLower();
+
+ res = keywords.Contains(name);
+ if ((name == "GET" || name == "SET") && handle_get_set == false)
+ return false;
+ return res;
+ }
+
+ int getKeyword (string name)
+ {
+ return (int) (keywords [name.ToLower()]);
+ }
+
+ public Location Location {
+ get {
+ return new Location (ref_line, col);
+ }
+ }
+
+ public bool PropertyParsing {
+ get {
+ return handle_get_set;
+ }
+
+ set {
+ handle_get_set = value;
+ }
+ }
+
+ bool is_identifier_start_character (char c)
+ {
+ return Char.IsLetter (c) || c == '_' ;
+ }
+
+ bool is_identifier_part_character (char c)
+ {
+ return (Char.IsLetter (c) || Char.IsDigit (c) || c == '_');
+ }
+
+ int is_punct (char c, ref bool doread)
+ {
+ int d;
+ int t;
+
+ doread = false;
+
+ error_details = c.ToString();
+
+ d = peekChar ();
+
+ switch (c){
+ case '[':
+ return Token.OPEN_BRACKET;
+ case ']':
+ return Token.CLOSE_BRACKET;
+ case '{':
+ return Token.OPEN_BRACE;
+ case '}':
+ return Token.CLOSE_BRACE;
+ case '(':
+ return Token.OPEN_PARENS;
+ case ')':
+ return Token.CLOSE_PARENS;
+ case ',':
+ return Token.COMMA;
+ case '?':
+ return Token.INTERR;
+ case '!':
+ if (is_identifier_start_character((char)d) || cant_have_a_type_character)
+ return Token.EXCLAMATION;
+ return Token.SINGLETYPECHAR;
+ case '$':
+ if (cant_have_a_type_character)
+ return Token.ERROR;
+ return Token.DOLAR_SIGN;
+ case '@':
+ if (cant_have_a_type_character)
+ return Token.ERROR;
+ return Token.AT_SIGN;
+ case '%':
+ if (cant_have_a_type_character)
+ return Token.ERROR;
+ return Token.PERCENT;
+ case '#':
+ if(tokens_seen)
+ {
+ if (cant_have_a_type_character)
+ return ExtractDateTimeLiteral();
+ else
+ return Token.NUMBER_SIGN;
+ }
+ else
+ {
+ tokens_seen = true;
+ return Token.HASH;
+ }
+ case '&':
+ if (!cant_have_a_type_character)
+ return Token.LONGTYPECHAR;
+ t = handle_integer_literal_in_other_bases(d);
+ if (t == Token.NONE) {
+ t = Token.OP_CONCAT;
+ }
+ return t;
+ }
+
+ if (c == '+'){
+ if (d == '+')
+ t = Token.OP_INC;
+ else
+ return Token.PLUS;
+ doread = true;
+ return t;
+ }
+ if (c == '-'){
+ return Token.MINUS;
+ }
+
+ if (c == '='){
+ return Token.ASSIGN;
+ }
+
+ if (c == '*'){
+ return Token.STAR;
+ }
+
+ if (c == '/'){
+ return Token.DIV;
+ }
+
+ if (c == '\\'){
+ return Token.OP_IDIV;
+ }
+
+ if (c == '^'){
+ return Token.OP_EXP;
+ }
+
+ if (c == '<'){
+ if (d == '>')
+ {
+ doread = true;
+ return Token.OP_NE;
+ }
+ if (d == '='){
+ doread = true;
+ return Token.OP_LE;
+ }
+ if (d == '<')
+ {
+ doread = true;
+ return Token.OP_SHIFT_LEFT;
+ }
+ return Token.OP_LT;
+ }
+
+ if (c == '>'){
+ if (d == '='){
+ doread = true;
+ return Token.OP_GE;
+ }
+ if (d == '>')
+ {
+ doread = true;
+ return Token.OP_SHIFT_RIGHT;
+ }
+ return Token.OP_GT;
+ }
+
+ if (c == ':'){
+ if (d == '='){
+ doread = true;
+ return Token.ATTR_ASSIGN;
+ }
+ return Token.COLON;
+ }
+
+ return Token.ERROR;
+ }
+
+ bool decimal_digits (int c)
+ {
+ int d;
+ bool seen_digits = false;
+
+ if (c != -1)
+ number.Append ((char) c);
+ while ((d = peekChar ()) != -1){
+ if (Char.IsDigit ((char)d)){
+ number.Append ((char) d);
+ getChar ();
+ seen_digits = true;
+ } else
+ break;
+ }
+ return seen_digits;
+ }
+
+
+ int real_type_suffix (int c)
+ {
+ int t;
+
+ switch (c){
+ case 'F': case 'f':
+ t = Token.LITERAL_SINGLE;
+ break;
+ case 'R': case 'r':
+ t = Token.LITERAL_DOUBLE;
+ break;
+ case 'D': case 'd':
+ t= Token.LITERAL_DECIMAL;
+ break;
+ default:
+ return Token.NONE;
+ }
+ getChar ();
+ return t;
+ }
+
+ int integer_type_suffix (int c)
+ {
+ int t;
+
+ try {
+
+ switch (c){
+ case 'S': case 's':
+ t = Token.LITERAL_INTEGER; // SHORT ?
+
+ // hexadecimal literals - like &H8000S is "-32768"
+ // and not an overflow exception
+ // Check for other literals ???
+
+ if(lon == 32768) {
+ val = (short) lon;
+ }
+ else
+ val = ((IConvertible)val).ToInt16(null);
+ break;
+ case 'I': case 'i':
+ t = Token.LITERAL_INTEGER;
+ val = ((IConvertible)val).ToInt32(null);
+ break;
+ case 'L': case 'l':
+ t= Token.LITERAL_INTEGER; // LONG ?
+ val = ((IConvertible)val).ToInt64(null);
+ break;
+ default:
+ if ((long)val <= System.Int32.MaxValue &&
+ (long)val >= System.Int32.MinValue) {
+ val = ((IConvertible)val).ToInt32(null);
+ return Token.LITERAL_INTEGER;
+ } else {
+ val = ((IConvertible)val).ToInt64(null);
+ return Token.LITERAL_INTEGER; // LONG ?
+ }
+ }
+ getChar ();
+ return t;
+ } catch (Exception e) {
+ val = e.ToString();
+ return Token.ERROR;
+ }
+ }
+
+ int adjust_real (int t)
+ {
+ string s = number.ToString ();
+
+ switch (t){
+ case Token.LITERAL_DECIMAL:
+ val = new System.Decimal ();
+ val = System.Decimal.Parse (
+ s, styles, csharp_format_info);
+ break;
+ case Token.LITERAL_DOUBLE:
+ val = new System.Double ();
+ val = System.Double.Parse (
+ s, styles, csharp_format_info);
+ break;
+ case Token.LITERAL_SINGLE:
+ val = new System.Double ();
+ val = (float) System.Double.Parse (
+ s, styles, csharp_format_info);
+ break;
+
+ case Token.NONE:
+ val = new System.Double ();
+ val = System.Double.Parse (
+ s, styles, csharp_format_info);
+ t = Token.LITERAL_DOUBLE;
+ break;
+ }
+ return t;
+ }
+
+ long hex_digits ()
+ {
+ StringBuilder hexNumber = new StringBuilder ();
+
+ int d;
+
+ while ((d = peekChar ()) != -1){
+ char e = Char.ToUpper ((char) d);
+
+ if (Char.IsDigit (e) || (e >= 'A' && e <= 'F')){
+ hexNumber.Append (e);
+ getChar ();
+ } else
+ break;
+ }
+ lon = System.Int64.Parse (hexNumber.ToString(), NumberStyles.HexNumber);
+ return lon;
+ }
+
+ long octal_digits ()
+ {
+ long valueToReturn = 0;
+
+ int d;
+
+ while ((d = peekChar ()) != -1){
+ char e = (char)d;
+ if (Char.IsDigit (e) && (e < '8')){
+ valueToReturn *= 8;
+ valueToReturn += (d - (int)'0');
+ getChar ();
+ } else
+ break;
+ }
+
+ return valueToReturn;
+ }
+
+ int handle_integer_literal_in_other_bases(int peek)
+ {
+ if (peek == 'h' || peek == 'H'){
+ getChar ();
+ val = hex_digits ();
+ return integer_type_suffix (peekChar ());
+ }
+
+ if (peek == 'o' || peek == 'O'){
+ getChar ();
+ val = octal_digits ();
+ return integer_type_suffix (peekChar ());
+ }
+
+ return Token.NONE;
+ }
+
+ //
+ // Invoked if we know we have .digits or digits
+ //
+ int is_number (int c)
+ {
+ bool is_real = false;
+ number = new StringBuilder ();
+ int type;
+ bool non_prefixdecimal = false; //To capture decimals like .50
+
+ number.Length = 0;
+
+ if (Char.IsDigit ((char)c)){
+ decimal_digits (c);
+ c = peekChar ();
+ non_prefixdecimal = true;
+ }
+
+ //
+ // We need to handle the case of
+ // "1.1" vs "1.ToString()" (LITERAL_SINGLE vs NUMBER DOT IDENTIFIER)
+ //
+ if (c == '.'){
+ if (non_prefixdecimal == false)
+ putback ('.');
+ if (decimal_digits (getChar())){
+ is_real = true;
+ c = peekChar ();
+ } else {
+ putback ('.');
+ number.Length -= 1;
+ val = System.Int64.Parse(number.ToString());
+ return integer_type_suffix('.');
+ }
+ }
+
+ if (c == 'e' || c == 'E'){
+ is_real = true;
+ number.Append ("e");
+ getChar ();
+
+ c = peekChar ();
+ if (c == '+'){
+ number.Append ((char) c);
+ getChar ();
+ c = peekChar ();
+ } else if (c == '-'){
+ number.Append ((char) c);
+ getChar ();
+ c = peekChar ();
+ }
+ decimal_digits (-1);
+ c = peekChar ();
+ }
+
+ type = real_type_suffix (c);
+ if (type == Token.NONE && !is_real){
+ val = System.Int64.Parse(number.ToString());
+ return integer_type_suffix(c);
+ }
+
+ return adjust_real (type);
+ }
+
+ int getChar ()
+ {
+ if (putback_char != -1){
+ int x = putback_char;
+ putback_char = -1;
+
+ return x;
+ }
+ return reader.Read ();
+ }
+
+ int peekChar ()
+ {
+ if (putback_char != -1)
+ return putback_char;
+ return reader.Peek ();
+ }
+
+
+ void putback (int c)
+ {
+ if (putback_char != -1)
+ throw new Exception ("This should not happen putback on putback");
+ putback_char = c;
+ }
+
+ public bool advance ()
+ {
+ return current_token != Token.EOF ;
+ }
+
+ public Object Value {
+ get {
+ return val;
+ }
+ }
+
+ public Object value ()
+ {
+ return val;
+ }
+
+ private bool IsEOL(int currentChar)
+ {
+ bool retVal;
+
+ if (currentChar == 0x0D) {
+ if (peekChar() == 0x0A) // if it is a CR-LF pair consume LF also
+ getChar();
+
+ retVal = true;
+ }
+ else {
+ retVal = (currentChar == -1 || currentChar == 0x0A || currentChar == 0x2028 || currentChar == 0x2029);
+ }
+
+ if(retVal) {
+ nextLine();
+ }
+
+ return retVal;
+ }
+
+ private int DropComments()
+ {
+ //int d;
+ while (!IsEOL(/*d =*/ getChar ()))
+ col++;
+
+ return Token.EOL;
+ }
+
+ public bool putbacktoken = false;
+ public bool flag = false;
+ int next_token;
+
+ public int token ()
+ {
+ int before_last_token = last_token;
+ last_token = current_token;
+ do
+ {
+ current_token = xtoken ();
+ if(current_token == Token.END) {
+ next_token = xtoken();
+ putbacktoken = true;
+ if (next_token == Token.EOL)
+ return Token.END_EOL;
+ else
+ return Token.END;
+ }
+ if (current_token == Token.COLON) {
+ next_token = xtoken();
+ putbacktoken = true;
+ if (next_token == Token.EOL) {
+ if (last_token != Token.LABELNAME && last_token != Token.LITERAL_INTEGER) {
+ current_token = Token.EOL;
+ putbacktoken = false;
+ }
+ else if (before_last_token == Token.GOTO) {
+ current_token = Token.EOL;
+ putbacktoken = false;
+ }
+ }
+ }
+ if (current_token == 0)
+ return Token.EOF;
+ if (current_token == Token.REM)
+ current_token = DropComments();
+ } while (last_token == Token.EOL && current_token == Token.EOL);
+
+ return current_token;
+ }
+
+ private string GetIdentifier()
+ {
+ int c = getChar();
+ if (is_identifier_start_character ((char) c))
+ return GetIdentifier(c);
+ else
+ return null;
+ }
+
+ private bool IsLabel ()
+ {
+ char c = (char) peekChar();
+ //putback (c);
+ return (c == ':');
+ }
+
+ private string GetIdentifier(int c)
+ {
+ StringBuilder id = new StringBuilder ();
+
+ id.Append ((char) c);
+
+ while ((c = peekChar ()) != -1)
+ {
+ if (is_identifier_part_character ((char) c))
+ {
+ id.Append ((char)getChar ());
+ col++;
+ }
+ else
+ break;
+ }
+
+ cant_have_a_type_character = false;
+
+ return id.ToString();
+ }
+
+ private bool is_doublequote(int currentChar)
+ {
+ return (currentChar == '"' ||
+ currentChar == 0x201C || // unicode left double-quote character
+ currentChar == 0x201D); // unicode right double-quote character
+ }
+
+ private bool is_whitespace(int c)
+ {
+ return (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0);
+ }
+
+ private bool tokens_seen = false;
+
+ private void nextLine()
+ {
+ cant_have_a_type_character = true;
+ line++;
+ ref_line++;
+ col = 0;
+ tokens_seen = false;
+ }
+
+ public int xtoken ()
+ {
+ int t;
+ bool doread = false;
+ int c;
+
+ if (putbacktoken == true) {
+ putbacktoken = false;
+ return next_token;
+ }
+
+ val = null;
+ for (;(c = getChar ()) != -1; col++) {
+
+ // Handle line continuation character
+ if (c == '_')
+ {
+ int d = peekChar();
+ if (!is_identifier_part_character((char)d)) {
+ while ((c = getChar ()) != -1 && !IsEOL(c)) {}
+ c = getChar ();
+ tokens_seen = true;
+ }
+ }
+
+
+ // white space
+ if (is_whitespace(c)) {
+ // expand tabs for location
+ if (c == '\t')
+ col = (((col + ExpandedTabsSize) / ExpandedTabsSize) * ExpandedTabsSize) - 1;
+ cant_have_a_type_character = true;
+ continue;
+ }
+
+ // Handle line comments.
+ if (c == '\'')
+ return Token.REM;
+
+ // Handle EOL.
+ if (IsEOL(c))
+ {
+ if (current_token == Token.EOL) // if last token was also EOL keep skipping
+ continue;
+ return Token.EOL;
+ }
+
+ // Handle escaped identifiers
+ if (c == '[')
+ {
+ bool is_first_token_in_line = !tokens_seen;
+ if ((val = GetIdentifier()) == null)
+ break;
+ if ((c = getChar()) != ']')
+ break;
+ tokens_seen = true;
+ if (IsLabel() && is_first_token_in_line)
+ return Token.LABELNAME;
+
+ if (last_token == Token.GOTO)
+ return Token.LABELNAME;
+ return Token.IDENTIFIER;
+ }
+
+ // Handle unescaped identifiers
+ if (is_identifier_start_character ((char) c))
+ {
+ string id;
+ bool is_first_token_in_line = !tokens_seen;
+ if ((id = GetIdentifier(c)) == null)
+ break;
+ val = id;
+ tokens_seen = true;
+ if (is_keyword(id) && (current_token != Token.DOT))
+ return getKeyword(id);
+
+ if (IsLabel() && is_first_token_in_line)
+ return Token.LABELNAME;
+
+ if (last_token == Token.GOTO)
+ return Token.LABELNAME;
+ return Token.IDENTIFIER;
+ }
+
+ // Treat string literals
+ if (is_doublequote(c)) {
+ cant_have_a_type_character = true;
+ return ExtractStringOrCharLiteral(c);
+ }
+
+ // handle numeric literals
+
+ if (Char.IsDigit ((char) c))
+ {
+ cant_have_a_type_character = false;
+ tokens_seen = true;
+ return is_number (c);
+ }
+
+ if (c == '.')
+ {
+ cant_have_a_type_character = true;
+ tokens_seen = true;
+ if (Char.IsDigit ((char) peekChar ()))
+ return is_number (c);
+ return Token.DOT;
+ }
+ if ((t = is_punct ((char)c, ref doread)) != Token.ERROR) {
+ cant_have_a_type_character = true;
+
+ if (t == Token.NONE)
+ continue;
+
+ if (doread){
+ getChar ();
+ col++;
+ }
+ tokens_seen = true;
+ return t;
+ }
+
+ error_details = ((char)c).ToString ();
+ return Token.ERROR;
+ }
+
+ if (current_token != Token.EOL) // if last token wasn't EOL send it before EOF
+ return Token.EOL;
+
+ return Token.EOF;
+ }
+
+ private int ExtractDateTimeLiteral()
+ {
+ int c;
+
+ StringBuilder sb = new StringBuilder();
+ for (;(c = getChar ()) != -1; col++)
+ {
+ if (c == '#') {
+ val = ParseDateLiteral(sb);
+ return Token.LITERAL_DATE;
+ }
+ if (IsEOL(c)) {
+ break;
+ }
+ if (c == '-')
+ c = '/';
+ sb.Append((char)c);
+ }
+ return Token.ERROR;
+ }
+
+ private int ExtractStringOrCharLiteral(int c)
+ {
+ StringBuilder s = new StringBuilder ();
+
+ tokens_seen = true;
+
+ while ((c = getChar ()) != -1){
+ if (is_doublequote(c)){
+ if (is_doublequote(peekChar()))
+ getChar();
+ else {
+ //handle Char Literals
+ if (peekChar() == 'C' || peekChar() == 'c') {
+ getChar();
+ if (s.Length == 1) {
+ val = s[0];
+ return Token.LITERAL_CHARACTER;
+ } else {
+ val = "Incorrect length for a character literal";
+ return Token.ERROR;
+ }
+ } else {
+ val = s.ToString ();
+ return Token.LITERAL_STRING;
+ }
+ }
+ }
+
+ if (IsEOL(c)) {
+ return Token.ERROR;
+ }
+
+ s.Append ((char) c);
+ }
+
+ return Token.ERROR;
+ }
+
+ static IFormatProvider enUSculture = new CultureInfo("en-US", true);
+
+ private DateTime ParseDateLiteral(StringBuilder value)
+ {
+ try
+ {
+ return DateTime.Parse(value.ToString(),
+ enUSculture,
+ DateTimeStyles.NoCurrentDateDefault | DateTimeStyles.AllowWhiteSpaces);
+ }
+ catch (FormatException ex)
+ {
+ //TODO: What is the correct error number and message?
+ Report.Error (1, Location, string.Format("Invalid date literal '{0}'", value.ToString())
+ + Environment.NewLine + ex.ToString());
+ }
+ catch (Exception)
+ {
+ Report.Error (1, Location, "Error parsing date literal"); //TODO: What is the correct error number and message?
+ }
+ return new DateTime();
+ }
+
+ public void PositionCursorAtNextPreProcessorDirective()
+ {
+ int t;
+
+ for(t = token(); t != Token.HASH && t != Token.EOF ; t = token());
+
+ if(t == Token.EOF)
+ throw new ApplicationException("Unexpected EOF while looking for a pre-processor directive");
+
+ if(t == Token.HASH) {
+ tokens_seen = false;
+ putback('#');
+ }
+ }
+
+ }
+}
projects / mono.git / blobdiff