//\r
// cs-tokenizer.cs: The Tokenizer for the C# compiler\r
+// This also implements the preprocessor\r
//\r
// Author: Miguel de Icaza (miguel@gnu.org)\r
//\r
//\r
\r
/*\r
- Todo:\r
-\r
- Do something with the integer and float suffixes, pass full datatype?\r
- Make sure we accept the proper Unicode ranges, per the spec.\r
-\r
- Open issues:\r
-\r
- * Data type handling\r
- \r
- Currently I am returning different tokens for the various\r
- kinds of floating point types (float, double, decimal) and I\r
- am only returning a single token for all integer values\r
- (integer, unsigned int, etc) as an experiment as to see\r
- which mechanism is better.\r
- \r
- I do not know yet how I will be doing the mapping of "int"\r
- to things like System.Int32 and so on. I am confused. MAN\r
- I AM C\r
- \r
- Indeed, this might be the core of the problem, I should\r
- *probably* just return a TYPE token and have the value of\r
- the token be stuff like `System.Int32', `System.UInt32',\r
- `System.Double' and so on. I will see.\r
-\r
- * Error reporting.\r
-\r
- I was returning Token.ERROR on errors and setting an\r
- internal error string with the details, but it might make sense\r
- to just use exceptions.\r
-\r
- Change of mind: I think I want to keep returning errors *UNLESS* the\r
- parser is catching errors from the tokenizer (at that point, there is\r
- not really any reason to use exceptions) so that I can continue the\r
- parsing \r
-\r
- * IDEA\r
-\r
- I think I have solved the problem. The idea is to not even *bother*\r
- about handling data types a lot here (except for fitting data into\r
- the proper places), but let the upper layer handle it.\r
-\r
- Ie, treat LITERAL_CHARACTER, LITERAL_INTEGER, LITERAL_FLOAT, LITERAL_DOUBLE, and\r
- return then as `LITERAL_LITERAL' with maybe subdetail information\r
-\r
+ * TODO:\r
+ * Make sure we accept the proper Unicode ranges, per the spec.\r
+ * Report error 1032\r
*/\r
\r
using System;\r
using System.Text;\r
-using CSC;\r
using System.Collections;\r
using System.IO;\r
using System.Globalization;\r
\r
-namespace CSC\r
+namespace Mono.CSharp\r
{\r
- using CSC;\r
- \r
/// <summary>\r
/// Tokenizer for C# source code. \r
/// </summary>\r
- \r
+\r
public class Tokenizer : yyParser.yyInput\r
{\r
StreamReader reader;\r
public int col = 1;\r
public int current_token;\r
bool handle_get_set = false;\r
+ bool handle_remove_add = false;\r
+ bool handle_assembly = false;\r
+\r
+ //\r
+ // Whether tokens have been seen on this line\r
+ //\r
+ bool tokens_seen = false;\r
\r
+ //\r
+ // Whether a token has been seen on the file\r
+ // This is needed because `define' is not allowed to be used\r
+ // after a token has been seen.\r
+ //\r
+ bool any_token_seen = false;\r
+ \r
+ //\r
+ // Returns a verbose representation of the current location\r
+ //\r
public string location {\r
get {\r
string det;\r
else\r
det = "";\r
\r
- return "Line: "+line+" Col: "+col + "\n" +\r
- "VirtLine: "+ref_line +\r
- " Token: "+current_token + " " + det;\r
+ // return "Line: "+line+" Col: "+col + "\n" +\r
+ // "VirtLine: "+ref_line +\r
+ // " Token: "+current_token + " " + det;\r
+\r
+ return ref_name + " " + "(" + line + "," + col + "), Token:" + current_token + " " + det;\r
}\r
}\r
\r
- public bool properties {\r
+ public bool PropertyParsing {\r
get {\r
return handle_get_set;\r
}\r
handle_get_set = value;\r
}\r
}\r
+\r
+ public bool AssemblyTargetParsing {\r
+ get {\r
+ return handle_assembly;\r
+ }\r
+\r
+ set {\r
+ handle_assembly = value;\r
+ }\r
+ }\r
+\r
+ public bool EventParsing {\r
+ get {\r
+ return handle_remove_add;\r
+ }\r
+\r
+ set {\r
+ handle_remove_add = value;\r
+ }\r
+ }\r
\r
//\r
// Class variables\r
//\r
// Values for the associated token returned\r
//\r
- System.Text.StringBuilder number;\r
int putback_char;\r
Object val;\r
+\r
+ //\r
+ // Pre-processor\r
+ //\r
+ Hashtable defines;\r
+\r
+ const int TAKING = 1;\r
+ const int TAKEN_BEFORE = 2;\r
+ const int ELSE_SEEN = 4;\r
+ const int PARENT_TAKING = 8;\r
+ \r
+ //\r
+ // pre-processor if stack state:\r
+ //\r
+ Stack ifstack;\r
+\r
+ static System.Text.StringBuilder id_builder;\r
+ static System.Text.StringBuilder string_builder;\r
+ static System.Text.StringBuilder number_builder;\r
\r
//\r
// Details about the error encoutered by the tokenizer\r
}\r
}\r
\r
- static void initTokens ()\r
+ static void InitTokens ()\r
{\r
keywords = new Hashtable ();\r
\r
keywords.Add ("abstract", Token.ABSTRACT);\r
keywords.Add ("as", Token.AS);\r
+ keywords.Add ("add", Token.ADD);\r
+ keywords.Add ("assembly", Token.ASSEMBLY);\r
keywords.Add ("base", Token.BASE);\r
keywords.Add ("bool", Token.BOOL);\r
keywords.Add ("break", Token.BREAK);\r
keywords.Add ("interface", Token.INTERFACE);\r
keywords.Add ("internal", Token.INTERNAL);\r
keywords.Add ("is", Token.IS);\r
- keywords.Add ("lock ", Token.LOCK );\r
+ keywords.Add ("lock", Token.LOCK);\r
keywords.Add ("long", Token.LONG);\r
keywords.Add ("namespace", Token.NAMESPACE);\r
keywords.Add ("new", Token.NEW);\r
keywords.Add ("public", Token.PUBLIC);\r
keywords.Add ("readonly", Token.READONLY);\r
keywords.Add ("ref", Token.REF);\r
+ keywords.Add ("remove", Token.REMOVE);\r
keywords.Add ("return", Token.RETURN);\r
keywords.Add ("sbyte", Token.SBYTE);\r
keywords.Add ("sealed", Token.SEALED);\r
keywords.Add ("set", Token.SET);\r
keywords.Add ("short", Token.SHORT);\r
keywords.Add ("sizeof", Token.SIZEOF);\r
+ keywords.Add ("stackalloc", Token.STACKALLOC);\r
keywords.Add ("static", Token.STATIC);\r
keywords.Add ("string", Token.STRING);\r
keywords.Add ("struct", Token.STRUCT);\r
keywords.Add ("using", Token.USING);\r
keywords.Add ("virtual", Token.VIRTUAL);\r
keywords.Add ("void", Token.VOID);\r
+ keywords.Add ("volatile", Token.VOLATILE);\r
keywords.Add ("while", Token.WHILE);\r
}\r
\r
// \r
static Tokenizer ()\r
{\r
- initTokens ();\r
+ InitTokens ();\r
csharp_format_info = new NumberFormatInfo ();\r
csharp_format_info.CurrencyDecimalSeparator = ".";\r
styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint;\r
+ id_builder = new System.Text.StringBuilder ();\r
+ string_builder = new System.Text.StringBuilder ();\r
+ number_builder = new System.Text.StringBuilder ();\r
}\r
\r
bool is_keyword (string name)\r
bool res;\r
\r
res = keywords.Contains (name);\r
- if ((name == "get" || name == "set") && handle_get_set == false)\r
+ if (handle_get_set == false && (name == "get" || name == "set"))\r
+ return false;\r
+ if (handle_remove_add == false && (name == "remove" || name == "add"))\r
+ return false;\r
+ if (handle_assembly == false && (name == "assembly"))\r
return false;\r
return res;\r
}\r
\r
- int getKeyword (string name)\r
+ int GetKeyword (string name)\r
{\r
return (int) (keywords [name]);\r
}\r
+\r
+ public Location Location {\r
+ get {\r
+ return new Location (ref_line);\r
+ }\r
+ }\r
\r
- public Tokenizer (System.IO.Stream input, string fname)\r
+ public Tokenizer (System.IO.Stream input, string fname, ArrayList defs)\r
{\r
this.ref_name = fname;\r
reader = new System.IO.StreamReader (input);\r
putback_char = -1;\r
+\r
+ if (defs != null){\r
+ defines = new Hashtable ();\r
+ foreach (string def in defs)\r
+ defines [def] = true;\r
+ }\r
+\r
+ //\r
+ // FIXME: This could be `Location.Push' but we have to\r
+ // find out why the MS compiler allows this\r
+ //\r
+ Mono.CSharp.Location.Push (fname);\r
}\r
\r
bool is_identifier_start_character (char c)\r
{\r
- return CharacterInfo.IsLetter (c) || c == '_' ;\r
+ return Char.IsLetter (c) || c == '_' ;\r
}\r
\r
bool is_identifier_part_character (char c)\r
{\r
- return (CharacterInfo.IsLetter (c) || CharacterInfo.IsDigit (c) || c == '_');\r
+ return (Char.IsLetter (c) || Char.IsDigit (c) || c == '_');\r
}\r
\r
int is_punct (char c, ref bool doread)\r
{\r
- int idx = "{}[](),:;~+-*/%&|^!=<>?".IndexOf (c);\r
int d;\r
int t;\r
\r
else if (d == '=')\r
t = Token.OP_SUB_ASSIGN;\r
else if (d == '>')\r
- return Token.OP_PTR;\r
+ t = Token.OP_PTR;\r
else\r
return Token.MINUS;\r
doread = true;\r
bool seen_digits = false;\r
\r
if (c != -1)\r
- number.Append ((char) c);\r
+ number_builder.Append ((char) c);\r
\r
while ((d = peekChar ()) != -1){\r
- if (CharacterInfo.IsDigit ((char)d)){\r
- number.Append ((char) d);\r
+ if (Char.IsDigit ((char)d)){\r
+ number_builder.Append ((char) d);\r
getChar ();\r
seen_digits = true;\r
} else\r
break;\r
}\r
+ \r
return seen_digits;\r
}\r
\r
+ bool is_hex (char e)\r
+ {\r
+ return Char.IsDigit (e) || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');\r
+ }\r
+ \r
void hex_digits (int c)\r
{\r
int d;\r
\r
if (c != -1)\r
- number.Append ((char) c);\r
+ number_builder.Append ((char) c);\r
while ((d = peekChar ()) != -1){\r
char e = Char.ToUpper ((char) d);\r
\r
- if (CharacterInfo.IsDigit (e) ||\r
- (e >= 'A' && e <= 'F')){\r
- number.Append ((char) e);\r
+ if (is_hex (e)){\r
+ number_builder.Append ((char) e);\r
getChar ();\r
} else\r
break;\r
int real_type_suffix (int c)\r
{\r
int t;\r
- \r
+\r
switch (c){\r
case 'F': case 'f':\r
t = Token.LITERAL_FLOAT;\r
default:\r
return Token.NONE;\r
}\r
- getChar ();\r
return t;\r
}\r
\r
- int integer_type_suffix (int c)\r
+ int integer_type_suffix (ulong ul, int c)\r
{\r
- // FIXME: Handle U and L suffixes.\r
- // We also need to see in which kind of\r
- // Int the thing fits better according to the spec.\r
+ bool is_unsigned = false;\r
+ bool is_long = false;\r
+\r
+ if (c != -1){\r
+ bool scanning = true;\r
+ do {\r
+ switch (c){\r
+ case 'U': case 'u':\r
+ if (is_unsigned)\r
+ scanning = false;\r
+ is_unsigned = true;\r
+ getChar ();\r
+ break;\r
+\r
+ case 'l':\r
+ if (!is_unsigned){\r
+ //\r
+ // if we have not seen anything in between\r
+ // report this error\r
+ //\r
+ Report.Warning (\r
+ 78, Location,\r
+ "the 'l' suffix is easily confused with digit `1'," +\r
+ " use 'L' for clarity");\r
+ }\r
+ goto case 'L';\r
+ \r
+ case 'L': \r
+ if (is_long)\r
+ scanning = false;\r
+ is_long = true;\r
+ getChar ();\r
+ break;\r
+ \r
+ default:\r
+ scanning = false;\r
+ break;\r
+ }\r
+ c = peekChar ();\r
+ } while (scanning);\r
+ }\r
+\r
+ if (is_long && is_unsigned){\r
+ val = ul;\r
+ return Token.LITERAL_INTEGER;\r
+ } else if (is_unsigned){\r
+ // uint if possible, or ulong else.\r
+\r
+ if ((ul & 0xffffffff00000000) == 0)\r
+ val = (uint) ul;\r
+ else\r
+ val = ul;\r
+ } else if (is_long){\r
+ // long if possible, ulong otherwise\r
+ if ((ul & 0x8000000000000000) != 0)\r
+ val = ul;\r
+ else\r
+ val = (long) ul;\r
+ } else {\r
+ // int, uint, long or ulong in that order\r
+ if ((ul & 0xffffffff00000000) == 0){\r
+ uint ui = (uint) ul;\r
+ \r
+ if ((ui & 0x80000000) != 0)\r
+ val = ui;\r
+ else\r
+ val = (int) ui;\r
+ } else {\r
+ if ((ul & 0x8000000000000000) != 0)\r
+ val = ul;\r
+ else\r
+ val = (long) ul;\r
+ }\r
+ }\r
return Token.LITERAL_INTEGER;\r
}\r
- \r
- void adjust_int (int t)\r
+ \r
+ //\r
+ // given `c' as the next char in the input decide whether\r
+ // we need to convert to a special type, and then choose\r
+ // the best representation for the integer\r
+ //\r
+ int adjust_int (int c)\r
{\r
- val = new System.Int32();\r
- val = System.Int32.Parse (number.ToString (), 0);\r
+ ulong ul = System.UInt64.Parse (number_builder.ToString ());\r
+ return integer_type_suffix (ul, c);\r
}\r
\r
int adjust_real (int t)\r
{\r
- string s = number.ToString ();\r
+ string s = number_builder.ToString ();\r
\r
- Console.WriteLine (s);\r
switch (t){\r
case Token.LITERAL_DECIMAL:\r
val = new System.Decimal ();\r
int is_number (int c)\r
{\r
bool is_real = false;\r
- number = new System.Text.StringBuilder ();\r
int type;\r
\r
- number.Length = 0;\r
+ number_builder.Length = 0;\r
\r
- if (CharacterInfo.IsDigit ((char)c)){\r
- if (peekChar () == 'x' || peekChar () == 'X'){\r
+ if (Char.IsDigit ((char)c)){\r
+ if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){\r
+ ulong ul;\r
getChar ();\r
hex_digits (-1);\r
- val = new System.Int32 ();\r
- val = System.Int32.Parse (number.ToString (), NumberStyles.HexNumber);\r
- return integer_type_suffix (peekChar ());\r
+\r
+ string s = number_builder.ToString ();\r
+\r
+ ul = System.UInt64.Parse (s, NumberStyles.HexNumber);\r
+ return integer_type_suffix (ul, peekChar ());\r
}\r
decimal_digits (c);\r
c = getChar ();\r
if (c == '.'){\r
if (decimal_digits ('.')){\r
is_real = true;\r
- c = peekChar ();\r
+ c = getChar ();\r
} else {\r
putback ('.');\r
- number.Length -= 1;\r
- adjust_int (Token.LITERAL_INTEGER);\r
- return Token.LITERAL_INTEGER;\r
+ number_builder.Length -= 1;\r
+ return adjust_int (-1);\r
}\r
}\r
\r
if (c == 'e' || c == 'E'){\r
is_real = true;\r
- number.Append ("e");\r
- getChar ();\r
+ number_builder.Append ("e");\r
+ c = getChar ();\r
\r
- c = peekChar ();\r
if (c == '+'){\r
- number.Append ((char) c);\r
- getChar ();\r
- c = peekChar ();\r
+ number_builder.Append ((char) c);\r
+ c = getChar ();\r
} else if (c == '-'){\r
- number.Append ((char) c);\r
- getChar ();\r
- c = peekChar ();\r
+ number_builder.Append ((char) c);\r
+ c = getChar ();\r
}\r
decimal_digits (-1);\r
- c = peekChar ();\r
+ c = getChar ();\r
}\r
\r
type = real_type_suffix (c);\r
if (type == Token.NONE && !is_real){\r
- type = integer_type_suffix (c);\r
- adjust_int (type);\r
putback (c);\r
- return type;\r
- } else\r
+ return adjust_int (c);\r
+ } else \r
is_real = true;\r
\r
+ if (type == Token.NONE){\r
+ putback (c);\r
+ }\r
+ \r
if (is_real)\r
return adjust_real (type);\r
\r
Console.WriteLine ("This should not be reached");\r
throw new Exception ("Is Number should never reach this point");\r
}\r
+\r
+ //\r
+ // Accepts exactly count (4 or 8) hex, no more no less\r
+ //\r
+ int getHex (int count, out bool error)\r
+ {\r
+ int [] buffer = new int [8];\r
+ int i;\r
+ int total = 0;\r
+ int c;\r
+ char e;\r
+ int top = count != -1 ? count : 4;\r
\r
+ getChar ();\r
+ error = false;\r
+ for (i = 0; i < top; i++){\r
+ c = getChar ();\r
+ e = Char.ToUpper ((char) c);\r
+ \r
+ if (!is_hex (e)){\r
+ error = true;\r
+ return 0;\r
+ }\r
+ if (Char.IsDigit (e))\r
+ c = (int) e - (int) '0';\r
+ else\r
+ c = (int) e - (int) 'A';\r
+ total = (total * 16) + c;\r
+ if (count == -1){\r
+ int p = peekChar ();\r
+ if (p == -1)\r
+ break;\r
+ if (!is_hex ((char)p))\r
+ break;\r
+ }\r
+ }\r
+ return total;\r
+ }\r
+\r
int escape (int c)\r
{\r
+ bool error;\r
int d;\r
int v;\r
\r
case 'v':\r
v = '\v'; break;\r
case 'r':\r
- v = 'c'; break;\r
+ v = '\r'; break;\r
case '\\':\r
v = '\\'; break;\r
case 'f':\r
v = '"'; break;\r
case '\'':\r
v = '\''; break;\r
+ case 'x':\r
+ v = getHex (-1, out error);\r
+ if (error)\r
+ goto default;\r
+ return v;\r
+ case 'u':\r
+ v = getHex (4, out error);\r
+ if (error)\r
+ goto default;\r
+ return v;\r
+ case 'U':\r
+ v = getHex (8, out error);\r
+ if (error)\r
+ goto default;\r
+ return v;\r
default:\r
- error_details = "cs1009: Unrecognized escape sequence " + (char)d;\r
+ Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);\r
return -1;\r
}\r
getChar ();\r
current_token = xtoken ();\r
return current_token;\r
}\r
+\r
+ static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();\r
+ \r
+ void get_cmd_arg (out string cmd, out string arg)\r
+ {\r
+ int c;\r
+ \r
+ tokens_seen = false;\r
+ arg = "";\r
+ static_cmd_arg.Length = 0;\r
+ \r
+ while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t')){\r
+ if (c == '\r')\r
+ continue;\r
+ static_cmd_arg.Append ((char) c);\r
+ }\r
+\r
+ cmd = static_cmd_arg.ToString ();\r
+\r
+ if (c == '\n'){\r
+ line++;\r
+ ref_line++;\r
+ return;\r
+ }\r
+\r
+ // skip over white space\r
+ while ((c = getChar ()) != -1 && (c != '\n') && ((c == ' ') || (c == '\t')))\r
+ ;\r
+\r
+ if (c == '\n'){\r
+ line++;\r
+ ref_line++;\r
+ return;\r
+ }\r
+ \r
+ static_cmd_arg.Length = 0;\r
+ static_cmd_arg.Append ((char) c);\r
+ \r
+ while ((c = getChar ()) != -1 && (c != '\n')){\r
+ if (c == '\r')\r
+ continue;\r
+ static_cmd_arg.Append ((char) c);\r
+ }\r
+\r
+ if (c == '\n'){\r
+ line++;\r
+ ref_line++;\r
+ }\r
+ arg = static_cmd_arg.ToString ().Trim ();\r
+ }\r
+\r
+ //\r
+ // Handles the #line directive\r
+ //\r
+ bool PreProcessLine (string arg)\r
+ {\r
+ if (arg == "")\r
+ return false;\r
+\r
+ if (arg == "default"){\r
+ line = ref_line = line;\r
+ return false;\r
+ }\r
+ \r
+ try {\r
+ int pos;\r
+\r
+ if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){\r
+ ref_line = System.Int32.Parse (arg.Substring (0, pos));\r
+ pos++;\r
+ \r
+ char [] quotes = { '\"' };\r
+ \r
+ ref_name = arg.Substring (pos). Trim(quotes);\r
+ } else {\r
+ ref_line = System.Int32.Parse (arg);\r
+ }\r
+ } catch {\r
+ return false;\r
+ }\r
+ \r
+ return true;\r
+ }\r
+\r
+ //\r
+ // Handles #define and #undef\r
+ //\r
+ void PreProcessDefinition (bool is_define, string arg)\r
+ {\r
+ if (arg == "" || arg == "true" || arg == "false"){\r
+ Report.Error(1001, Location, "Missing identifer to pre-processor directive");\r
+ return;\r
+ }\r
+\r
+ if (is_define){\r
+ if (defines == null)\r
+ defines = new Hashtable ();\r
+ defines [arg] = 1;\r
+ } else {\r
+ if (defines == null)\r
+ return;\r
+ if (defines.Contains (arg))\r
+ defines.Remove (arg);\r
+ }\r
+ }\r
+\r
+ bool eval_val (string s)\r
+ {\r
+ if (s == "true")\r
+ return true;\r
+ if (s == "false")\r
+ return false;\r
+ \r
+ if (defines == null)\r
+ return false;\r
+ if (defines.Contains (s))\r
+ return true;\r
+\r
+ return false;\r
+ }\r
+\r
+ bool pp_primary (ref string s)\r
+ {\r
+ s = s.Trim ();\r
+ int len = s.Length;\r
+\r
+ if (len > 0){\r
+ char c = s [0];\r
+ \r
+ if (c == '('){\r
+ s = s.Substring (1);\r
+ bool val = pp_expr (ref s);\r
+ if (s.Length > 0 && s [0] == ')')\r
+ return val;\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ \r
+ if (Char.IsLetter (c) || c == '_'){\r
+ int j = 1;\r
+\r
+ while (j < len){\r
+ c = s [j];\r
+ \r
+ if (Char.IsLetter (c) || Char.IsDigit (c) || c == '_'){\r
+ j++;\r
+ continue;\r
+ }\r
+ bool v = eval_val (s.Substring (0, j));\r
+ s = s.Substring (j);\r
+ return v;\r
+ }\r
+ bool vv = eval_val (s);\r
+ s = "";\r
+ return vv;\r
+ }\r
+ }\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ \r
+ bool pp_unary (ref string s)\r
+ {\r
+ s = s.Trim ();\r
+ int len = s.Length;\r
+\r
+ if (len > 0){\r
+ if (s [0] == '!'){\r
+ if (len > 1 && s [1] == '='){\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ s = s.Substring (1);\r
+ return ! pp_primary (ref s);\r
+ } else\r
+ return pp_primary (ref s);\r
+ } else {\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ }\r
+ \r
+ bool pp_eq (ref string s)\r
+ {\r
+ bool va = pp_unary (ref s);\r
+\r
+ s = s.Trim ();\r
+ int len = s.Length;\r
+ if (len > 0){\r
+ if (s [0] == '='){\r
+ if (len > 2 && s [1] == '='){\r
+ s = s.Substring (2);\r
+ return va == pp_unary (ref s);\r
+ } else {\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ } else if (s [0] == '!' && len > 1 && s [1] == '='){\r
+ s = s.Substring (2);\r
+\r
+ return va != pp_unary (ref s);\r
+\r
+ }\r
+ }\r
+\r
+ return va;\r
+ \r
+ }\r
+ \r
+ bool pp_and (ref string s)\r
+ {\r
+ bool va = pp_eq (ref s);\r
+\r
+ s = s.Trim ();\r
+ int len = s.Length;\r
+ if (len > 0){\r
+ if (s [0] == '&'){\r
+ if (len > 2 && s [1] == '&'){\r
+ s = s.Substring (2);\r
+ return va && pp_eq (ref s);\r
+ } else {\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ } \r
+ }\r
+ return va;\r
+ }\r
+ \r
+ //\r
+ // Evaluates an expression for `#if' or `#elif'\r
+ //\r
+ bool pp_expr (ref string s)\r
+ {\r
+ bool va = pp_and (ref s);\r
+\r
+ s = s.Trim ();\r
+ int len = s.Length;\r
+ if (len > 0){\r
+ if (s [0] == '|'){\r
+ if (len > 2 && s [1] == '|'){\r
+ s = s.Substring (2);\r
+ return va || pp_and (ref s);\r
+ } else {\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ } else {\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ }\r
+\r
+ return va;\r
+ }\r
+\r
+ bool eval (string s)\r
+ {\r
+ bool v = pp_expr (ref s);\r
+\r
+ return v;\r
+ }\r
+ \r
+ void Error_InvalidDirective ()\r
+ {\r
+ Report.Error (1517, Location, "Invalid pre-processor directive");\r
+ }\r
+\r
+ void Error_UnexpectedDirective (string extra)\r
+ {\r
+ Report.Error (\r
+ 1028, Location,\r
+ "Unexpected processor directive (" + extra + ")");\r
+ }\r
+\r
+ void Error_TokensSeen ()\r
+ {\r
+ Report.Error (\r
+ 1032, Location,\r
+ "Cannot define or undefine pre-processor symbols after a token in the file");\r
+ }\r
+ \r
+ //\r
+ // if true, then the code continues processing the code\r
+ // if false, the code stays in a loop until another directive is\r
+ // reached.\r
+ //\r
+ bool handle_preprocessing_directive (bool caller_is_taking)\r
+ {\r
+ char [] blank = { ' ', '\t' };\r
+ string cmd, arg;\r
+ \r
+ get_cmd_arg (out cmd, out arg);\r
+\r
+ //\r
+ // The first group of pre-processing instructions is always processed\r
+ //\r
+ switch (cmd){\r
+ case "line":\r
+ if (!PreProcessLine (arg))\r
+ Report.Error (\r
+ 1576, Location,\r
+ "Argument to #line directive is missing or invalid");\r
+ return true;\r
+\r
+ case "region":\r
+ arg = "true";\r
+ goto case "if";\r
+\r
+ case "endregion":\r
+ goto case "endif";\r
+ \r
+ case "if":\r
+ if (arg == ""){\r
+ Error_InvalidDirective ();\r
+ return true;\r
+ }\r
+ bool taking = false;\r
+ if (ifstack == null)\r
+ ifstack = new Stack ();\r
+\r
+ if (ifstack.Count == 0){\r
+ taking = true;\r
+ } else {\r
+ int state = (int) ifstack.Peek ();\r
+ if ((state & TAKING) != 0)\r
+ taking = true;\r
+ }\r
+ \r
+ if (eval (arg) && taking){\r
+ ifstack.Push (TAKING | TAKEN_BEFORE | PARENT_TAKING);\r
+ return true;\r
+ } else {\r
+ ifstack.Push (taking ? PARENT_TAKING : 0);\r
+ return false;\r
+ }\r
+ \r
+ case "endif":\r
+ if (ifstack == null || ifstack.Count == 0){\r
+ Error_UnexpectedDirective ("no #if for this #endif");\r
+ return true;\r
+ } else {\r
+ ifstack.Pop ();\r
+ if (ifstack.Count == 0)\r
+ return true;\r
+ else {\r
+ int state = (int) ifstack.Peek ();\r
+\r
+ if ((state & TAKING) != 0)\r
+ return true;\r
+ else\r
+ return false;\r
+ }\r
+ }\r
+\r
+ case "elif":\r
+ if (ifstack == null || ifstack.Count == 0){\r
+ Error_UnexpectedDirective ("no #if for this #elif");\r
+ return true;\r
+ } else {\r
+ int state = (int) ifstack.Peek ();\r
+\r
+ if ((state & ELSE_SEEN) != 0){\r
+ Error_UnexpectedDirective ("#elif not valid after #else");\r
+ return true;\r
+ }\r
+\r
+ if ((state & (TAKEN_BEFORE | TAKING)) != 0)\r
+ return false;\r
+\r
+ if (eval (arg) && ((state & PARENT_TAKING) != 0)){\r
+ state = (int) ifstack.Pop ();\r
+ ifstack.Push (state | TAKING | TAKEN_BEFORE);\r
+ return true;\r
+ } else \r
+ return false;\r
+ }\r
+\r
+ case "else":\r
+ if (ifstack == null || ifstack.Count == 0){\r
+ Report.Error (\r
+ 1028, Location,\r
+ "Unexpected processor directive (no #if for this #else)");\r
+ return true;\r
+ } else {\r
+ int state = (int) ifstack.Peek ();\r
+\r
+ if ((state & ELSE_SEEN) != 0){\r
+ Error_UnexpectedDirective ("#else within #else");\r
+ return true;\r
+ }\r
+\r
+ ifstack.Pop ();\r
+ ifstack.Push (state | ELSE_SEEN);\r
+\r
+ if ((state & TAKEN_BEFORE) == 0){\r
+ if ((state & PARENT_TAKING) != 0)\r
+ return true;\r
+ else\r
+ return false;\r
+ }\r
+ return false;\r
+ }\r
+ }\r
+\r
+ //\r
+ // These are only processed if we are in a `taking' block\r
+ //\r
+ if (!caller_is_taking)\r
+ return false;\r
+ \r
+ switch (cmd){\r
+ case "define":\r
+ if (any_token_seen){\r
+ Error_TokensSeen ();\r
+ return true;\r
+ }\r
+ PreProcessDefinition (true, arg);\r
+ return true;\r
+\r
+ case "undef":\r
+ if (any_token_seen){\r
+ Error_TokensSeen ();\r
+ return true;\r
+ }\r
+ PreProcessDefinition (false, arg);\r
+ return true;\r
+\r
+ case "error":\r
+ Report.Error (1029, Location, "#error: '" + arg + "'");\r
+ return true;\r
+\r
+ case "warning":\r
+ Report.Warning (1030, Location, "#warning: '" + arg + "'");\r
+ return true;\r
+ }\r
+\r
+ Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");\r
+ return true;\r
+ }\r
\r
public int xtoken ()\r
{\r
int t;\r
- bool allow_keyword = false;\r
+ bool allow_keyword_as_ident = false;\r
bool doread = false;\r
int c;\r
\r
val = null;\r
+ // optimization: eliminate col and implement #directive semantic correctly.\r
for (;(c = getChar ()) != -1; col++) {\r
- \r
- if (is_identifier_start_character ((char) c)){\r
- System.Text.StringBuilder id = new System.Text.StringBuilder ();\r
+ if (Char.IsLetter ((char)c) || c == '_'){\r
string ids;\r
- \r
- id.Append ((char) c);\r
+\r
+ id_builder.Length = 0;\r
+ tokens_seen = true;\r
+ id_builder.Append ((char) c);\r
\r
while ((c = peekChar ()) != -1) {\r
if (is_identifier_part_character ((char) c)){\r
- id.Append ((char)getChar ());\r
+ id_builder.Append ((char)getChar ());\r
col++;\r
} else \r
break;\r
}\r
\r
- ids = id.ToString ();\r
- \r
- if (!is_keyword (ids)){\r
- val = id.ToString ();\r
- return Token.IDENTIFIER;\r
- }\r
- \r
- if (allow_keyword) {\r
+ ids = id_builder.ToString ();\r
+\r
+ if (!is_keyword (ids) || allow_keyword_as_ident) {\r
val = ids;\r
+ if (ids.Length > 512){\r
+ Report.Error (\r
+ 645, Location,\r
+ "Identifier too long (limit is 512 chars)");\r
+ }\r
+ allow_keyword_as_ident = false;\r
return Token.IDENTIFIER;\r
}\r
\r
- if (ids == "true")\r
- return Token.TRUE;\r
- else if (ids == "false")\r
- return Token.FALSE;\r
- else if (ids == "null")\r
- return Token.NULL;\r
- \r
- return getKeyword (ids);\r
+ // true, false and null are in the hash anyway.\r
+ return GetKeyword (ids);\r
+\r
}\r
\r
if (c == '.'){\r
- if (CharacterInfo.IsDigit ((char) peekChar ()))\r
+ tokens_seen = true;\r
+ if (Char.IsDigit ((char) peekChar ()))\r
return is_number (c);\r
return Token.DOT;\r
}\r
\r
- if (CharacterInfo.IsDigit ((char) c))\r
+ if (Char.IsDigit ((char) c)){\r
+ tokens_seen = true;\r
return is_number (c);\r
+ }\r
\r
// Handle double-slash comments.\r
if (c == '/'){\r
col++;\r
line++;\r
ref_line++;\r
+ col = 0;\r
+ any_token_seen |= tokens_seen;\r
+ tokens_seen = false;\r
continue;\r
} else if (d == '*'){\r
getChar ();\r
if (d == '\n'){\r
line++;\r
ref_line++;\r
+ col = 0;\r
+ any_token_seen |= tokens_seen;\r
+ tokens_seen = false;\r
}\r
- col++;\r
}\r
continue;\r
}\r
}\r
\r
/* For now, ignore pre-processor commands */\r
- if (col == 1 && c == '#'){\r
- System.Text.StringBuilder s = new System.Text.StringBuilder ();\r
+ // FIXME: In C# the '#' is not limited to appear\r
+ // on the first column.\r
+ if (c == '#' && !tokens_seen){\r
+ bool cont = true;\r
\r
- while ((c = getChar ()) != -1 && (c != '\n')){\r
- s.Append ((char) c);\r
- }\r
- if (String.Compare (s.ToString (), 0, "line", 0, 4) == 0){\r
- string arg = s.ToString ().Substring (5);\r
- int pos;\r
-\r
- if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){\r
- ref_line = System.Int32.Parse (arg.Substring (0, pos));\r
- pos++;\r
-\r
- char [] quotes = { '\"' };\r
+ start_again:\r
+ \r
+ cont = handle_preprocessing_directive (cont);\r
\r
- ref_name = arg.Substring (pos);\r
- ref_name.TrimStart (quotes);\r
- ref_name.TrimEnd (quotes);\r
- } else\r
- ref_line = System.Int32.Parse (arg);\r
+ if (cont){\r
+ col = 0;\r
+ continue;\r
}\r
- line++;\r
- ref_line++;\r
+ col = 1;\r
+\r
+ bool skipping = false;\r
+ for (;(c = getChar ()) != -1; col++){\r
+ if (c == '\n'){\r
+ col = 0;\r
+ line++;\r
+ ref_line++;\r
+ skipping = false;\r
+ } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r')\r
+ continue;\r
+ else if (c != '#')\r
+ skipping = true;\r
+ if (c == '#' && !skipping)\r
+ goto start_again;\r
+ }\r
+ any_token_seen |= tokens_seen;\r
+ tokens_seen = false;\r
+ if (c == -1)\r
+ Report.Error (1027, Location, "#endif expected");\r
continue;\r
}\r
\r
if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){\r
+ tokens_seen = true;\r
if (doread){\r
getChar ();\r
col++;\r
}\r
\r
if (c == '"'){\r
- System.Text.StringBuilder s = new System.Text.StringBuilder ();\r
-\r
+ string_builder.Length = 0;\r
+ \r
+ tokens_seen = true;\r
+ \r
while ((c = getChar ()) != -1){\r
if (c == '"'){\r
- val = s.ToString ();\r
+ if (allow_keyword_as_ident && peekChar () == '"'){\r
+ string_builder.Append ((char) c);\r
+ getChar ();\r
+ continue;\r
+ } \r
+ allow_keyword_as_ident = false;\r
+ val = string_builder.ToString ();\r
return Token.LITERAL_STRING;\r
}\r
\r
- c = escape (c);\r
- if (c == -1)\r
- return Token.ERROR;\r
- s.Append ((char) c);\r
+ if (!allow_keyword_as_ident){\r
+ c = escape (c);\r
+ if (c == -1)\r
+ return Token.ERROR;\r
+ }\r
+ string_builder.Append ((char) c);\r
}\r
}\r
\r
if (c == '\''){\r
c = getChar ();\r
+ tokens_seen = true;\r
if (c == '\''){\r
- error_details = "CS1011: Empty character literal";\r
+ error_details = "Empty character literal";\r
+ Report.Error (1011, Location, error_details);\r
return Token.ERROR;\r
}\r
c = escape (c);\r
val = new System.Char ();\r
val = (char) c;\r
c = getChar ();\r
+\r
if (c != '\''){\r
- error_details = "CS1012: Too many characters in character literal";\r
+ error_details = "Too many characters in character literal";\r
+ Report.Error (1012, Location, error_details);\r
+\r
// Try to recover, read until newline or next "'"\r
while ((c = getChar ()) != -1){\r
if (c == '\n' || c == '\'')\r
line++;\r
ref_line++;\r
col = 0;\r
+ any_token_seen |= tokens_seen;\r
+ tokens_seen = false;\r
continue;\r
}\r
+\r
if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r'){\r
if (c == '\t')\r
col = (((col + 8) / 8) * 8) - 1;\r
- \r
continue;\r
}\r
\r
if (c == '@'){\r
- allow_keyword = true;\r
+ tokens_seen = true;\r
+ allow_keyword_as_ident = true;\r
continue;\r
}\r
\r
return Token.ERROR;\r
}\r
\r
+ if (ifstack != null && ifstack.Count > 1)\r
+ Report.Error (1027, Location, "#endif expected");\r
return Token.EOF;\r
}\r
}\r
}\r
+\r
projects / mono.git / blobdiff