+// -*- coding: dos -*-\r
//\r
// cs-tokenizer.cs: The Tokenizer for the C# compiler\r
// This also implements the preprocessor\r
//\r
// Licensed under the terms of the GNU GPL\r
//\r
-// (C) 2001 Ximian, Inc (http://www.ximian.com)\r
+// (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)\r
//\r
\r
/*\r
using System.Collections;\r
using System.IO;\r
using System.Globalization;\r
+using System.Reflection;\r
\r
namespace Mono.CSharp\r
{\r
\r
public class Tokenizer : yyParser.yyInput\r
{\r
- StreamReader reader;\r
- public string ref_name;\r
+ SeekableStreamReader reader;\r
+ public SourceFile ref_name;\r
+ public SourceFile file_name;\r
public int ref_line = 1;\r
public int line = 1;\r
public int col = 1;\r
public int current_token;\r
bool handle_get_set = false;\r
bool handle_remove_add = false;\r
+ bool handle_assembly = false;\r
+\r
+ //\r
+ // Whether tokens have been seen on this line\r
+ //\r
+ bool tokens_seen = false;\r
+\r
+ //\r
+ // Whether a token has been seen on the file\r
+ // This is needed because `define' is not allowed to be used\r
+ // after a token has been seen.\r
+ //\r
+ bool any_token_seen = false;\r
+ static Hashtable tokenValues;\r
+ \r
+ private static Hashtable TokenValueName\r
+ {\r
+ get {\r
+ if (tokenValues == null)\r
+ tokenValues = GetTokenValueNameHash ();\r
+\r
+ return tokenValues;\r
+ }\r
+ }\r
+\r
+ private static Hashtable GetTokenValueNameHash ()\r
+ {\r
+ Type t = typeof (Token);\r
+ FieldInfo [] fields = t.GetFields ();\r
+ Hashtable hash = new Hashtable ();\r
+ foreach (FieldInfo field in fields) {\r
+ if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))\r
+ hash.Add (field.GetValue (null), field.Name);\r
+ }\r
+ return hash;\r
+ }\r
\r
//\r
// Returns a verbose representation of the current location\r
// return "Line: "+line+" Col: "+col + "\n" +\r
// "VirtLine: "+ref_line +\r
// " Token: "+current_token + " " + det;\r
-\r
- return ref_name + " " + "(" + line + "," + col + "), Token:" + current_token + " " + det;\r
+ string current_token_name = TokenValueName [current_token] as string;\r
+ if (current_token_name == null)\r
+ current_token_name = current_token.ToString ();\r
+\r
+ return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,\r
+ ref_line,\r
+ col,\r
+ current_token_name,\r
+ det);\r
}\r
}\r
\r
}\r
}\r
\r
+ public bool AssemblyTargetParsing {\r
+ get {\r
+ return handle_assembly;\r
+ }\r
+\r
+ set {\r
+ handle_assembly = value;\r
+ }\r
+ }\r
+\r
public bool EventParsing {\r
get {\r
return handle_remove_add;\r
//\r
// Class variables\r
// \r
- static Hashtable keywords;\r
+ static CharArrayHashtable[] keywords;\r
static NumberStyles styles;\r
static NumberFormatInfo csharp_format_info;\r
\r
//\r
// Values for the associated token returned\r
//\r
- System.Text.StringBuilder number;\r
int putback_char;\r
Object val;\r
\r
const int TAKEN_BEFORE = 2;\r
const int ELSE_SEEN = 4;\r
const int PARENT_TAKING = 8;\r
- \r
+ const int REGION = 16; \r
+\r
//\r
// pre-processor if stack state:\r
//\r
Stack ifstack;\r
+\r
+ static System.Text.StringBuilder string_builder;\r
+\r
+ const int max_id_size = 512;\r
+ static char [] id_builder = new char [max_id_size];\r
+\r
+ static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];\r
+\r
+ const int max_number_size = 128;\r
+ static char [] number_builder = new char [max_number_size];\r
+ static int number_pos;\r
\r
//\r
// Details about the error encoutered by the tokenizer\r
\r
public int Line {\r
get {\r
- return line;\r
+ return ref_line;\r
}\r
}\r
\r
return col;\r
}\r
}\r
- \r
+\r
+ static void AddKeyword (string kw, int token) {\r
+ if (keywords [kw.Length] == null) {\r
+ keywords [kw.Length] = new CharArrayHashtable (kw.Length);\r
+ }\r
+ keywords [kw.Length] [kw.ToCharArray ()] = token;\r
+ }\r
+\r
static void InitTokens ()\r
{\r
- keywords = new Hashtable ();\r
-\r
- keywords.Add ("abstract", Token.ABSTRACT);\r
- keywords.Add ("as", Token.AS);\r
- keywords.Add ("add", Token.ADD);\r
- keywords.Add ("base", Token.BASE);\r
- keywords.Add ("bool", Token.BOOL);\r
- keywords.Add ("break", Token.BREAK);\r
- keywords.Add ("byte", Token.BYTE);\r
- keywords.Add ("case", Token.CASE);\r
- keywords.Add ("catch", Token.CATCH);\r
- keywords.Add ("char", Token.CHAR);\r
- keywords.Add ("checked", Token.CHECKED);\r
- keywords.Add ("class", Token.CLASS);\r
- keywords.Add ("const", Token.CONST);\r
- keywords.Add ("continue", Token.CONTINUE);\r
- keywords.Add ("decimal", Token.DECIMAL);\r
- keywords.Add ("default", Token.DEFAULT);\r
- keywords.Add ("delegate", Token.DELEGATE);\r
- keywords.Add ("do", Token.DO);\r
- keywords.Add ("double", Token.DOUBLE);\r
- keywords.Add ("else", Token.ELSE);\r
- keywords.Add ("enum", Token.ENUM);\r
- keywords.Add ("event", Token.EVENT);\r
- keywords.Add ("explicit", Token.EXPLICIT);\r
- keywords.Add ("extern", Token.EXTERN);\r
- keywords.Add ("false", Token.FALSE);\r
- keywords.Add ("finally", Token.FINALLY);\r
- keywords.Add ("fixed", Token.FIXED);\r
- keywords.Add ("float", Token.FLOAT);\r
- keywords.Add ("for", Token.FOR);\r
- keywords.Add ("foreach", Token.FOREACH);\r
- keywords.Add ("goto", Token.GOTO);\r
- keywords.Add ("get", Token.GET);\r
- keywords.Add ("if", Token.IF);\r
- keywords.Add ("implicit", Token.IMPLICIT);\r
- keywords.Add ("in", Token.IN);\r
- keywords.Add ("int", Token.INT);\r
- keywords.Add ("interface", Token.INTERFACE);\r
- keywords.Add ("internal", Token.INTERNAL);\r
- keywords.Add ("is", Token.IS);\r
- keywords.Add ("lock", Token.LOCK);\r
- keywords.Add ("long", Token.LONG);\r
- keywords.Add ("namespace", Token.NAMESPACE);\r
- keywords.Add ("new", Token.NEW);\r
- keywords.Add ("null", Token.NULL);\r
- keywords.Add ("object", Token.OBJECT);\r
- keywords.Add ("operator", Token.OPERATOR);\r
- keywords.Add ("out", Token.OUT);\r
- keywords.Add ("override", Token.OVERRIDE);\r
- keywords.Add ("params", Token.PARAMS);\r
- keywords.Add ("private", Token.PRIVATE);\r
- keywords.Add ("protected", Token.PROTECTED);\r
- keywords.Add ("public", Token.PUBLIC);\r
- keywords.Add ("readonly", Token.READONLY);\r
- keywords.Add ("ref", Token.REF);\r
- keywords.Add ("remove", Token.REMOVE);\r
- keywords.Add ("return", Token.RETURN);\r
- keywords.Add ("sbyte", Token.SBYTE);\r
- keywords.Add ("sealed", Token.SEALED);\r
- keywords.Add ("set", Token.SET);\r
- keywords.Add ("short", Token.SHORT);\r
- keywords.Add ("sizeof", Token.SIZEOF);\r
- keywords.Add ("static", Token.STATIC);\r
- keywords.Add ("string", Token.STRING);\r
- keywords.Add ("struct", Token.STRUCT);\r
- keywords.Add ("switch", Token.SWITCH);\r
- keywords.Add ("this", Token.THIS);\r
- keywords.Add ("throw", Token.THROW);\r
- keywords.Add ("true", Token.TRUE);\r
- keywords.Add ("try", Token.TRY);\r
- keywords.Add ("typeof", Token.TYPEOF);\r
- keywords.Add ("uint", Token.UINT);\r
- keywords.Add ("ulong", Token.ULONG);\r
- keywords.Add ("unchecked", Token.UNCHECKED);\r
- keywords.Add ("unsafe", Token.UNSAFE);\r
- keywords.Add ("ushort", Token.USHORT);\r
- keywords.Add ("using", Token.USING);\r
- keywords.Add ("virtual", Token.VIRTUAL);\r
- keywords.Add ("void", Token.VOID);\r
- keywords.Add ("while", Token.WHILE);\r
+ keywords = new CharArrayHashtable [64];\r
+\r
+ AddKeyword ("__arglist", Token.ARGLIST);\r
+ AddKeyword ("abstract", Token.ABSTRACT);\r
+ AddKeyword ("as", Token.AS);\r
+ AddKeyword ("add", Token.ADD);\r
+ AddKeyword ("assembly", Token.ASSEMBLY);\r
+ AddKeyword ("base", Token.BASE);\r
+ AddKeyword ("bool", Token.BOOL);\r
+ AddKeyword ("break", Token.BREAK);\r
+ AddKeyword ("byte", Token.BYTE);\r
+ AddKeyword ("case", Token.CASE);\r
+ AddKeyword ("catch", Token.CATCH);\r
+ AddKeyword ("char", Token.CHAR);\r
+ AddKeyword ("checked", Token.CHECKED);\r
+ AddKeyword ("class", Token.CLASS);\r
+ AddKeyword ("const", Token.CONST);\r
+ AddKeyword ("continue", Token.CONTINUE);\r
+ AddKeyword ("decimal", Token.DECIMAL);\r
+ AddKeyword ("default", Token.DEFAULT);\r
+ AddKeyword ("delegate", Token.DELEGATE);\r
+ AddKeyword ("do", Token.DO);\r
+ AddKeyword ("double", Token.DOUBLE);\r
+ AddKeyword ("else", Token.ELSE);\r
+ AddKeyword ("enum", Token.ENUM);\r
+ AddKeyword ("event", Token.EVENT);\r
+ AddKeyword ("explicit", Token.EXPLICIT);\r
+ AddKeyword ("extern", Token.EXTERN);\r
+ AddKeyword ("false", Token.FALSE);\r
+ AddKeyword ("finally", Token.FINALLY);\r
+ AddKeyword ("fixed", Token.FIXED);\r
+ AddKeyword ("float", Token.FLOAT);\r
+ AddKeyword ("for", Token.FOR);\r
+ AddKeyword ("foreach", Token.FOREACH);\r
+ AddKeyword ("goto", Token.GOTO);\r
+ AddKeyword ("get", Token.GET);\r
+ AddKeyword ("if", Token.IF);\r
+ AddKeyword ("implicit", Token.IMPLICIT);\r
+ AddKeyword ("in", Token.IN);\r
+ AddKeyword ("int", Token.INT);\r
+ AddKeyword ("interface", Token.INTERFACE);\r
+ AddKeyword ("internal", Token.INTERNAL);\r
+ AddKeyword ("is", Token.IS);\r
+ AddKeyword ("lock", Token.LOCK);\r
+ AddKeyword ("long", Token.LONG);\r
+ AddKeyword ("namespace", Token.NAMESPACE);\r
+ AddKeyword ("new", Token.NEW);\r
+ AddKeyword ("null", Token.NULL);\r
+ AddKeyword ("object", Token.OBJECT);\r
+ AddKeyword ("operator", Token.OPERATOR);\r
+ AddKeyword ("out", Token.OUT);\r
+ AddKeyword ("override", Token.OVERRIDE);\r
+ AddKeyword ("params", Token.PARAMS);\r
+ AddKeyword ("private", Token.PRIVATE);\r
+ AddKeyword ("protected", Token.PROTECTED);\r
+ AddKeyword ("public", Token.PUBLIC);\r
+ AddKeyword ("readonly", Token.READONLY);\r
+ AddKeyword ("ref", Token.REF);\r
+ AddKeyword ("remove", Token.REMOVE);\r
+ AddKeyword ("return", Token.RETURN);\r
+ AddKeyword ("sbyte", Token.SBYTE);\r
+ AddKeyword ("sealed", Token.SEALED);\r
+ AddKeyword ("set", Token.SET);\r
+ AddKeyword ("short", Token.SHORT);\r
+ AddKeyword ("sizeof", Token.SIZEOF);\r
+ AddKeyword ("stackalloc", Token.STACKALLOC);\r
+ AddKeyword ("static", Token.STATIC);\r
+ AddKeyword ("string", Token.STRING);\r
+ AddKeyword ("struct", Token.STRUCT);\r
+ AddKeyword ("switch", Token.SWITCH);\r
+ AddKeyword ("this", Token.THIS);\r
+ AddKeyword ("throw", Token.THROW);\r
+ AddKeyword ("true", Token.TRUE);\r
+ AddKeyword ("try", Token.TRY);\r
+ AddKeyword ("typeof", Token.TYPEOF);\r
+ AddKeyword ("uint", Token.UINT);\r
+ AddKeyword ("ulong", Token.ULONG);\r
+ AddKeyword ("unchecked", Token.UNCHECKED);\r
+ AddKeyword ("unsafe", Token.UNSAFE);\r
+ AddKeyword ("ushort", Token.USHORT);\r
+ AddKeyword ("using", Token.USING);\r
+ AddKeyword ("virtual", Token.VIRTUAL);\r
+ AddKeyword ("void", Token.VOID);\r
+ AddKeyword ("volatile", Token.VOLATILE);\r
+ AddKeyword ("while", Token.WHILE);\r
}\r
\r
//\r
static Tokenizer ()\r
{\r
InitTokens ();\r
- csharp_format_info = new NumberFormatInfo ();\r
- csharp_format_info.CurrencyDecimalSeparator = ".";\r
- styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint;\r
+ csharp_format_info = NumberFormatInfo.InvariantInfo;\r
+ styles = NumberStyles.Float;\r
+ \r
+ string_builder = new System.Text.StringBuilder ();\r
}\r
\r
- bool is_keyword (string name)\r
+ int GetKeyword (char[] id, int id_len)\r
{\r
- bool res;\r
+ /*\r
+ * Keywords are stored in an array of hashtables grouped by their\r
+ * length.\r
+ */\r
+\r
+ if ((id_len >= keywords.Length) || (keywords [id_len] == null))\r
+ return -1;\r
+ object o = keywords [id_len] [id];\r
+\r
+ if (o == null)\r
+ return -1;\r
\r
- res = keywords.Contains (name);\r
- if (handle_get_set == false && (name == "get" || name == "set"))\r
- return false;\r
- if (handle_remove_add == false && (name == "remove" || name == "add"))\r
- return false;\r
- return res;\r
- }\r
+ int res = (int) o;\r
\r
- int GetKeyword (string name)\r
- {\r
- return (int) (keywords [name]);\r
+ if (handle_get_set == false && (res == Token.GET || res == Token.SET))\r
+ return -1;\r
+ if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))\r
+ return -1;\r
+ if (handle_assembly == false && res == Token.ASSEMBLY)\r
+ return -1;\r
+ return res;\r
+ \r
}\r
\r
public Location Location {\r
return new Location (ref_line);\r
}\r
}\r
+\r
+ void define (string def)\r
+ {\r
+ if (!RootContext.AllDefines.Contains (def)){\r
+ RootContext.AllDefines [def] = true;\r
+ }\r
+ if (defines.Contains (def))\r
+ return;\r
+ defines [def] = true;\r
+ }\r
\r
- public Tokenizer (System.IO.Stream input, string fname, ArrayList defs)\r
+ public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)\r
{\r
- this.ref_name = fname;\r
- reader = new System.IO.StreamReader (input);\r
+ this.ref_name = file;\r
+ this.file_name = file;\r
+ reader = input;\r
+ \r
putback_char = -1;\r
\r
if (defs != null){\r
defines = new Hashtable ();\r
foreach (string def in defs)\r
- defines [def] = true;\r
+ define (def);\r
}\r
\r
//\r
// FIXME: This could be `Location.Push' but we have to\r
// find out why the MS compiler allows this\r
//\r
- Mono.CSharp.Location.Push (fname);\r
+ Mono.CSharp.Location.Push (file);\r
}\r
\r
- bool is_identifier_start_character (char c)\r
+ public static void Cleanup () {\r
+ identifiers = null;\r
+ }\r
+\r
+ static bool is_identifier_start_character (char c)\r
{\r
- return Char.IsLetter (c) || c == '_' ;\r
+ return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c);\r
}\r
\r
- bool is_identifier_part_character (char c)\r
+ static bool is_identifier_part_character (char c)\r
+ {\r
+ return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);\r
+ }\r
+ \r
+ public static bool IsValidIdentifier (string s)\r
{\r
- return (Char.IsLetter (c) || Char.IsDigit (c) || c == '_');\r
+ if (s == null || s.Length == 0)\r
+ return false;\r
+ \r
+ if (!is_identifier_start_character (s [0]))\r
+ return false;\r
+ \r
+ for (int i = 1; i < s.Length; i ++)\r
+ if (! is_identifier_part_character (s [i]))\r
+ return false;\r
+ \r
+ return true;\r
}\r
\r
int is_punct (char c, ref bool doread)\r
return Token.CLOSE_BRACKET;\r
case '(':\r
return Token.OPEN_PARENS;\r
- case ')':\r
- return Token.CLOSE_PARENS;\r
+ case ')': {\r
+ if (deambiguate_close_parens == 0)\r
+ return Token.CLOSE_PARENS;\r
+\r
+ --deambiguate_close_parens;\r
+\r
+ // Save current position and parse next token.\r
+ int old = reader.Position;\r
+ int new_token = token ();\r
+ reader.Position = old;\r
+ putback_char = -1;\r
+\r
+ if (new_token == Token.OPEN_PARENS)\r
+ return Token.CLOSE_PARENS_OPEN_PARENS;\r
+ else if (new_token == Token.MINUS)\r
+ return Token.CLOSE_PARENS_MINUS;\r
+ else if (IsCastToken (new_token))\r
+ return Token.CLOSE_PARENS_CAST;\r
+ else\r
+ return Token.CLOSE_PARENS_NO_CAST;\r
+ }\r
+\r
case ',':\r
return Token.COMMA;\r
case ':':\r
else if (d == '=')\r
t = Token.OP_SUB_ASSIGN;\r
else if (d == '>')\r
- return Token.OP_PTR;\r
+ t = Token.OP_PTR;\r
else\r
return Token.MINUS;\r
doread = true;\r
return Token.ERROR;\r
}\r
\r
+ int deambiguate_close_parens = 0;\r
+\r
+ public void Deambiguate_CloseParens ()\r
+ {\r
+ putback (')');\r
+ deambiguate_close_parens++;\r
+ }\r
+\r
+ void Error_NumericConstantTooLong ()\r
+ {\r
+ Report.Error (1021, Location, "Numeric constant too long"); \r
+ }\r
+ \r
bool decimal_digits (int c)\r
{\r
int d;\r
bool seen_digits = false;\r
\r
- if (c != -1)\r
- number.Append ((char) c);\r
+ if (c != -1){\r
+ if (number_pos == max_number_size)\r
+ Error_NumericConstantTooLong ();\r
+ number_builder [number_pos++] = (char) c;\r
+ }\r
\r
- while ((d = peekChar ()) != -1){\r
- if (Char.IsDigit ((char)d)){\r
- number.Append ((char) d);\r
+ //\r
+ // We use peekChar2, because decimal_digits needs to do a \r
+ // 2-character look-ahead (5.ToString for example).\r
+ //\r
+ while ((d = peekChar2 ()) != -1){\r
+ if (d >= '0' && d <= '9'){\r
+ if (number_pos == max_number_size)\r
+ Error_NumericConstantTooLong ();\r
+ number_builder [number_pos++] = (char) d;\r
getChar ();\r
seen_digits = true;\r
} else\r
break;\r
}\r
+ \r
return seen_digits;\r
}\r
\r
+ bool is_hex (int e)\r
+ {\r
+ return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');\r
+ }\r
+ \r
void hex_digits (int c)\r
{\r
- int d;\r
-\r
if (c != -1)\r
- number.Append ((char) c);\r
- while ((d = peekChar ()) != -1){\r
- char e = Char.ToUpper ((char) d);\r
- \r
- if (Char.IsDigit (e) || (e >= 'A' && e <= 'F')){\r
- number.Append ((char) e);\r
- getChar ();\r
- } else\r
- break;\r
- }\r
+ number_builder [number_pos++] = (char) c;\r
+ \r
}\r
\r
int real_type_suffix (int c)\r
default:\r
return Token.NONE;\r
}\r
- getChar ();\r
return t;\r
}\r
\r
//\r
int adjust_int (int c)\r
{\r
- ulong ul = System.UInt64.Parse (number.ToString ());\r
- return integer_type_suffix (ul, c);\r
- }\r
+ try {\r
+ if (number_pos > 9){\r
+ ulong ul = (uint) (number_builder [0] - '0');\r
\r
+ for (int i = 1; i < number_pos; i++){\r
+ ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));\r
+ }\r
+ return integer_type_suffix (ul, c);\r
+ } else {\r
+ uint ui = (uint) (number_builder [0] - '0');\r
+\r
+ for (int i = 1; i < number_pos; i++){\r
+ ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));\r
+ }\r
+ return integer_type_suffix (ui, c);\r
+ }\r
+ } catch (OverflowException) {\r
+ error_details = "Integral constant is too large";\r
+ Report.Error (1021, Location, error_details);\r
+ val = 0ul;\r
+ return Token.LITERAL_INTEGER;\r
+ }\r
+ }\r
+ \r
int adjust_real (int t)\r
{\r
- string s = number.ToString ();\r
+ string s = new String (number_builder, 0, number_pos);\r
\r
switch (t){\r
case Token.LITERAL_DECIMAL:\r
- val = new System.Decimal ();\r
- val = System.Decimal.Parse (\r
- s, styles, csharp_format_info);\r
- break;\r
- case Token.LITERAL_DOUBLE:\r
- val = new System.Double ();\r
- val = System.Double.Parse (\r
- s, styles, csharp_format_info);\r
+ try {\r
+ val = System.Decimal.Parse (s, styles, csharp_format_info);\r
+ } catch (OverflowException) {\r
+ val = 0m; \r
+ error_details = "Floating-point constant is outside the range of the type 'decimal'";\r
+ Report.Error (594, Location, error_details);\r
+ }\r
break;\r
case Token.LITERAL_FLOAT:\r
- val = new System.Double ();\r
- val = (float) System.Double.Parse (\r
- s, styles, csharp_format_info);\r
+ try {\r
+ val = (float) System.Double.Parse (s, styles, csharp_format_info);\r
+ } catch (OverflowException) {\r
+ val = 0.0f; \r
+ error_details = "Floating-point constant is outside the range of the type 'float'";\r
+ Report.Error (594, Location, error_details);\r
+ }\r
break;\r
-\r
+ \r
+ case Token.LITERAL_DOUBLE:\r
case Token.NONE:\r
- val = new System.Double ();\r
- val = System.Double.Parse (\r
- s, styles, csharp_format_info);\r
t = Token.LITERAL_DOUBLE;\r
+ try {\r
+ val = System.Double.Parse (s, styles, csharp_format_info);\r
+ } catch (OverflowException) {\r
+ val = 0.0; \r
+ error_details = "Floating-point constant is outside the range of the type 'double'";\r
+ Report.Error (594, Location, error_details);\r
+ }\r
break;\r
}\r
return t;\r
}\r
\r
+ int handle_hex ()\r
+ {\r
+ int d;\r
+ ulong ul;\r
+ \r
+ getChar ();\r
+ while ((d = peekChar ()) != -1){\r
+ if (is_hex (d)){\r
+ number_builder [number_pos++] = (char) d;\r
+ getChar ();\r
+ } else\r
+ break;\r
+ }\r
+ \r
+ string s = new String (number_builder, 0, number_pos);\r
+ try {\r
+ if (number_pos <= 8)\r
+ ul = System.UInt32.Parse (s, NumberStyles.HexNumber);\r
+ else\r
+ ul = System.UInt64.Parse (s, NumberStyles.HexNumber);\r
+ } catch (OverflowException){\r
+ error_details = "Integral constant is too large";\r
+ Report.Error (1021, Location, error_details);\r
+ val = 0ul;\r
+ return Token.LITERAL_INTEGER;\r
+ }\r
+ \r
+ return integer_type_suffix (ul, peekChar ());\r
+ }\r
+\r
//\r
// Invoked if we know we have .digits or digits\r
//\r
int is_number (int c)\r
{\r
bool is_real = false;\r
- number = new System.Text.StringBuilder ();\r
int type;\r
\r
- number.Length = 0;\r
-\r
- if (Char.IsDigit ((char)c)){\r
- if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){\r
- ulong ul;\r
- getChar ();\r
- hex_digits (-1);\r
+ number_pos = 0;\r
\r
- string s = number.ToString ();\r
+ if (c >= '0' && c <= '9'){\r
+ if (c == '0'){\r
+ int peek = peekChar ();\r
\r
- ul = System.UInt64.Parse (s, NumberStyles.HexNumber);\r
- return integer_type_suffix (ul, peekChar ());\r
+ if (peek == 'x' || peek == 'X')\r
+ return handle_hex ();\r
}\r
decimal_digits (c);\r
c = getChar ();\r
if (c == '.'){\r
if (decimal_digits ('.')){\r
is_real = true;\r
- c = peekChar ();\r
+ c = getChar ();\r
} else {\r
putback ('.');\r
- number.Length -= 1;\r
+ number_pos--;\r
return adjust_int (-1);\r
}\r
}\r
\r
if (c == 'e' || c == 'E'){\r
is_real = true;\r
- number.Append ("e");\r
- getChar ();\r
+ if (number_pos == max_number_size)\r
+ Error_NumericConstantTooLong ();\r
+ number_builder [number_pos++] = 'e';\r
+ c = getChar ();\r
\r
- c = peekChar ();\r
if (c == '+'){\r
- number.Append ((char) c);\r
- getChar ();\r
- c = peekChar ();\r
- } else if (c == '-'){\r
- number.Append ((char) c);\r
- getChar ();\r
- c = peekChar ();\r
+ if (number_pos == max_number_size)\r
+ Error_NumericConstantTooLong ();\r
+ number_builder [number_pos++] = '+';\r
+ c = -1;\r
+ } else if (c == '-') {\r
+ if (number_pos == max_number_size)\r
+ Error_NumericConstantTooLong ();\r
+ number_builder [number_pos++] = '-';\r
+ c = -1;\r
+ } else {\r
+ if (number_pos == max_number_size)\r
+ Error_NumericConstantTooLong ();\r
+ number_builder [number_pos++] = '+';\r
}\r
- decimal_digits (-1);\r
- c = peekChar ();\r
+ \r
+ decimal_digits (c);\r
+ c = getChar ();\r
}\r
\r
type = real_type_suffix (c);\r
if (type == Token.NONE && !is_real){\r
putback (c);\r
return adjust_int (c);\r
- } else\r
+ } else \r
is_real = true;\r
\r
+ if (type == Token.NONE){\r
+ putback (c);\r
+ }\r
+ \r
if (is_real)\r
return adjust_real (type);\r
\r
Console.WriteLine ("This should not be reached");\r
throw new Exception ("Is Number should never reach this point");\r
}\r
+\r
+ //\r
+ // Accepts exactly count (4 or 8) hex, no more no less\r
+ //\r
+ int getHex (int count, out bool error)\r
+ {\r
+ int i;\r
+ int total = 0;\r
+ int c;\r
+ int top = count != -1 ? count : 4;\r
\r
+ getChar ();\r
+ error = false;\r
+ for (i = 0; i < top; i++){\r
+ c = getChar ();\r
+ \r
+ if (c >= '0' && c <= '9')\r
+ c = (int) c - (int) '0';\r
+ else if (c >= 'A' && c <= 'F')\r
+ c = (int) c - (int) 'A' + 10;\r
+ else if (c >= 'a' && c <= 'f')\r
+ c = (int) c - (int) 'a' + 10;\r
+ else {\r
+ error = true;\r
+ return 0;\r
+ }\r
+ \r
+ total = (total * 16) + c;\r
+ if (count == -1){\r
+ int p = peekChar ();\r
+ if (p == -1)\r
+ break;\r
+ if (!is_hex ((char)p))\r
+ break;\r
+ }\r
+ }\r
+ return total;\r
+ }\r
+\r
int escape (int c)\r
{\r
+ bool error;\r
int d;\r
int v;\r
\r
case 'v':\r
v = '\v'; break;\r
case 'r':\r
- v = 'c'; break;\r
+ v = '\r'; break;\r
case '\\':\r
v = '\\'; break;\r
case 'f':\r
v = '"'; break;\r
case '\'':\r
v = '\''; break;\r
+ case 'x':\r
+ v = getHex (-1, out error);\r
+ if (error)\r
+ goto default;\r
+ return v;\r
+ case 'u':\r
+ v = getHex (4, out error);\r
+ if (error)\r
+ goto default;\r
+ return v;\r
+ case 'U':\r
+ v = getHex (8, out error);\r
+ if (error)\r
+ goto default;\r
+ return v;\r
default:\r
- error_details = "cs1009: Unrecognized escape sequence " + (char)d;\r
- return -1;\r
+ Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);\r
+ return d;\r
}\r
getChar ();\r
return v;\r
{\r
if (putback_char != -1)\r
return putback_char;\r
- return reader.Peek ();\r
+ putback_char = reader.Read ();\r
+ return putback_char;\r
}\r
\r
- void putback (int c)\r
+ int peekChar2 ()\r
{\r
if (putback_char != -1)\r
+ return putback_char;\r
+ return reader.Peek ();\r
+ }\r
+ \r
+ void putback (int c)\r
+ {\r
+ if (putback_char != -1){\r
+ Console.WriteLine ("Col: " + col);\r
+ Console.WriteLine ("Row: " + line);\r
+ Console.WriteLine ("Name: " + ref_name.Name);\r
+ Console.WriteLine ("Current [{0}] putting back [{1}] ", putback_char, c);\r
throw new Exception ("This should not happen putback on putback");\r
+ }\r
putback_char = c;\r
}\r
\r
{\r
return val;\r
}\r
- \r
- public int token ()\r
+\r
+ bool IsCastToken (int token)\r
{\r
- current_token = xtoken ();\r
- return current_token;\r
+ switch (token) {\r
+ case Token.BANG:\r
+ case Token.TILDE:\r
+ case Token.IDENTIFIER:\r
+ case Token.LITERAL_INTEGER:\r
+ case Token.LITERAL_FLOAT:\r
+ case Token.LITERAL_DOUBLE:\r
+ case Token.LITERAL_DECIMAL:\r
+ case Token.LITERAL_CHARACTER:\r
+ case Token.LITERAL_STRING:\r
+ case Token.BASE:\r
+ case Token.CHECKED:\r
+ case Token.FALSE:\r
+ case Token.FIXED:\r
+ case Token.NEW:\r
+ case Token.NULL:\r
+ case Token.SIZEOF:\r
+ case Token.THIS:\r
+ case Token.THROW:\r
+ case Token.TRUE:\r
+ case Token.TYPEOF:\r
+ case Token.UNCHECKED:\r
+ case Token.UNSAFE:\r
+\r
+ //\r
+ // These can be part of a member access\r
+ //\r
+ case Token.INT:\r
+ case Token.UINT:\r
+ case Token.SHORT:\r
+ case Token.USHORT:\r
+ case Token.LONG:\r
+ case Token.ULONG:\r
+ case Token.DOUBLE:\r
+ case Token.FLOAT:\r
+ case Token.CHAR:\r
+ return true;\r
+\r
+ default:\r
+ return false;\r
+ }\r
}\r
\r
+ public int token ()\r
+ {\r
+ current_token = xtoken ();\r
+ return current_token;\r
+ }\r
+\r
static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();\r
\r
void get_cmd_arg (out string cmd, out string arg)\r
{\r
int c;\r
\r
+ tokens_seen = false;\r
arg = "";\r
static_cmd_arg.Length = 0;\r
- \r
- while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ')){\r
- if (c == '\r')\r
- continue;\r
- static_cmd_arg.Append ((char) c);\r
+\r
+ // skip over white space\r
+ while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))\r
+ ;\r
+\r
+\r
+ while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){\r
+ if (is_identifier_part_character ((char) c)){\r
+ static_cmd_arg.Append ((char) c);\r
+ c = getChar ();\r
+ } else {\r
+ putback (c);\r
+ break;\r
+ }\r
}\r
\r
cmd = static_cmd_arg.ToString ();\r
line++;\r
ref_line++;\r
return;\r
- }\r
+ } else if (c == '\r')\r
+ col = 0;\r
\r
// skip over white space\r
- while ((c = getChar ()) != -1 && (c != '\n') && (c == ' '))\r
+ while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))\r
;\r
\r
if (c == '\n'){\r
line++;\r
ref_line++;\r
return;\r
+ } else if (c == '\r'){\r
+ col = 0;\r
+ return;\r
}\r
\r
static_cmd_arg.Length = 0;\r
static_cmd_arg.Append ((char) c);\r
\r
- while ((c = getChar ()) != -1 && (c != '\n')){\r
- if (c == '\r')\r
- continue;\r
+ while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){\r
static_cmd_arg.Append ((char) c);\r
}\r
\r
if (c == '\n'){\r
line++;\r
ref_line++;\r
- }\r
+ } else if (c == '\r')\r
+ col = 0;\r
arg = static_cmd_arg.ToString ().Trim ();\r
}\r
\r
return false;\r
\r
if (arg == "default"){\r
- line = ref_line = line;\r
- return false;\r
+ ref_line = line;\r
+ ref_name = file_name;\r
+ Location.Push (ref_name);\r
+ return true;\r
+ } else if (arg == "hidden"){\r
+ //\r
+ // We ignore #line hidden\r
+ //\r
+ return true;\r
}\r
- \r
+\r
try {\r
int pos;\r
\r
\r
char [] quotes = { '\"' };\r
\r
- ref_name = arg.Substring (pos);\r
- ref_name.TrimStart (quotes);\r
- ref_name.TrimEnd (quotes);\r
+ string name = arg.Substring (pos). Trim (quotes);\r
+ ref_name = Location.LookupFile (name);\r
+ file_name.HasLineDirective = true;\r
+ ref_name.HasLineDirective = true;\r
+ Location.Push (ref_name);\r
} else {\r
ref_line = System.Int32.Parse (arg);\r
}\r
void PreProcessDefinition (bool is_define, string arg)\r
{\r
if (arg == "" || arg == "true" || arg == "false"){\r
- Report.Error(1001, Location, "Missing identifer to pre-processor directive");\r
+ Report.Error (1001, Location, "Missing identifer to pre-processor directive");\r
return;\r
}\r
\r
+ char[] whitespace = { ' ', '\t' };\r
+ if (arg.IndexOfAny (whitespace) != -1){\r
+ Report.Error (1025, Location, "Single-line comment or end-of-line expected");\r
+ return;\r
+ }\r
+\r
+ if (!is_identifier_start_character (arg [0]))\r
+ Report.Error (1001, Location, "Identifier expected: " + arg);\r
+ \r
+ foreach (char c in arg.Substring (1)){\r
+ if (!is_identifier_part_character (c)){\r
+ Report.Error (1001, Location, "Identifier expected: " + arg);\r
+ return;\r
+ }\r
+ }\r
+\r
if (is_define){\r
if (defines == null)\r
defines = new Hashtable ();\r
- defines [arg] = 1;\r
+ define (arg);\r
} else {\r
if (defines == null)\r
return;\r
\r
return false;\r
}\r
+\r
+ bool pp_primary (ref string s)\r
+ {\r
+ s = s.Trim ();\r
+ int len = s.Length;\r
+\r
+ if (len > 0){\r
+ char c = s [0];\r
+ \r
+ if (c == '('){\r
+ s = s.Substring (1);\r
+ bool val = pp_expr (ref s);\r
+ if (s.Length > 0 && s [0] == ')'){\r
+ s = s.Substring (1);\r
+ return val;\r
+ }\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ \r
+ if (is_identifier_start_character (c)){\r
+ int j = 1;\r
+\r
+ while (j < len){\r
+ c = s [j];\r
+ \r
+ if (is_identifier_part_character (c)){\r
+ j++;\r
+ continue;\r
+ }\r
+ bool v = eval_val (s.Substring (0, j));\r
+ s = s.Substring (j);\r
+ return v;\r
+ }\r
+ bool vv = eval_val (s);\r
+ s = "";\r
+ return vv;\r
+ }\r
+ }\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ \r
+ bool pp_unary (ref string s)\r
+ {\r
+ s = s.Trim ();\r
+ int len = s.Length;\r
+\r
+ if (len > 0){\r
+ if (s [0] == '!'){\r
+ if (len > 1 && s [1] == '='){\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ s = s.Substring (1);\r
+ return ! pp_primary (ref s);\r
+ } else\r
+ return pp_primary (ref s);\r
+ } else {\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ }\r
+ \r
+ bool pp_eq (ref string s)\r
+ {\r
+ bool va = pp_unary (ref s);\r
+\r
+ s = s.Trim ();\r
+ int len = s.Length;\r
+ if (len > 0){\r
+ if (s [0] == '='){\r
+ if (len > 2 && s [1] == '='){\r
+ s = s.Substring (2);\r
+ return va == pp_unary (ref s);\r
+ } else {\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ } else if (s [0] == '!' && len > 1 && s [1] == '='){\r
+ s = s.Substring (2);\r
+\r
+ return va != pp_unary (ref s);\r
+\r
+ } \r
+ }\r
+\r
+ return va;\r
+ \r
+ }\r
+ \r
+ bool pp_and (ref string s)\r
+ {\r
+ bool va = pp_eq (ref s);\r
+\r
+ s = s.Trim ();\r
+ int len = s.Length;\r
+ if (len > 0){\r
+ if (s [0] == '&'){\r
+ if (len > 2 && s [1] == '&'){\r
+ s = s.Substring (2);\r
+ return (va & pp_eq (ref s));\r
+ } else {\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ } \r
+ }\r
+ return va;\r
+ }\r
\r
//\r
// Evaluates an expression for `#if' or `#elif'\r
//\r
+ bool pp_expr (ref string s)\r
+ {\r
+ bool va = pp_and (ref s);\r
+ s = s.Trim ();\r
+ int len = s.Length;\r
+ if (len > 0){\r
+ char c = s [0];\r
+ \r
+ if (c == '|'){\r
+ if (len > 2 && s [1] == '|'){\r
+ s = s.Substring (2);\r
+ return va | pp_expr (ref s);\r
+ } else {\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+ } \r
+ }\r
+ \r
+ return va;\r
+ }\r
+\r
bool eval (string s)\r
{\r
- return eval_val (s);\r
+ bool v = pp_expr (ref s);\r
+ s = s.Trim ();\r
+ if (s.Length != 0){\r
+ Error_InvalidDirective ();\r
+ return false;\r
+ }\r
+\r
+ return v;\r
+ }\r
+ \r
+ void Error_InvalidDirective ()\r
+ {\r
+ Report.Error (1517, Location, "Invalid pre-processor directive");\r
}\r
\r
- void report1028 (string extra)\r
+ void Error_UnexpectedDirective (string extra)\r
{\r
Report.Error (\r
1028, Location,\r
"Unexpected processor directive (" + extra + ")");\r
}\r
+\r
+ void Error_TokensSeen ()\r
+ {\r
+ Report.Error (\r
+ 1032, Location,\r
+ "Cannot define or undefine pre-processor symbols after a token in the file");\r
+ }\r
\r
//\r
// if true, then the code continues processing the code\r
// if false, the code stays in a loop until another directive is\r
// reached.\r
//\r
- bool handle_preprocessing_directive ()\r
+ bool handle_preprocessing_directive (bool caller_is_taking)\r
{\r
- char [] blank = { ' ', '\t' };\r
string cmd, arg;\r
- \r
+ bool region_directive = false;\r
+\r
get_cmd_arg (out cmd, out arg);\r
\r
+ // Eat any trailing whitespaces and single-line comments\r
+ if (arg.IndexOf ("//") != -1)\r
+ arg = arg.Substring (0, arg.IndexOf ("//"));\r
+ arg = arg.TrimEnd (' ', '\t');\r
+\r
+ //\r
+ // The first group of pre-processing instructions is always processed\r
+ //\r
switch (cmd){\r
+ case "pragma":\r
+ if (RootContext.V2)\r
+ return true;\r
+ break;\r
+ \r
case "line":\r
if (!PreProcessLine (arg))\r
Report.Error (\r
"Argument to #line directive is missing or invalid");\r
return true;\r
\r
- case "define":\r
- PreProcessDefinition (true, arg);\r
- return true;\r
-\r
- case "undef":\r
- PreProcessDefinition (false, arg);\r
- return true;\r
-\r
- case "error":\r
- Report.Error (1029, Location, "#error: '" + arg + "'");\r
- return true;\r
-\r
- case "warning":\r
- Report.Warning (1030, Location, "#warning: '" + arg + "'");\r
- return true;\r
-\r
case "region":\r
+ region_directive = true;\r
arg = "true";\r
goto case "if";\r
\r
case "endregion":\r
+ region_directive = true;\r
goto case "endif";\r
\r
case "if":\r
if (arg == ""){\r
- Report.Error (1517, Location, "Invalid pre-processor directive");\r
+ Error_InvalidDirective ();\r
return true;\r
}\r
bool taking = false;\r
if ((state & TAKING) != 0)\r
taking = true;\r
}\r
- \r
+\r
if (eval (arg) && taking){\r
- ifstack.Push (TAKING | TAKEN_BEFORE | PARENT_TAKING);\r
+ int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;\r
+ if (region_directive)\r
+ push |= REGION;\r
+ ifstack.Push (push);\r
return true;\r
} else {\r
- ifstack.Push (taking ? PARENT_TAKING : 0);\r
+ int push = (taking ? PARENT_TAKING : 0);\r
+ if (region_directive)\r
+ push |= REGION;\r
+ ifstack.Push (push);\r
return false;\r
}\r
\r
case "endif":\r
if (ifstack == null || ifstack.Count == 0){\r
- report1028 ("no #if for this #endif");\r
+ Error_UnexpectedDirective ("no #if for this #endif");\r
return true;\r
} else {\r
- ifstack.Pop ();\r
+ int pop = (int) ifstack.Pop ();\r
+ \r
+ if (region_directive && ((pop & REGION) == 0))\r
+ Report.Error (1027, Location, "#endif directive expected");\r
+ else if (!region_directive && ((pop & REGION) != 0))\r
+ Report.Error (1038, Location, "#endregion directive expected");\r
+ \r
if (ifstack.Count == 0)\r
return true;\r
else {\r
\r
case "elif":\r
if (ifstack == null || ifstack.Count == 0){\r
- report1028 ("no #if for this #elif");\r
+ Error_UnexpectedDirective ("no #if for this #elif");\r
return true;\r
} else {\r
int state = (int) ifstack.Peek ();\r
\r
+ if ((state & REGION) != 0) {\r
+ Report.Error (1038, Location, "#endregion directive expected");\r
+ return true;\r
+ }\r
+\r
if ((state & ELSE_SEEN) != 0){\r
- report1028 ("#elif not valid after #else");\r
+ Error_UnexpectedDirective ("#elif not valid after #else");\r
return true;\r
}\r
\r
} else {\r
int state = (int) ifstack.Peek ();\r
\r
+ if ((state & REGION) != 0) {\r
+ Report.Error (1038, Location, "#endregion directive expected");\r
+ return true;\r
+ }\r
+\r
if ((state & ELSE_SEEN) != 0){\r
- report1028 ("#else within #else");\r
+ Error_UnexpectedDirective ("#else within #else");\r
return true;\r
}\r
\r
ifstack.Pop ();\r
- ifstack.Push (state | ELSE_SEEN);\r
\r
+ bool ret;\r
if ((state & TAKEN_BEFORE) == 0){\r
- if ((state & PARENT_TAKING) != 0)\r
- return true;\r
- else\r
- return false;\r
+ ret = ((state & PARENT_TAKING) != 0);\r
+ } else\r
+ ret = false;\r
+ \r
+ if (ret)\r
+ state |= TAKING;\r
+ else\r
+ state &= ~TAKING;\r
+ \r
+ ifstack.Push (state | ELSE_SEEN);\r
+ \r
+ return ret;\r
+ }\r
+ }\r
+\r
+ //\r
+ // These are only processed if we are in a `taking' block\r
+ //\r
+ if (!caller_is_taking)\r
+ return false;\r
+ \r
+ switch (cmd){\r
+ case "define":\r
+ if (any_token_seen){\r
+ Error_TokensSeen ();\r
+ return true;\r
+ }\r
+ PreProcessDefinition (true, arg);\r
+ return true;\r
+\r
+ case "undef":\r
+ if (any_token_seen){\r
+ Error_TokensSeen ();\r
+ return true;\r
+ }\r
+ PreProcessDefinition (false, arg);\r
+ return true;\r
+\r
+ case "error":\r
+ Report.Error (1029, Location, "#error: '" + arg + "'");\r
+ return true;\r
+\r
+ case "warning":\r
+ Report.Warning (1030, Location, "#warning: '" + arg + "'");\r
+ return true;\r
+ }\r
+\r
+ Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");\r
+ return true;\r
+\r
+ }\r
+\r
+ private int consume_string (bool quoted) \r
+ {\r
+ int c;\r
+ string_builder.Length = 0;\r
+ \r
+ while ((c = getChar ()) != -1){\r
+ if (c == '"'){\r
+ if (quoted && peekChar () == '"'){\r
+ string_builder.Append ((char) c);\r
+ getChar ();\r
+ continue;\r
+ } else {\r
+ val = string_builder.ToString ();\r
+ return Token.LITERAL_STRING;\r
}\r
- return false;\r
}\r
+\r
+ if (c == '\n'){\r
+ if (!quoted)\r
+ Report.Error (1010, Location, "Newline in constant");\r
+ line++;\r
+ ref_line++;\r
+ col = 0;\r
+ } else\r
+ col++;\r
+\r
+ if (!quoted){\r
+ c = escape (c);\r
+ if (c == -1)\r
+ return Token.ERROR;\r
+ }\r
+ string_builder.Append ((char) c);\r
}\r
+\r
+ Report.Error (1039, Location, "Unterminated string literal");\r
+ return Token.EOF;\r
+ }\r
+\r
+ private int consume_identifier (int s, bool quoted) \r
+ {\r
+ int pos = 1;\r
+ int c;\r
\r
- Report.Error (1024, "Preprocessor directive expected (got: " + cmd + ")");\r
- return true;\r
+ id_builder [0] = (char) s;\r
+ \r
+ while ((c = reader.Read ()) != -1) {\r
+ if (is_identifier_part_character ((char) c)){\r
+ if (pos == max_id_size){\r
+ Report.Error (645, Location, "Identifier too long (limit is 512 chars)");\r
+ return Token.ERROR;\r
+ }\r
+ \r
+ id_builder [pos++] = (char) c;\r
+ putback_char = -1;\r
+ col++;\r
+ } else {\r
+ putback_char = c;\r
+ break;\r
+ }\r
+ }\r
+\r
+ //\r
+ // Optimization: avoids doing the keyword lookup\r
+ // on uppercase letters and _\r
+ //\r
+ if (s >= 'a' || s == '_'){\r
+ int keyword = GetKeyword (id_builder, pos);\r
+ if (keyword != -1 && !quoted)\r
+ return keyword;\r
+ }\r
+\r
+ //\r
+ // Keep identifiers in an array of hashtables to avoid needless\r
+ // allocations\r
+ //\r
+\r
+ if (identifiers [pos] != null) {\r
+ val = identifiers [pos][id_builder];\r
+ if (val != null) {\r
+ return Token.IDENTIFIER;\r
+ }\r
+ }\r
+ else\r
+ identifiers [pos] = new CharArrayHashtable (pos);\r
+\r
+ val = new String (id_builder, 0, pos);\r
+\r
+ char [] chars = new char [pos];\r
+ Array.Copy (id_builder, chars, pos);\r
+\r
+ identifiers [pos] [chars] = val;\r
+\r
+ return Token.IDENTIFIER;\r
}\r
\r
public int xtoken ()\r
{\r
int t;\r
- bool allow_keyword_as_ident = false;\r
bool doread = false;\r
int c;\r
\r
val = null;\r
// optimization: eliminate col and implement #directive semantic correctly.\r
for (;(c = getChar ()) != -1; col++) {\r
- if (is_identifier_start_character ((char) c)){\r
- System.Text.StringBuilder id = new System.Text.StringBuilder ();\r
- string ids;\r
- \r
- id.Append ((char) c);\r
- \r
- while ((c = peekChar ()) != -1) {\r
- if (is_identifier_part_character ((char) c)){\r
- id.Append ((char)getChar ());\r
- col++;\r
- } else \r
- break;\r
- }\r
- \r
- ids = id.ToString ();\r
-\r
- if (!is_keyword (ids) || allow_keyword_as_ident) {\r
- val = ids;\r
- return Token.IDENTIFIER;\r
- }\r
-\r
- // true, false and null are in the hash anyway.\r
- return GetKeyword (ids);\r
-\r
+ if (c == ' ')\r
+ continue;\r
+ \r
+ if (c == '\t') {\r
+ col = (((col + 8) / 8) * 8) - 1;\r
+ continue;\r
}\r
+ \r
+ if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0)\r
+ continue;\r
\r
- if (c == '.'){\r
- if (Char.IsDigit ((char) peekChar ()))\r
- return is_number (c);\r
- return Token.DOT;\r
+ if (c == '\r') {\r
+ if (peekChar () == '\n')\r
+ getChar ();\r
+\r
+ line++;\r
+ ref_line++;\r
+ col = 0;\r
+ any_token_seen |= tokens_seen;\r
+ tokens_seen = false;\r
+ continue;\r
}\r
- \r
- if (Char.IsDigit ((char) c))\r
- return is_number (c);\r
\r
// Handle double-slash comments.\r
if (c == '/'){\r
\r
if (d == '/'){\r
getChar ();\r
- while ((d = getChar ()) != -1 && (d != '\n'))\r
+ while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')\r
col++;\r
- line++;\r
- ref_line++;\r
- col = 0;\r
+ if (d == '\n'){\r
+ line++;\r
+ ref_line++;\r
+ col = 0;\r
+ }\r
+ any_token_seen |= tokens_seen;\r
+ tokens_seen = false;\r
continue;\r
} else if (d == '*'){\r
getChar ();\r
line++;\r
ref_line++;\r
col = 0;\r
+ any_token_seen |= tokens_seen;\r
+ tokens_seen = false;\r
}\r
}\r
continue;\r
}\r
+ goto is_punct_label;\r
+ }\r
+\r
+ \r
+ if (is_identifier_start_character ((char)c)){\r
+ tokens_seen = true;\r
+ return consume_identifier (c, false);\r
}\r
\r
+ is_punct_label:\r
+ if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){\r
+ tokens_seen = true;\r
+ if (doread){\r
+ getChar ();\r
+ col++;\r
+ }\r
+ return t;\r
+ }\r
+\r
+ // white space\r
+ if (c == '\n'){\r
+ line++;\r
+ ref_line++;\r
+ col = 0;\r
+ any_token_seen |= tokens_seen;\r
+ tokens_seen = false;\r
+ continue;\r
+ }\r
+\r
+ if (c >= '0' && c <= '9'){\r
+ tokens_seen = true;\r
+ return is_number (c);\r
+ }\r
+\r
+ if (c == '.'){\r
+ tokens_seen = true;\r
+ int peek = peekChar ();\r
+ if (peek >= '0' && peek <= '9')\r
+ return is_number (c);\r
+ return Token.DOT;\r
+ }\r
+ \r
/* For now, ignore pre-processor commands */\r
// FIXME: In C# the '#' is not limited to appear\r
// on the first column.\r
- if (col <= 1 && c == '#'){\r
+ if (c == '#' && !tokens_seen){\r
+ bool cont = true;\r
+ \r
start_again:\r
\r
- bool cont = handle_preprocessing_directive ();\r
+ cont = handle_preprocessing_directive (cont);\r
\r
if (cont){\r
col = 0;\r
}\r
col = 1;\r
\r
+ bool skipping = false;\r
for (;(c = getChar ()) != -1; col++){\r
if (c == '\n'){\r
col = 0;\r
line++;\r
ref_line++;\r
- } else if (col == 1 && c == '#'){\r
+ skipping = false;\r
+ } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)\r
+ continue;\r
+ else if (c != '#')\r
+ skipping = true;\r
+ if (c == '#' && !skipping)\r
goto start_again;\r
- }\r
}\r
+ any_token_seen |= tokens_seen;\r
+ tokens_seen = false;\r
if (c == -1)\r
- Report.Error (1027, Location, "#endif expected");\r
+ Report.Error (1027, Location, "#endif/#endregion expected");\r
continue;\r
}\r
\r
- if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){\r
- if (doread){\r
- getChar ();\r
- col++;\r
- }\r
- return t;\r
- }\r
- \r
- if (c == '"'){\r
- System.Text.StringBuilder s = new System.Text.StringBuilder ();\r
-\r
- while ((c = getChar ()) != -1){\r
- if (c == '"'){\r
- val = s.ToString ();\r
- return Token.LITERAL_STRING;\r
- }\r
-\r
- c = escape (c);\r
- if (c == -1)\r
- return Token.ERROR;\r
- s.Append ((char) c);\r
- }\r
- }\r
+ if (c == '"') \r
+ return consume_string (false);\r
\r
if (c == '\''){\r
c = getChar ();\r
+ tokens_seen = true;\r
if (c == '\''){\r
- error_details = "CS1011: Empty character literal";\r
+ error_details = "Empty character literal";\r
+ Report.Error (1011, Location, error_details);\r
return Token.ERROR;\r
}\r
c = escape (c);\r
val = new System.Char ();\r
val = (char) c;\r
c = getChar ();\r
+\r
if (c != '\''){\r
- error_details = "CS1012: Too many characters in character literal";\r
+ error_details = "Too many characters in character literal";\r
+ Report.Error (1012, Location, error_details);\r
+\r
// Try to recover, read until newline or next "'"\r
while ((c = getChar ()) != -1){\r
- if (c == '\n' || c == '\'')\r
+ if (c == '\n' || c == '\''){\r
+ line++;\r
+ ref_line++;\r
+ col = 0;\r
break;\r
+ } else\r
+ col++;\r
\r
}\r
return Token.ERROR;\r
return Token.LITERAL_CHARACTER;\r
}\r
\r
- // white space\r
- if (c == '\n'){\r
- line++;\r
- ref_line++;\r
- col = 0;\r
- continue;\r
- }\r
- if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r'){\r
- if (c == '\t')\r
- col = (((col + 8) / 8) * 8) - 1;\r
- \r
- continue;\r
+ if (c == '@') {\r
+ c = getChar ();\r
+ if (c == '"') {\r
+ tokens_seen = true;\r
+ return consume_string (true);\r
+ } else if (is_identifier_start_character ((char) c)){\r
+ return consume_identifier (c, true);\r
+ } else {\r
+ Report.Error (1033, Location, "'@' must be followed by string constant or identifier");\r
+ }\r
}\r
\r
- if (c == '@'){\r
- allow_keyword_as_ident = true;\r
- continue;\r
+ if (c == '#') {\r
+ error_details = "Preprocessor directives must appear as the first non-whitespace " +\r
+ "character on a line.";\r
+\r
+ Report.Error (1040, Location, error_details);\r
+\r
+ return Token.ERROR;\r
}\r
\r
error_details = ((char)c).ToString ();\r
return Token.ERROR;\r
}\r
\r
- if (ifstack != null && ifstack.Count > 1)\r
- Report.Error (1027, Location, "#endif expected");\r
return Token.EOF;\r
}\r
+\r
+ public void cleanup ()\r
+ {\r
+ if (ifstack != null && ifstack.Count >= 1) {\r
+ int state = (int) ifstack.Pop ();\r
+ if ((state & REGION) != 0)\r
+ Report.Error (1038, "#endregion directive expected");\r
+ else \r
+ Report.Error (1027, "#endif directive expected");\r
+ }\r
+ \r
+ }\r
}\r
}\r
\r