//\r
// Licensed under the terms of the GNU GPL\r
//\r
-// (C) 2001 Ximian, Inc (http://www.ximian.com)\r
+// (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)\r
//\r
\r
/*\r
public class Tokenizer : yyParser.yyInput\r
{\r
StreamReader reader;\r
- public string ref_name;\r
+ public SourceFile ref_name;\r
+ public SourceFile file_name;\r
public int ref_line = 1;\r
public int line = 1;\r
public int col = 1;\r
if (current_token_name == null)\r
current_token_name = current_token.ToString ();\r
\r
- return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name,\r
- line,\r
+ return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,\r
+ ref_line,\r
col,\r
current_token_name,\r
det);\r
const int TAKEN_BEFORE = 2;\r
const int ELSE_SEEN = 4;\r
const int PARENT_TAKING = 8;\r
- \r
+ const int REGION = 16; \r
+\r
//\r
// pre-processor if stack state:\r
//\r
Stack ifstack;\r
\r
- static System.Text.StringBuilder id_builder;\r
static System.Text.StringBuilder string_builder;\r
- static System.Text.StringBuilder number_builder;\r
+\r
+ const int max_id_size = 512;\r
+ static char [] id_builder = new char [max_id_size];\r
+\r
+ const int max_number_size = 128;\r
+ static char [] number_builder = new char [max_number_size];\r
+ static int number_pos;\r
\r
//\r
// Details about the error encoutered by the tokenizer\r
\r
public int Line {\r
get {\r
- return line;\r
+ return ref_line;\r
}\r
}\r
\r
return col;\r
}\r
}\r
- \r
+\r
static void InitTokens ()\r
{\r
keywords = new Hashtable ();\r
keywords.Add ("void", Token.VOID);\r
keywords.Add ("volatile", Token.VOLATILE);\r
keywords.Add ("while", Token.WHILE);\r
+\r
+ if (RootContext.V2){\r
+ keywords.Add ("__yield", Token.YIELD);\r
+ keywords.Add ("yield", Token.YIELD);\r
+ }\r
}\r
\r
//\r
csharp_format_info = NumberFormatInfo.InvariantInfo;\r
styles = NumberStyles.Float;\r
\r
- id_builder = new System.Text.StringBuilder ();\r
string_builder = new System.Text.StringBuilder ();\r
- number_builder = new System.Text.StringBuilder ();\r
}\r
\r
int GetKeyword (string name)\r
return -1;\r
\r
int res = (int) o;\r
- \r
+\r
if (handle_get_set == false && (res == Token.GET || res == Token.SET))\r
return -1;\r
if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))\r
defines [def] = true;\r
}\r
\r
- public Tokenizer (StreamReader input, string fname, ArrayList defs)\r
+ public Tokenizer (StreamReader input, SourceFile file, ArrayList defs)\r
{\r
- this.ref_name = fname;\r
+ this.ref_name = file;\r
+ this.file_name = file;\r
reader = input;\r
\r
putback_char = -1;\r
// FIXME: This could be `Location.Push' but we have to\r
// find out why the MS compiler allows this\r
//\r
- Mono.CSharp.Location.Push (fname);\r
+ Mono.CSharp.Location.Push (file);\r
}\r
\r
bool is_identifier_start_character (char c)\r
return Token.ERROR;\r
}\r
\r
+ void Error_NumericConstantTooLong ()\r
+ {\r
+ Report.Error (1021, Location, "Numeric constant too long"); \r
+ }\r
+ \r
bool decimal_digits (int c)\r
{\r
int d;\r
bool seen_digits = false;\r
\r
- if (c != -1)\r
- number_builder.Append ((char) c);\r
+ if (c != -1){\r
+ if (number_pos == max_number_size)\r
+ Error_NumericConstantTooLong ();\r
+ number_builder [number_pos++] = (char) c;\r
+ }\r
\r
- while ((d = peekChar ()) != -1){\r
+ //\r
+ // We use peekChar2, because decimal_digits needs to do a \r
+ // 2-character look-ahead (5.ToString for example).\r
+ //\r
+ while ((d = peekChar2 ()) != -1){\r
if (d >= '0' && d <= '9'){\r
- number_builder.Append ((char) d);\r
+ if (number_pos == max_number_size)\r
+ Error_NumericConstantTooLong ();\r
+ number_builder [number_pos++] = (char) d;\r
getChar ();\r
seen_digits = true;\r
} else\r
\r
void hex_digits (int c)\r
{\r
- int d;\r
-\r
if (c != -1)\r
- number_builder.Append ((char) c);\r
- while ((d = peekChar ()) != -1){\r
- if (is_hex (d)){\r
- number_builder.Append ((char) d);\r
- getChar ();\r
- } else\r
- break;\r
- }\r
+ number_builder [number_pos++] = (char) c;\r
+ \r
}\r
\r
int real_type_suffix (int c)\r
int adjust_int (int c)\r
{\r
try {\r
- ulong ul = System.UInt64.Parse (number_builder.ToString ());\r
- return integer_type_suffix (ul, c);\r
+ if (number_pos > 9){\r
+ ulong ul = (uint) (number_builder [0] - '0');\r
+\r
+ for (int i = 1; i < number_pos; i++){\r
+ ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));\r
+ }\r
+ return integer_type_suffix (ul, c);\r
+ } else {\r
+ uint ui = (uint) (number_builder [0] - '0');\r
+\r
+ for (int i = 1; i < number_pos; i++){\r
+ ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));\r
+ }\r
+ return integer_type_suffix (ui, c);\r
+ }\r
} catch (OverflowException) {\r
error_details = "Integral constant is too large";\r
Report.Error (1021, Location, error_details);\r
\r
int adjust_real (int t)\r
{\r
- string s = number_builder.ToString ();\r
+ string s = new String (number_builder, 0, number_pos);\r
\r
switch (t){\r
case Token.LITERAL_DECIMAL:\r
return t;\r
}\r
\r
+ int handle_hex ()\r
+ {\r
+ int d;\r
+ ulong ul;\r
+ \r
+ getChar ();\r
+ while ((d = peekChar ()) != -1){\r
+ if (is_hex (d)){\r
+ if (number_pos == 16){\r
+ Report.Error (1021, Location, "Integral constant too large");\r
+ return Token.ERROR;\r
+ }\r
+ number_builder [number_pos++] = (char) d;\r
+ getChar ();\r
+ } else\r
+ break;\r
+ }\r
+ \r
+ string s = new String (number_builder, 0, number_pos);\r
+ if (number_pos <= 8)\r
+ ul = System.UInt32.Parse (s, NumberStyles.HexNumber);\r
+ else\r
+ ul = System.UInt64.Parse (s, NumberStyles.HexNumber);\r
+ return integer_type_suffix (ul, peekChar ());\r
+ }\r
+\r
//\r
// Invoked if we know we have .digits or digits\r
//\r
bool is_real = false;\r
int type;\r
\r
- number_builder.Length = 0;\r
+ number_pos = 0;\r
\r
if (c >= '0' && c <= '9'){\r
- if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){\r
- ulong ul;\r
- getChar ();\r
- hex_digits (-1);\r
-\r
- string s = number_builder.ToString ();\r
+ if (c == '0'){\r
+ int peek = peekChar ();\r
\r
- ul = System.UInt64.Parse (s, NumberStyles.HexNumber);\r
- return integer_type_suffix (ul, peekChar ());\r
+ if (peek == 'x' || peek == 'X')\r
+ return handle_hex ();\r
}\r
decimal_digits (c);\r
c = getChar ();\r
c = getChar ();\r
} else {\r
putback ('.');\r
- number_builder.Length -= 1;\r
+ number_pos--;\r
return adjust_int (-1);\r
}\r
}\r
\r
if (c == 'e' || c == 'E'){\r
is_real = true;\r
- number_builder.Append ("e");\r
+ if (number_pos == max_number_size)\r
+ Error_NumericConstantTooLong ();\r
+ number_builder [number_pos++] = 'e';\r
c = getChar ();\r
\r
if (c == '+'){\r
- number_builder.Append ((char) c);\r
+ if (number_pos == max_number_size)\r
+ Error_NumericConstantTooLong ();\r
+ number_builder [number_pos++] = '+';\r
c = -1;\r
} else if (c == '-') {\r
- number_builder.Append ((char) c);\r
+ if (number_pos == max_number_size)\r
+ Error_NumericConstantTooLong ();\r
+ number_builder [number_pos++] = '-';\r
c = -1;\r
- } else \r
- number_builder.Append ('+');\r
+ } else {\r
+ if (number_pos == max_number_size)\r
+ Error_NumericConstantTooLong ();\r
+ number_builder [number_pos++] = '+';\r
+ }\r
\r
decimal_digits (c);\r
c = getChar ();\r
int i;\r
int total = 0;\r
int c;\r
- char e;\r
int top = count != -1 ? count : 4;\r
\r
getChar ();\r
{\r
if (putback_char != -1)\r
return putback_char;\r
- return reader.Peek ();\r
+ putback_char = reader.Read ();\r
+ return putback_char;\r
}\r
\r
- void putback (int c)\r
+ int peekChar2 ()\r
{\r
if (putback_char != -1)\r
+ return putback_char;\r
+ return reader.Peek ();\r
+ }\r
+ \r
+ void putback (int c)\r
+ {\r
+ if (putback_char != -1){\r
+ Console.WriteLine ("Col: " + col);\r
+ Console.WriteLine ("Row: " + line);\r
+ Console.WriteLine ("Name: " + ref_name.Name);\r
+ Console.WriteLine ("Current [{0}] putting back [{1}] ", putback_char, c);\r
throw new Exception ("This should not happen putback on putback");\r
+ }\r
putback_char = c;\r
}\r
\r
arg = "";\r
static_cmd_arg.Length = 0;\r
\r
- while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t')){\r
- if (c == '\r')\r
- continue;\r
+ while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){\r
static_cmd_arg.Append ((char) c);\r
}\r
\r
line++;\r
ref_line++;\r
return;\r
- }\r
+ } else if (c == '\r')\r
+ col = 0;\r
\r
// skip over white space\r
- while ((c = getChar ()) != -1 && (c != '\n') && ((c == ' ') || (c == '\t')))\r
+ while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))\r
;\r
\r
if (c == '\n'){\r
line++;\r
ref_line++;\r
return;\r
+ } else if (c == '\r'){\r
+ col = 0;\r
+ return;\r
}\r
\r
static_cmd_arg.Length = 0;\r
static_cmd_arg.Append ((char) c);\r
\r
- while ((c = getChar ()) != -1 && (c != '\n')){\r
- if (c == '\r')\r
- continue;\r
+ while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){\r
static_cmd_arg.Append ((char) c);\r
}\r
\r
if (c == '\n'){\r
line++;\r
ref_line++;\r
- }\r
+ } else if (c == '\r')\r
+ col = 0;\r
arg = static_cmd_arg.ToString ().Trim ();\r
}\r
\r
return false;\r
\r
if (arg == "default"){\r
- line = ref_line = line;\r
- return false;\r
+ ref_line = line;\r
+ ref_name = file_name;\r
+ Location.Push (ref_name);\r
+ return true;\r
}\r
\r
try {\r
\r
char [] quotes = { '\"' };\r
\r
- ref_name = arg.Substring (pos). Trim (quotes);\r
+ string name = arg.Substring (pos). Trim (quotes);\r
+ ref_name = Location.LookupFile (name);\r
+ Location.Push (ref_name);\r
} else {\r
ref_line = System.Int32.Parse (arg);\r
}\r
bool pp_expr (ref string s)\r
{\r
bool va = pp_and (ref s);\r
-\r
s = s.Trim ();\r
int len = s.Length;\r
if (len > 0){\r
if (c == '|'){\r
if (len > 2 && s [1] == '|'){\r
s = s.Substring (2);\r
- return va | pp_and (ref s);\r
+ return va | pp_expr (ref s);\r
} else {\r
Error_InvalidDirective ();\r
return false;\r
}\r
} \r
}\r
-\r
+ \r
return va;\r
}\r
\r
{\r
char [] blank = { ' ', '\t' };\r
string cmd, arg;\r
- \r
+ bool region_directive = false;\r
+\r
get_cmd_arg (out cmd, out arg);\r
\r
// Eat any trailing whitespaces and single-line comments\r
return true;\r
\r
case "region":\r
+ region_directive = true;\r
arg = "true";\r
goto case "if";\r
\r
case "endregion":\r
+ region_directive = true;\r
goto case "endif";\r
\r
case "if":\r
if ((state & TAKING) != 0)\r
taking = true;\r
}\r
- \r
+\r
if (eval (arg) && taking){\r
- ifstack.Push (TAKING | TAKEN_BEFORE | PARENT_TAKING);\r
+ int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;\r
+ if (region_directive)\r
+ push |= REGION;\r
+ ifstack.Push (push);\r
return true;\r
} else {\r
- ifstack.Push (taking ? PARENT_TAKING : 0);\r
+ int push = (taking ? PARENT_TAKING : 0);\r
+ if (region_directive)\r
+ push |= REGION;\r
+ ifstack.Push (push);\r
return false;\r
}\r
\r
Error_UnexpectedDirective ("no #if for this #endif");\r
return true;\r
} else {\r
- ifstack.Pop ();\r
+ int pop = (int) ifstack.Pop ();\r
+ \r
+ if (region_directive && ((pop & REGION) == 0))\r
+ Report.Error (1027, Location, "#endif directive expected");\r
+ else if (!region_directive && ((pop & REGION) != 0))\r
+ Report.Error (1038, Location, "#endregion directive expected");\r
+ \r
if (ifstack.Count == 0)\r
return true;\r
else {\r
} else {\r
int state = (int) ifstack.Peek ();\r
\r
+ if ((state & REGION) != 0) {\r
+ Report.Error (1038, Location, "#endregion directive expected");\r
+ return true;\r
+ }\r
+\r
if ((state & ELSE_SEEN) != 0){\r
Error_UnexpectedDirective ("#elif not valid after #else");\r
return true;\r
} else {\r
int state = (int) ifstack.Peek ();\r
\r
+ if ((state & REGION) != 0) {\r
+ Report.Error (1038, Location, "#endregion directive expected");\r
+ return true;\r
+ }\r
+\r
if ((state & ELSE_SEEN) != 0){\r
Error_UnexpectedDirective ("#else within #else");\r
return true;\r
}\r
\r
ifstack.Pop ();\r
- ifstack.Push (state | ELSE_SEEN);\r
\r
+ bool ret;\r
if ((state & TAKEN_BEFORE) == 0){\r
- if ((state & PARENT_TAKING) != 0)\r
- return true;\r
- else\r
- return false;\r
- }\r
- return false;\r
+ ret = ((state & PARENT_TAKING) != 0);\r
+ } else\r
+ ret = false;\r
+ \r
+ if (ret)\r
+ state |= TAKING;\r
+ else\r
+ state &= ~TAKING;\r
+ \r
+ ifstack.Push (state | ELSE_SEEN);\r
+ \r
+ return ret;\r
}\r
}\r
\r
return Token.EOF;\r
}\r
\r
- private int consume_identifier (int c, bool quoted) \r
+ private int consume_identifier (int s, bool quoted) \r
{\r
- id_builder.Length = 0;\r
-\r
- id_builder.Append ((char) c);\r
+ int pos = 1;\r
+ int c;\r
+ \r
+ id_builder [0] = (char) s;\r
\r
- while ((c = peekChar ()) != -1) {\r
+ while ((c = reader.Read ()) != -1) {\r
if (is_identifier_part_character ((char) c)){\r
- id_builder.Append ((char)getChar ());\r
+ if (pos == max_id_size){\r
+ Report.Error (645, Location, "Identifier too long (limit is 512 chars)");\r
+ return Token.ERROR;\r
+ }\r
+ \r
+ id_builder [pos++] = (char) c;\r
+ putback_char = -1;\r
col++;\r
- } else \r
+ } else {\r
+ putback_char = c;\r
break;\r
- }\r
- \r
- string ids = id_builder.ToString ();\r
- int keyword = GetKeyword (ids);\r
- \r
- if (keyword == -1 || quoted){\r
- val = ids;\r
- if (ids.Length > 512){\r
- Report.Error (\r
- 645, Location,\r
- "Identifier too long (limit is 512 chars)");\r
}\r
- return Token.IDENTIFIER;\r
}\r
\r
- return keyword;\r
+ string ids = new String (id_builder, 0, pos);\r
+\r
+ if (s >= 'a'){\r
+ int keyword = GetKeyword (ids);\r
+ if (keyword == -1 || quoted){\r
+ val = ids;\r
+ return Token.IDENTIFIER;\r
+ }\r
+ return keyword;\r
+ }\r
+ val = ids;\r
+ return Token.IDENTIFIER;\r
}\r
\r
public int xtoken ()\r
val = null;\r
// optimization: eliminate col and implement #directive semantic correctly.\r
for (;(c = getChar ()) != -1; col++) {\r
- if (is_identifier_start_character ((char)c)){\r
- tokens_seen = true;\r
- return consume_identifier (c, false);\r
- }\r
-\r
- if (c == '.'){\r
- tokens_seen = true;\r
- int peek = peekChar ();\r
- if (peek >= '0' && peek <= '9')\r
- return is_number (c);\r
- return Token.DOT;\r
- }\r
- \r
- if (c >= '0' && c <= '9'){\r
- tokens_seen = true;\r
- return is_number (c);\r
+ if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r' || c == 0xa0){\r
+ \r
+ if (c == '\t')\r
+ col = (((col + 8) / 8) * 8) - 1;\r
+ continue;\r
}\r
\r
// Handle double-slash comments.\r
}\r
continue;\r
}\r
+ goto is_punct_label;\r
}\r
\r
+ \r
+ if (is_identifier_start_character ((char)c)){\r
+ tokens_seen = true;\r
+ return consume_identifier (c, false);\r
+ }\r
+\r
+ is_punct_label:\r
+ if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){\r
+ tokens_seen = true;\r
+ if (doread){\r
+ getChar ();\r
+ col++;\r
+ }\r
+ return t;\r
+ }\r
+\r
+ // white space\r
+ if (c == '\n'){\r
+ line++;\r
+ ref_line++;\r
+ col = 0;\r
+ any_token_seen |= tokens_seen;\r
+ tokens_seen = false;\r
+ continue;\r
+ }\r
+\r
+ if (c >= '0' && c <= '9'){\r
+ tokens_seen = true;\r
+ return is_number (c);\r
+ }\r
+\r
+ if (c == '.'){\r
+ tokens_seen = true;\r
+ int peek = peekChar ();\r
+ if (peek >= '0' && peek <= '9')\r
+ return is_number (c);\r
+ return Token.DOT;\r
+ }\r
+ \r
/* For now, ignore pre-processor commands */\r
// FIXME: In C# the '#' is not limited to appear\r
// on the first column.\r
line++;\r
ref_line++;\r
skipping = false;\r
- } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r')\r
+ } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)\r
continue;\r
else if (c != '#')\r
skipping = true;\r
continue;\r
}\r
\r
- if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){\r
- tokens_seen = true;\r
- if (doread){\r
- getChar ();\r
- col++;\r
- }\r
- return t;\r
- }\r
- \r
- if (c == '"') {\r
+ if (c == '"') \r
return consume_string (false);\r
- }\r
\r
if (c == '\''){\r
c = getChar ();\r
return Token.LITERAL_CHARACTER;\r
}\r
\r
- // white space\r
- if (c == '\n'){\r
- line++;\r
- ref_line++;\r
- col = 0;\r
- any_token_seen |= tokens_seen;\r
- tokens_seen = false;\r
- continue;\r
- }\r
-\r
- if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r'){\r
- if (c == '\t')\r
- col = (((col + 8) / 8) * 8) - 1;\r
- continue;\r
- }\r
-\r
if (c == '@') {\r
c = getChar ();\r
if (c == '"') {\r
return Token.ERROR;\r
}\r
\r
- if (ifstack != null && ifstack.Count >= 1)\r
- Report.Error (1027, Location, "#endif/#endregion expected");\r
-\r
return Token.EOF;\r
}\r
+\r
+ public void cleanup ()\r
+ {\r
+ if (ifstack != null && ifstack.Count >= 1) {\r
+ int state = (int) ifstack.Pop ();\r
+ if ((state & REGION) != 0)\r
+ Report.Error (1038, "#endregion directive expected");\r
+ else \r
+ Report.Error (1027, "#endif directive expected");\r
+ }\r
+ \r
+ }\r
+\r
}\r
}\r
\r
projects / mono.git / blobdiff