X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mcs%2Fmcs%2Fcs-tokenizer.cs;h=5193fa277ef15ba7b5c6a858d9ae33cfb3b838b6;hb=27ac138a66975973b4517b43b9ceda6abe976184;hp=0cb9609097ed20e9cb13d9a47a3da067de6e4319;hpb=6d73a098f2e3840d7db52d276f5d6b32835cb18b;p=mono.git diff --git a/mcs/mcs/cs-tokenizer.cs b/mcs/mcs/cs-tokenizer.cs index 0cb9609097e..5193fa277ef 100755 --- a/mcs/mcs/cs-tokenizer.cs +++ b/mcs/mcs/cs-tokenizer.cs @@ -6,7 +6,7 @@ // // Licensed under the terms of the GNU GPL // -// (C) 2001 Ximian, Inc (http://www.ximian.com) +// (C) 2001, 2002 Ximian, Inc (http://www.ximian.com) // /* @@ -20,6 +20,7 @@ using System.Text; using System.Collections; using System.IO; using System.Globalization; +using System.Reflection; namespace Mono.CSharp { @@ -43,6 +44,36 @@ namespace Mono.CSharp // Whether tokens have been seen on this line // bool tokens_seen = false; + + // + // Whether a token has been seen on the file + // This is needed because `define' is not allowed to be used + // after a token has been seen. + // + bool any_token_seen = false; + static Hashtable tokenValues; + + private static Hashtable TokenValueName + { + get { + if (tokenValues == null) + tokenValues = GetTokenValueNameHash (); + + return tokenValues; + } + } + + private static Hashtable GetTokenValueNameHash () + { + Type t = typeof (Token); + FieldInfo [] fields = t.GetFields (); + Hashtable hash = new Hashtable (); + foreach (FieldInfo field in fields) { + if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int)) + hash.Add (field.GetValue (null), field.Name); + } + return hash; + } // // Returns a verbose representation of the current location @@ -59,8 +90,15 @@ namespace Mono.CSharp // return "Line: "+line+" Col: "+col + "\n" + // "VirtLine: "+ref_line + // " Token: "+current_token + " " + det; - - return ref_name + " " + "(" + line + "," + col + "), Token:" + current_token + " " + det; + string current_token_name = TokenValueName [current_token] as string; + if (current_token_name == null) + current_token_name = current_token.ToString (); + + return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name, + line, + col, + current_token_name, + det); } } @@ -104,7 +142,6 @@ namespace Mono.CSharp // // Values for the associated token returned // - System.Text.StringBuilder number; int putback_char; Object val; @@ -122,6 +159,15 @@ namespace Mono.CSharp // pre-processor if stack state: // Stack ifstack; + + static System.Text.StringBuilder string_builder; + + const int max_id_size = 512; + static char [] id_builder = new char [max_id_size]; + + const int max_number_size = 128; + static char [] number_builder = new char [max_number_size]; + static int number_pos; // // Details about the error encoutered by the tokenizer @@ -145,7 +191,7 @@ namespace Mono.CSharp return col; } } - + static void InitTokens () { keywords = new Hashtable (); @@ -212,6 +258,7 @@ namespace Mono.CSharp keywords.Add ("set", Token.SET); keywords.Add ("short", Token.SHORT); keywords.Add ("sizeof", Token.SIZEOF); + keywords.Add ("stackalloc", Token.STACKALLOC); keywords.Add ("static", Token.STATIC); keywords.Add ("string", Token.STRING); keywords.Add ("struct", Token.STRUCT); @@ -239,28 +286,29 @@ namespace Mono.CSharp static Tokenizer () { InitTokens (); - csharp_format_info = new NumberFormatInfo (); - csharp_format_info.CurrencyDecimalSeparator = "."; - styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint; - } - - bool is_keyword (string name) - { - bool res; + csharp_format_info = NumberFormatInfo.InvariantInfo; + styles = NumberStyles.Float; - res = keywords.Contains (name); - if (handle_get_set == false && (name == "get" || name == "set")) - return false; - if (handle_remove_add == false && (name == "remove" || name == "add")) - return false; - if (handle_assembly == false && (name == "assembly")) - return false; - return res; + string_builder = new System.Text.StringBuilder (); } int GetKeyword (string name) { - return (int) (keywords [name]); + object o = keywords [name]; + + if (o == null) + return -1; + + int res = (int) o; + + if (handle_get_set == false && (res == Token.GET || res == Token.SET)) + return -1; + if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD)) + return -1; + if (handle_assembly == false && res == Token.ASSEMBLY) + return -1; + return res; + } public Location Location { @@ -268,17 +316,28 @@ namespace Mono.CSharp return new Location (ref_line); } } + + void define (string def) + { + if (!RootContext.AllDefines.Contains (def)){ + RootContext.AllDefines [def] = true; + } + if (defines.Contains (def)) + return; + defines [def] = true; + } - public Tokenizer (System.IO.Stream input, string fname, ArrayList defs) + public Tokenizer (StreamReader input, string fname, ArrayList defs) { this.ref_name = fname; - reader = new System.IO.StreamReader (input); + reader = input; + putback_char = -1; if (defs != null){ defines = new Hashtable (); foreach (string def in defs) - defines [def] = true; + define (def); } // @@ -290,12 +349,12 @@ namespace Mono.CSharp bool is_identifier_start_character (char c) { - return Char.IsLetter (c) || c == '_' ; + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c); } bool is_identifier_part_character (char c) { - return (Char.IsLetter (c) || Char.IsDigit (c) || c == '_'); + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c); } int is_punct (char c, ref bool doread) @@ -461,17 +520,31 @@ namespace Mono.CSharp return Token.ERROR; } + void Error_NumericConstantTooLong () + { + Report.Error (1021, Location, "Numeric constant too long"); + } + bool decimal_digits (int c) { int d; bool seen_digits = false; - if (c != -1) - number.Append ((char) c); + if (c != -1){ + if (number_pos == max_number_size) + Error_NumericConstantTooLong (); + number_builder [number_pos++] = (char) c; + } - while ((d = peekChar ()) != -1){ - if (Char.IsDigit ((char)d)){ - number.Append ((char) d); + // + // We use peekChar2, because decimal_digits needs to do a + // 2-character look-ahead (5.ToString for example). + // + while ((d = peekChar2 ()) != -1){ + if (d >= '0' && d <= '9'){ + if (number_pos == max_number_size) + Error_NumericConstantTooLong (); + number_builder [number_pos++] = (char) d; getChar (); seen_digits = true; } else @@ -481,21 +554,18 @@ namespace Mono.CSharp return seen_digits; } + bool is_hex (int e) + { + return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f'); + } + void hex_digits (int c) { int d; if (c != -1) - number.Append ((char) c); - while ((d = peekChar ()) != -1){ - char e = Char.ToUpper ((char) d); - - if (Char.IsDigit (e) || (e >= 'A' && e <= 'F')){ - number.Append ((char) e); - getChar (); - } else - break; - } + number_builder [number_pos++] = (char) c; + } int real_type_suffix (int c) @@ -604,62 +674,111 @@ namespace Mono.CSharp // int adjust_int (int c) { - ulong ul = System.UInt64.Parse (number.ToString ()); - return integer_type_suffix (ul, c); - } + try { + if (number_pos > 9){ + ulong ul = (uint) (number_builder [0] - '0'); + for (int i = 1; i < number_pos; i++){ + ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0'))); + } + return integer_type_suffix (ul, c); + } else { + uint ui = (uint) (number_builder [0] - '0'); + + for (int i = 1; i < number_pos; i++){ + ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0'))); + } + return integer_type_suffix (ui, c); + } + } catch (OverflowException) { + error_details = "Integral constant is too large"; + Report.Error (1021, Location, error_details); + val = 0ul; + return Token.LITERAL_INTEGER; + } + } + int adjust_real (int t) { - string s = number.ToString (); + string s = new String (number_builder, 0, number_pos); switch (t){ case Token.LITERAL_DECIMAL: - val = new System.Decimal (); - val = System.Decimal.Parse ( - s, styles, csharp_format_info); - break; - case Token.LITERAL_DOUBLE: - val = new System.Double (); - val = System.Double.Parse ( - s, styles, csharp_format_info); + try { + val = System.Decimal.Parse (s, styles, csharp_format_info); + } catch (OverflowException) { + val = 0m; + error_details = "Floating-point constant is outside the range of the type 'decimal'"; + Report.Error (594, Location, error_details); + } break; case Token.LITERAL_FLOAT: - val = new System.Double (); - val = (float) System.Double.Parse ( - s, styles, csharp_format_info); + try { + val = (float) System.Double.Parse (s, styles, csharp_format_info); + } catch (OverflowException) { + val = 0.0f; + error_details = "Floating-point constant is outside the range of the type 'float'"; + Report.Error (594, Location, error_details); + } break; - + + case Token.LITERAL_DOUBLE: case Token.NONE: - val = new System.Double (); - val = System.Double.Parse ( - s, styles, csharp_format_info); t = Token.LITERAL_DOUBLE; + try { + val = System.Double.Parse (s, styles, csharp_format_info); + } catch (OverflowException) { + val = 0.0; + error_details = "Floating-point constant is outside the range of the type 'double'"; + Report.Error (594, Location, error_details); + } break; } return t; } + int handle_hex () + { + int d; + ulong ul; + + getChar (); + while ((d = peekChar ()) != -1){ + if (is_hex (d)){ + if (number_pos == 16){ + Report.Error (1021, Location, "Integral constant too large"); + return Token.ERROR; + } + number_builder [number_pos++] = (char) d; + getChar (); + } else + break; + } + + string s = new String (number_builder, 0, number_pos); + if (number_pos <= 8) + ul = System.UInt32.Parse (s, NumberStyles.HexNumber); + else + ul = System.UInt64.Parse (s, NumberStyles.HexNumber); + return integer_type_suffix (ul, peekChar ()); + } + // // Invoked if we know we have .digits or digits // int is_number (int c) { bool is_real = false; - number = new System.Text.StringBuilder (); int type; - number.Length = 0; + number_pos = 0; - if (Char.IsDigit ((char)c)){ - if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){ - ulong ul; - getChar (); - hex_digits (-1); - - string s = number.ToString (); + if (c >= '0' && c <= '9'){ + if (c == '0'){ + int peek = peekChar (); - ul = System.UInt64.Parse (s, NumberStyles.HexNumber); - return integer_type_suffix (ul, peekChar ()); + if (peek == 'x' || peek == 'X') + return handle_hex (); } decimal_digits (c); c = getChar (); @@ -675,24 +794,35 @@ namespace Mono.CSharp c = getChar (); } else { putback ('.'); - number.Length -= 1; + number_pos--; return adjust_int (-1); } } if (c == 'e' || c == 'E'){ is_real = true; - number.Append ("e"); + if (number_pos == max_number_size) + Error_NumericConstantTooLong (); + number_builder [number_pos++] = 'e'; c = getChar (); if (c == '+'){ - number.Append ((char) c); - c = getChar (); - } else if (c == '-'){ - number.Append ((char) c); - c = getChar (); + if (number_pos == max_number_size) + Error_NumericConstantTooLong (); + number_builder [number_pos++] = '+'; + c = -1; + } else if (c == '-') { + if (number_pos == max_number_size) + Error_NumericConstantTooLong (); + number_builder [number_pos++] = '-'; + c = -1; + } else { + if (number_pos == max_number_size) + Error_NumericConstantTooLong (); + number_builder [number_pos++] = '+'; } - decimal_digits (-1); + + decimal_digits (c); c = getChar (); } @@ -713,9 +843,49 @@ namespace Mono.CSharp Console.WriteLine ("This should not be reached"); throw new Exception ("Is Number should never reach this point"); } + + // + // Accepts exactly count (4 or 8) hex, no more no less + // + int getHex (int count, out bool error) + { + int [] buffer = new int [8]; + int i; + int total = 0; + int c; + int top = count != -1 ? count : 4; + getChar (); + error = false; + for (i = 0; i < top; i++){ + c = getChar (); + + if (c >= '0' && c <= '9') + c = (int) c - (int) '0'; + else if (c >= 'A' && c <= 'F') + c = (int) c - (int) 'A' + 10; + else if (c >= 'a' && c <= 'f') + c = (int) c - (int) 'a' + 10; + else { + error = true; + return 0; + } + + total = (total * 16) + c; + if (count == -1){ + int p = peekChar (); + if (p == -1) + break; + if (!is_hex ((char)p)) + break; + } + } + return total; + } + int escape (int c) { + bool error; int d; int v; @@ -746,9 +916,24 @@ namespace Mono.CSharp v = '"'; break; case '\'': v = '\''; break; + case 'x': + v = getHex (-1, out error); + if (error) + goto default; + return v; + case 'u': + v = getHex (4, out error); + if (error) + goto default; + return v; + case 'U': + v = getHex (8, out error); + if (error) + goto default; + return v; default: - error_details = "cs1009: Unrecognized escape sequence " + (char)d; - return -1; + Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d); + return d; } getChar (); return v; @@ -769,13 +954,26 @@ namespace Mono.CSharp { if (putback_char != -1) return putback_char; - return reader.Peek (); + putback_char = reader.Read (); + return putback_char; } - void putback (int c) + int peekChar2 () { if (putback_char != -1) + return putback_char; + return reader.Peek (); + } + + void putback (int c) + { + if (putback_char != -1){ + Console.WriteLine ("Col: " + col); + Console.WriteLine ("Row: " + line); + Console.WriteLine ("Name: " + ref_name); + Console.WriteLine ("Current [{0}] putting back [{1}] ", putback_char, c); throw new Exception ("This should not happen putback on putback"); + } putback_char = c; } @@ -811,9 +1009,7 @@ namespace Mono.CSharp arg = ""; static_cmd_arg.Length = 0; - while ((c = getChar ()) != -1 && (c != '\n') && ((c != ' ') || c == '\t')){ - if (c == '\r') - continue; + while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){ static_cmd_arg.Append ((char) c); } @@ -823,31 +1019,34 @@ namespace Mono.CSharp line++; ref_line++; return; - } + } else if (c == '\r') + col = 0; // skip over white space - while ((c = getChar ()) != -1 && (c != '\n') && ((c == ' ') || (c == '\t'))) + while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t'))) ; if (c == '\n'){ line++; ref_line++; return; + } else if (c == '\r'){ + col = 0; + return; } static_cmd_arg.Length = 0; static_cmd_arg.Append ((char) c); - while ((c = getChar ()) != -1 && (c != '\n')){ - if (c == '\r') - continue; + while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){ static_cmd_arg.Append ((char) c); } if (c == '\n'){ line++; ref_line++; - } + } else if (c == '\r') + col = 0; arg = static_cmd_arg.ToString ().Trim (); } @@ -873,9 +1072,7 @@ namespace Mono.CSharp char [] quotes = { '\"' }; - ref_name = arg.Substring (pos); - ref_name.TrimStart (quotes); - ref_name.TrimEnd (quotes); + ref_name = arg.Substring (pos). Trim (quotes); } else { ref_line = System.Int32.Parse (arg); } @@ -892,14 +1089,27 @@ namespace Mono.CSharp void PreProcessDefinition (bool is_define, string arg) { if (arg == "" || arg == "true" || arg == "false"){ - Report.Error(1001, Location, "Missing identifer to pre-processor directive"); + Report.Error (1001, Location, "Missing identifer to pre-processor directive"); + return; + } + + char[] whitespace = { ' ', '\t' }; + if (arg.IndexOfAny (whitespace) != -1){ + Report.Error (1025, Location, "Single-line comment or end-of-line expected"); return; } + foreach (char c in arg){ + if (!Char.IsLetter (c) && (c != '_')){ + Report.Error (1001, Location, "Identifier expected"); + return; + } + } + if (is_define){ if (defines == null) defines = new Hashtable (); - defines [arg] = 1; + define (arg); } else { if (defines == null) return; @@ -925,7 +1135,7 @@ namespace Mono.CSharp bool pp_primary (ref string s) { - s.Trim (); + s = s.Trim (); int len = s.Length; if (len > 0){ @@ -934,19 +1144,21 @@ namespace Mono.CSharp if (c == '('){ s = s.Substring (1); bool val = pp_expr (ref s); - if (s.Length > 0 && s [0] == ')') + if (s.Length > 0 && s [0] == ')'){ + s = s.Substring (1); return val; - report1517 (); + } + Error_InvalidDirective (); return false; } - if (Char.IsLetter (c) || c == '_'){ + if (is_identifier_start_character (c)){ int j = 1; while (j < len){ c = s [j]; - if (Char.IsLetter (c) || Char.IsDigit (c) || c == '_'){ + if (is_identifier_part_character (c)){ j++; continue; } @@ -959,7 +1171,7 @@ namespace Mono.CSharp return vv; } } - report1517 (); + Error_InvalidDirective (); return false; } @@ -971,7 +1183,7 @@ namespace Mono.CSharp if (len > 0){ if (s [0] == '!'){ if (len > 1 && s [1] == '='){ - report1517 (); + Error_InvalidDirective (); return false; } s = s.Substring (1); @@ -979,7 +1191,7 @@ namespace Mono.CSharp } else return pp_primary (ref s); } else { - report1517 (); + Error_InvalidDirective (); return false; } } @@ -996,7 +1208,7 @@ namespace Mono.CSharp s = s.Substring (2); return va == pp_unary (ref s); } else { - report1517 (); + Error_InvalidDirective (); return false; } } else if (s [0] == '!' && len > 1 && s [1] == '='){ @@ -1004,7 +1216,7 @@ namespace Mono.CSharp return va != pp_unary (ref s); - } + } } return va; @@ -1021,9 +1233,9 @@ namespace Mono.CSharp if (s [0] == '&'){ if (len > 2 && s [1] == '&'){ s = s.Substring (2); - return va && pp_eq (ref s); + return (va & pp_eq (ref s)); } else { - report1517 (); + Error_InvalidDirective (); return false; } } @@ -1037,58 +1249,76 @@ namespace Mono.CSharp bool pp_expr (ref string s) { bool va = pp_and (ref s); - s = s.Trim (); int len = s.Length; if (len > 0){ - if (s [0] == '|'){ + char c = s [0]; + + if (c == '|'){ if (len > 2 && s [1] == '|'){ s = s.Substring (2); - return va || pp_and (ref s); + return va | pp_expr (ref s); } else { - report1517 (); + Error_InvalidDirective (); return false; } - } else { - report1517 (); - return false; - } + } } - + return va; } bool eval (string s) { bool v = pp_expr (ref s); + s = s.Trim (); + if (s.Length != 0){ + Error_InvalidDirective (); + return false; + } return v; } - void report1517 () + void Error_InvalidDirective () { Report.Error (1517, Location, "Invalid pre-processor directive"); } - void report1028 (string extra) + void Error_UnexpectedDirective (string extra) { Report.Error ( 1028, Location, "Unexpected processor directive (" + extra + ")"); } + + void Error_TokensSeen () + { + Report.Error ( + 1032, Location, + "Cannot define or undefine pre-processor symbols after a token in the file"); + } // // if true, then the code continues processing the code // if false, the code stays in a loop until another directive is // reached. // - bool handle_preprocessing_directive () + bool handle_preprocessing_directive (bool caller_is_taking) { char [] blank = { ' ', '\t' }; string cmd, arg; get_cmd_arg (out cmd, out arg); - + + // Eat any trailing whitespaces and single-line comments + if (arg.IndexOf ("//") != -1) + arg = arg.Substring (0, arg.IndexOf ("//")); + arg = arg.TrimEnd (' ', '\t'); + + // + // The first group of pre-processing instructions is always processed + // switch (cmd){ case "line": if (!PreProcessLine (arg)) @@ -1097,22 +1327,6 @@ namespace Mono.CSharp "Argument to #line directive is missing or invalid"); return true; - case "define": - PreProcessDefinition (true, arg); - return true; - - case "undef": - PreProcessDefinition (false, arg); - return true; - - case "error": - Report.Error (1029, Location, "#error: '" + arg + "'"); - return true; - - case "warning": - Report.Warning (1030, Location, "#warning: '" + arg + "'"); - return true; - case "region": arg = "true"; goto case "if"; @@ -1122,7 +1336,7 @@ namespace Mono.CSharp case "if": if (arg == ""){ - report1517 (); + Error_InvalidDirective (); return true; } bool taking = false; @@ -1147,7 +1361,7 @@ namespace Mono.CSharp case "endif": if (ifstack == null || ifstack.Count == 0){ - report1028 ("no #if for this #endif"); + Error_UnexpectedDirective ("no #if for this #endif"); return true; } else { ifstack.Pop (); @@ -1165,13 +1379,13 @@ namespace Mono.CSharp case "elif": if (ifstack == null || ifstack.Count == 0){ - report1028 ("no #if for this #elif"); + Error_UnexpectedDirective ("no #if for this #elif"); return true; } else { int state = (int) ifstack.Peek (); if ((state & ELSE_SEEN) != 0){ - report1028 ("#elif not valid after #else"); + Error_UnexpectedDirective ("#elif not valid after #else"); return true; } @@ -1196,79 +1410,155 @@ namespace Mono.CSharp int state = (int) ifstack.Peek (); if ((state & ELSE_SEEN) != 0){ - report1028 ("#else within #else"); + Error_UnexpectedDirective ("#else within #else"); return true; } ifstack.Pop (); - ifstack.Push (state | ELSE_SEEN); + bool ret; if ((state & TAKEN_BEFORE) == 0){ - if ((state & PARENT_TAKING) != 0) - return true; - else - return false; + ret = ((state & PARENT_TAKING) != 0); + } else + ret = false; + + if (ret) + state |= TAKING; + else + state &= ~TAKING; + + ifstack.Push (state | ELSE_SEEN); + + return ret; + } + } + + // + // These are only processed if we are in a `taking' block + // + if (!caller_is_taking) + return false; + + switch (cmd){ + case "define": + if (any_token_seen){ + Error_TokensSeen (); + return true; + } + PreProcessDefinition (true, arg); + return true; + + case "undef": + if (any_token_seen){ + Error_TokensSeen (); + return true; + } + PreProcessDefinition (false, arg); + return true; + + case "error": + Report.Error (1029, Location, "#error: '" + arg + "'"); + return true; + + case "warning": + Report.Warning (1030, Location, "#warning: '" + arg + "'"); + return true; + } + + Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")"); + return true; + + } + + private int consume_string (bool quoted) + { + int c; + string_builder.Length = 0; + + while ((c = getChar ()) != -1){ + if (c == '"'){ + if (quoted && peekChar () == '"'){ + string_builder.Append ((char) c); + getChar (); + continue; + } else { + val = string_builder.ToString (); + return Token.LITERAL_STRING; } - return false; } + + if (c == '\n'){ + if (!quoted) + Report.Error (1010, Location, "Newline in constant"); + line++; + ref_line++; + col = 0; + } else + col++; + + if (!quoted){ + c = escape (c); + if (c == -1) + return Token.ERROR; + } + string_builder.Append ((char) c); } + + Report.Error (1039, Location, "Unterminated string literal"); + return Token.EOF; + } + + private int consume_identifier (int s, bool quoted) + { + int pos = 1; + int c; - Report.Error (1024, "Preprocessor directive expected (got: " + cmd + ")"); - return true; + id_builder [0] = (char) s; + + while ((c = reader.Read ()) != -1) { + if (is_identifier_part_character ((char) c)){ + if (pos == max_id_size){ + Report.Error (645, Location, "Identifier too long (limit is 512 chars)"); + return Token.ERROR; + } + + id_builder [pos++] = (char) c; + putback_char = -1; + col++; + } else { + putback_char = c; + break; + } + } + + string ids = new String (id_builder, 0, pos); + + if (s >= 'a'){ + int keyword = GetKeyword (ids); + if (keyword == -1 || quoted){ + val = ids; + return Token.IDENTIFIER; + } + return keyword; + } + val = ids; + return Token.IDENTIFIER; } public int xtoken () { int t; - bool allow_keyword_as_ident = false; bool doread = false; int c; val = null; // optimization: eliminate col and implement #directive semantic correctly. for (;(c = getChar ()) != -1; col++) { - if (is_identifier_start_character ((char) c)){ - System.Text.StringBuilder id = new System.Text.StringBuilder (); - string ids; - - tokens_seen = true; - id.Append ((char) c); - - while ((c = peekChar ()) != -1) { - if (is_identifier_part_character ((char) c)){ - id.Append ((char)getChar ()); - col++; - } else - break; - } + if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r' || c == 0xa0){ - ids = id.ToString (); - - if (!is_keyword (ids) || allow_keyword_as_ident) { - val = ids; - if (ids.Length > 512){ - Report.Error ( - 645, Location, - "Identifier too long (limit is 512 chars)"); - } - return Token.IDENTIFIER; - } - - // true, false and null are in the hash anyway. - return GetKeyword (ids); - - } - - if (c == '.'){ - tokens_seen = true; - if (Char.IsDigit ((char) peekChar ())) - return is_number (c); - return Token.DOT; - } - - if (Char.IsDigit ((char) c)){ - tokens_seen = true; - return is_number (c); + if (c == '\t') + col = (((col + 8) / 8) * 8) - 1; + continue; } // Handle double-slash comments. @@ -1277,11 +1567,15 @@ namespace Mono.CSharp if (d == '/'){ getChar (); - while ((d = getChar ()) != -1 && (d != '\n')) + while ((d = getChar ()) != -1 && (d != '\n') && d != '\r') col++; - line++; - ref_line++; - col = 0; + if (d == '\n'){ + line++; + ref_line++; + col = 0; + } + any_token_seen |= tokens_seen; + tokens_seen = false; continue; } else if (d == '*'){ getChar (); @@ -1296,19 +1590,63 @@ namespace Mono.CSharp line++; ref_line++; col = 0; + any_token_seen |= tokens_seen; + tokens_seen = false; } } continue; } + goto is_punct_label; } + + if (is_identifier_start_character ((char)c)){ + tokens_seen = true; + return consume_identifier (c, false); + } + + is_punct_label: + if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){ + tokens_seen = true; + if (doread){ + getChar (); + col++; + } + return t; + } + + // white space + if (c == '\n'){ + line++; + ref_line++; + col = 0; + any_token_seen |= tokens_seen; + tokens_seen = false; + continue; + } + + if (c >= '0' && c <= '9'){ + tokens_seen = true; + return is_number (c); + } + + if (c == '.'){ + tokens_seen = true; + int peek = peekChar (); + if (peek >= '0' && peek <= '9') + return is_number (c); + return Token.DOT; + } + /* For now, ignore pre-processor commands */ // FIXME: In C# the '#' is not limited to appear // on the first column. if (c == '#' && !tokens_seen){ + bool cont = true; + start_again: - bool cont = handle_preprocessing_directive (); + cont = handle_preprocessing_directive (cont); if (cont){ col = 0; @@ -1323,44 +1661,22 @@ namespace Mono.CSharp line++; ref_line++; skipping = false; - } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r') + } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0) continue; else if (c != '#') skipping = true; if (c == '#' && !skipping) goto start_again; } + any_token_seen |= tokens_seen; tokens_seen = false; if (c == -1) - Report.Error (1027, Location, "#endif expected"); + Report.Error (1027, Location, "#endif/#endregion expected"); continue; } - if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){ - tokens_seen = true; - if (doread){ - getChar (); - col++; - } - return t; - } - - if (c == '"'){ - System.Text.StringBuilder s = new System.Text.StringBuilder (); - tokens_seen = true; - - while ((c = getChar ()) != -1){ - if (c == '"'){ - val = s.ToString (); - return Token.LITERAL_STRING; - } - - c = escape (c); - if (c == -1) - return Token.ERROR; - s.Append ((char) c); - } - } + if (c == '"') + return consume_string (false); if (c == '\''){ c = getChar (); @@ -1383,8 +1699,13 @@ namespace Mono.CSharp // Try to recover, read until newline or next "'" while ((c = getChar ()) != -1){ - if (c == '\n' || c == '\'') + if (c == '\n' || c == '\''){ + line++; + ref_line++; + col = 0; break; + } else + col++; } return Token.ERROR; @@ -1392,25 +1713,16 @@ namespace Mono.CSharp return Token.LITERAL_CHARACTER; } - // white space - if (c == '\n'){ - line++; - ref_line++; - col = 0; - tokens_seen = false; - continue; - } - - if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r'){ - if (c == '\t') - col = (((col + 8) / 8) * 8) - 1; - continue; - } - - if (c == '@'){ - tokens_seen = true; - allow_keyword_as_ident = true; - continue; + if (c == '@') { + c = getChar (); + if (c == '"') { + tokens_seen = true; + return consume_string (true); + } else if (is_identifier_start_character ((char) c)){ + return consume_identifier (c, true); + } else { + Report.Error (1033, Location, "'@' must be followed by string constant or identifier"); + } } error_details = ((char)c).ToString (); @@ -1418,8 +1730,9 @@ namespace Mono.CSharp return Token.ERROR; } - if (ifstack != null && ifstack.Count > 1) - Report.Error (1027, Location, "#endif expected"); + if (ifstack != null && ifstack.Count >= 1) + Report.Error (1027, Location, "#endif/#endregion expected"); + return Token.EOF; } }