X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mcs%2Fmcs%2Fcs-tokenizer.cs;h=582266be494d938b444181cba1042d42c601bf5e;hb=cfacf6bed374a9aef8e87c20e7fb8cbdc50184a8;hp=f27439a557179d8e3b19ae83fa8fe68e3f4aa70e;hpb=6f4428a7e70ff8daf80f43ea9bf02ce7538a4e1b;p=mono.git diff --git a/mcs/mcs/cs-tokenizer.cs b/mcs/mcs/cs-tokenizer.cs index f27439a5571..582266be494 100755 --- a/mcs/mcs/cs-tokenizer.cs +++ b/mcs/mcs/cs-tokenizer.cs @@ -1,5 +1,6 @@ // // cs-tokenizer.cs: The Tokenizer for the C# compiler +// This also implements the preprocessor // // Author: Miguel de Icaza (miguel@gnu.org) // @@ -9,50 +10,9 @@ // /* - Todo: - - Do something with the integer and float suffixes, pass full datatype? - Make sure we accept the proper Unicode ranges, per the spec. - - Open issues: - - * Data type handling - - Currently I am returning different tokens for the various - kinds of floating point types (float, double, decimal) and I - am only returning a single token for all integer values - (integer, unsigned int, etc) as an experiment as to see - which mechanism is better. - - I do not know yet how I will be doing the mapping of "int" - to things like System.Int32 and so on. I am confused. MAN - I AM C - - Indeed, this might be the core of the problem, I should - *probably* just return a TYPE token and have the value of - the token be stuff like `System.Int32', `System.UInt32', - `System.Double' and so on. I will see. - - * Error reporting. - - I was returning Token.ERROR on errors and setting an - internal error string with the details, but it might make sense - to just use exceptions. - - Change of mind: I think I want to keep returning errors *UNLESS* the - parser is catching errors from the tokenizer (at that point, there is - not really any reason to use exceptions) so that I can continue the - parsing - - * IDEA - - I think I have solved the problem. The idea is to not even *bother* - about handling data types a lot here (except for fitting data into - the proper places), but let the upper layer handle it. - - Ie, treat LITERAL_CHARACTER, LITERAL_INTEGER, LITERAL_FLOAT, LITERAL_DOUBLE, and - return then as `LITERAL_LITERAL' with maybe subdetail information - + * TODO: + * Make sure we accept the proper Unicode ranges, per the spec. + * Report error 1032 */ using System; @@ -61,7 +21,7 @@ using System.Collections; using System.IO; using System.Globalization; -namespace CIR +namespace Mono.CSharp { /// /// Tokenizer for C# source code. @@ -76,7 +36,21 @@ namespace CIR public int col = 1; public int current_token; bool handle_get_set = false; + bool handle_remove_add = false; + bool handle_assembly = false; + // + // Whether tokens have been seen on this line + // + bool tokens_seen = false; + + // + // Whether a token has been seen on the file + // This is needed because `define' is not allowed to be used + // after a token has been seen. + // + bool any_token_seen = false; + // // Returns a verbose representation of the current location // @@ -89,15 +63,15 @@ namespace CIR else det = ""; - //return "Line: "+line+" Col: "+col + "\n" + + // return "Line: "+line+" Col: "+col + "\n" + // "VirtLine: "+ref_line + // " Token: "+current_token + " " + det; - return ref_name + " " + "(" + line + "," + col + ")"; + return ref_name + " " + "(" + line + "," + col + "), Token:" + current_token + " " + det; } } - public bool properties { + public bool PropertyParsing { get { return handle_get_set; } @@ -106,6 +80,26 @@ namespace CIR handle_get_set = value; } } + + public bool AssemblyTargetParsing { + get { + return handle_assembly; + } + + set { + handle_assembly = value; + } + } + + public bool EventParsing { + get { + return handle_remove_add; + } + + set { + handle_remove_add = value; + } + } // // Class variables @@ -117,9 +111,27 @@ namespace CIR // // Values for the associated token returned // - System.Text.StringBuilder number; int putback_char; Object val; + + // + // Pre-processor + // + Hashtable defines; + + const int TAKING = 1; + const int TAKEN_BEFORE = 2; + const int ELSE_SEEN = 4; + const int PARENT_TAKING = 8; + + // + // pre-processor if stack state: + // + Stack ifstack; + + static System.Text.StringBuilder id_builder; + static System.Text.StringBuilder string_builder; + static System.Text.StringBuilder number_builder; // // Details about the error encoutered by the tokenizer @@ -144,13 +156,14 @@ namespace CIR } } - static void initTokens () + static void InitTokens () { keywords = new Hashtable (); keywords.Add ("abstract", Token.ABSTRACT); keywords.Add ("as", Token.AS); keywords.Add ("add", Token.ADD); + keywords.Add ("assembly", Token.ASSEMBLY); keywords.Add ("base", Token.BASE); keywords.Add ("bool", Token.BOOL); keywords.Add ("break", Token.BREAK); @@ -187,7 +200,7 @@ namespace CIR keywords.Add ("interface", Token.INTERFACE); keywords.Add ("internal", Token.INTERNAL); keywords.Add ("is", Token.IS); - keywords.Add ("lock ", Token.LOCK ); + keywords.Add ("lock", Token.LOCK); keywords.Add ("long", Token.LONG); keywords.Add ("namespace", Token.NAMESPACE); keywords.Add ("new", Token.NEW); @@ -209,6 +222,7 @@ namespace CIR keywords.Add ("set", Token.SET); keywords.Add ("short", Token.SHORT); keywords.Add ("sizeof", Token.SIZEOF); + keywords.Add ("stackalloc", Token.STACKALLOC); keywords.Add ("static", Token.STATIC); keywords.Add ("string", Token.STRING); keywords.Add ("struct", Token.STRUCT); @@ -226,6 +240,7 @@ namespace CIR keywords.Add ("using", Token.USING); keywords.Add ("virtual", Token.VIRTUAL); keywords.Add ("void", Token.VOID); + keywords.Add ("volatile", Token.VOLATILE); keywords.Add ("while", Token.WHILE); } @@ -234,10 +249,13 @@ namespace CIR // static Tokenizer () { - initTokens (); + InitTokens (); csharp_format_info = new NumberFormatInfo (); csharp_format_info.CurrencyDecimalSeparator = "."; styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint; + id_builder = new System.Text.StringBuilder (); + string_builder = new System.Text.StringBuilder (); + number_builder = new System.Text.StringBuilder (); } bool is_keyword (string name) @@ -245,27 +263,43 @@ namespace CIR bool res; res = keywords.Contains (name); - if ((name == "get" || name == "set") && handle_get_set == false) + if (handle_get_set == false && (name == "get" || name == "set")) + return false; + if (handle_remove_add == false && (name == "remove" || name == "add")) + return false; + if (handle_assembly == false && (name == "assembly")) return false; return res; } - int getKeyword (string name) + int GetKeyword (string name) { return (int) (keywords [name]); } public Location Location { get { - return new Location (ref_name, col, ref_line); + return new Location (ref_line); } } - public Tokenizer (System.IO.Stream input, string fname) + public Tokenizer (System.IO.Stream input, string fname, ArrayList defs) { this.ref_name = fname; reader = new System.IO.StreamReader (input); putback_char = -1; + + if (defs != null){ + defines = new Hashtable (); + foreach (string def in defs) + defines [def] = true; + } + + // + // FIXME: This could be `Location.Push' but we have to + // find out why the MS compiler allows this + // + Mono.CSharp.Location.Push (fname); } bool is_identifier_start_character (char c) @@ -280,7 +314,6 @@ namespace CIR int is_punct (char c, ref bool doread) { - int idx = "{}[](),:;~+-*/%&|^!=<>?".IndexOf (c); int d; int t; @@ -329,7 +362,7 @@ namespace CIR else if (d == '=') t = Token.OP_SUB_ASSIGN; else if (d == '>') - return Token.OP_PTR; + t = Token.OP_PTR; else return Token.MINUS; doread = true; @@ -448,31 +481,36 @@ namespace CIR bool seen_digits = false; if (c != -1) - number.Append ((char) c); + number_builder.Append ((char) c); while ((d = peekChar ()) != -1){ if (Char.IsDigit ((char)d)){ - number.Append ((char) d); + number_builder.Append ((char) d); getChar (); seen_digits = true; } else break; } + return seen_digits; } + bool is_hex (char e) + { + return Char.IsDigit (e) || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f'); + } + void hex_digits (int c) { int d; if (c != -1) - number.Append ((char) c); + number_builder.Append ((char) c); while ((d = peekChar ()) != -1){ char e = Char.ToUpper ((char) d); - if (Char.IsDigit (e) || - (e >= 'A' && e <= 'F')){ - number.Append ((char) e); + if (is_hex (e)){ + number_builder.Append ((char) e); getChar (); } else break; @@ -482,7 +520,7 @@ namespace CIR int real_type_suffix (int c) { int t; - + switch (c){ case 'F': case 'f': t = Token.LITERAL_FLOAT; @@ -496,27 +534,102 @@ namespace CIR default: return Token.NONE; } - //getChar (); return t; } - int integer_type_suffix (int c) + int integer_type_suffix (ulong ul, int c) { - // FIXME: Handle U and L suffixes. - // We also need to see in which kind of - // Int the thing fits better according to the spec. + bool is_unsigned = false; + bool is_long = false; + + if (c != -1){ + bool scanning = true; + do { + switch (c){ + case 'U': case 'u': + if (is_unsigned) + scanning = false; + is_unsigned = true; + getChar (); + break; + + case 'l': + if (!is_unsigned){ + // + // if we have not seen anything in between + // report this error + // + Report.Warning ( + 78, Location, + "the 'l' suffix is easily confused with digit `1'," + + " use 'L' for clarity"); + } + goto case 'L'; + + case 'L': + if (is_long) + scanning = false; + is_long = true; + getChar (); + break; + + default: + scanning = false; + break; + } + c = peekChar (); + } while (scanning); + } + + if (is_long && is_unsigned){ + val = ul; + return Token.LITERAL_INTEGER; + } else if (is_unsigned){ + // uint if possible, or ulong else. + + if ((ul & 0xffffffff00000000) == 0) + val = (uint) ul; + else + val = ul; + } else if (is_long){ + // long if possible, ulong otherwise + if ((ul & 0x8000000000000000) != 0) + val = ul; + else + val = (long) ul; + } else { + // int, uint, long or ulong in that order + if ((ul & 0xffffffff00000000) == 0){ + uint ui = (uint) ul; + + if ((ui & 0x80000000) != 0) + val = ui; + else + val = (int) ui; + } else { + if ((ul & 0x8000000000000000) != 0) + val = ul; + else + val = (long) ul; + } + } return Token.LITERAL_INTEGER; } - - void adjust_int (int t) + + // + // given `c' as the next char in the input decide whether + // we need to convert to a special type, and then choose + // the best representation for the integer + // + int adjust_int (int c) { - val = new System.Int32(); - val = System.Int32.Parse (number.ToString (), 0); + ulong ul = System.UInt64.Parse (number_builder.ToString ()); + return integer_type_suffix (ul, c); } int adjust_real (int t) { - string s = number.ToString (); + string s = number_builder.ToString (); switch (t){ case Token.LITERAL_DECIMAL: @@ -551,10 +664,9 @@ namespace CIR int is_number (int c) { bool is_real = false; - number = new System.Text.StringBuilder (); int type; - number.Length = 0; + number_builder.Length = 0; if (Char.IsDigit ((char)c)){ if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){ @@ -562,24 +674,10 @@ namespace CIR getChar (); hex_digits (-1); - string s = number.ToString (); + string s = number_builder.ToString (); ul = System.UInt64.Parse (s, NumberStyles.HexNumber); - if ((ul & 0xffffffff00000000) == 0){ - uint ui = (uint) ul; - - if ((ui & 0x80000000) != 0) - val = ui; - else - val = (int) ui; - } else { - if ((ul & 0x8000000000000000) != 0) - val = ul; - else - val = (long) ul; - } - - return integer_type_suffix (peekChar ()); + return integer_type_suffix (ul, peekChar ()); } decimal_digits (c); c = getChar (); @@ -592,52 +690,89 @@ namespace CIR if (c == '.'){ if (decimal_digits ('.')){ is_real = true; - c = peekChar (); + c = getChar (); } else { putback ('.'); - number.Length -= 1; - adjust_int (Token.LITERAL_INTEGER); - return Token.LITERAL_INTEGER; + number_builder.Length -= 1; + return adjust_int (-1); } } if (c == 'e' || c == 'E'){ is_real = true; - number.Append ("e"); - getChar (); + number_builder.Append ("e"); + c = getChar (); - c = peekChar (); if (c == '+'){ - number.Append ((char) c); - getChar (); - c = peekChar (); + number_builder.Append ((char) c); + c = getChar (); } else if (c == '-'){ - number.Append ((char) c); - getChar (); - c = peekChar (); + number_builder.Append ((char) c); + c = getChar (); } decimal_digits (-1); - c = peekChar (); + c = getChar (); } type = real_type_suffix (c); if (type == Token.NONE && !is_real){ - type = integer_type_suffix (c); - adjust_int (type); putback (c); - return type; - } else + return adjust_int (c); + } else is_real = true; + if (type == Token.NONE){ + putback (c); + } + if (is_real) return adjust_real (type); Console.WriteLine ("This should not be reached"); throw new Exception ("Is Number should never reach this point"); } + + // + // Accepts exactly count (4 or 8) hex, no more no less + // + int getHex (int count, out bool error) + { + int [] buffer = new int [8]; + int i; + int total = 0; + int c; + char e; + int top = count != -1 ? count : 4; + getChar (); + error = false; + for (i = 0; i < top; i++){ + c = getChar (); + e = Char.ToUpper ((char) c); + + if (!is_hex (e)){ + error = true; + return 0; + } + if (Char.IsDigit (e)) + c = (int) e - (int) '0'; + else + c = (int) e - (int) 'A'; + total = (total * 16) + c; + if (count == -1){ + int p = peekChar (); + if (p == -1) + break; + if (!is_hex ((char)p)) + break; + } + } + return total; + } + int escape (int c) { + bool error; int d; int v; @@ -657,7 +792,7 @@ namespace CIR case 'v': v = '\v'; break; case 'r': - v = 'c'; break; + v = '\r'; break; case '\\': v = '\\'; break; case 'f': @@ -668,8 +803,23 @@ namespace CIR v = '"'; break; case '\'': v = '\''; break; + case 'x': + v = getHex (-1, out error); + if (error) + goto default; + return v; + case 'u': + v = getHex (4, out error); + if (error) + goto default; + return v; + case 'U': + v = getHex (8, out error); + if (error) + goto default; + return v; default: - error_details = "cs1009: Unrecognized escape sequence " + (char)d; + Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d); return -1; } getChar (); @@ -722,6 +872,446 @@ namespace CIR current_token = xtoken (); return current_token; } + + static StringBuilder static_cmd_arg = new System.Text.StringBuilder (); + + void get_cmd_arg (out string cmd, out string arg) + { + int c; + + tokens_seen = false; + arg = ""; + static_cmd_arg.Length = 0; + + while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t')){ + if (c == '\r') + continue; + static_cmd_arg.Append ((char) c); + } + + cmd = static_cmd_arg.ToString (); + + if (c == '\n'){ + line++; + ref_line++; + return; + } + + // skip over white space + while ((c = getChar ()) != -1 && (c != '\n') && ((c == ' ') || (c == '\t'))) + ; + + if (c == '\n'){ + line++; + ref_line++; + return; + } + + static_cmd_arg.Length = 0; + static_cmd_arg.Append ((char) c); + + while ((c = getChar ()) != -1 && (c != '\n')){ + if (c == '\r') + continue; + static_cmd_arg.Append ((char) c); + } + + if (c == '\n'){ + line++; + ref_line++; + } + arg = static_cmd_arg.ToString ().Trim (); + } + + // + // Handles the #line directive + // + bool PreProcessLine (string arg) + { + if (arg == "") + return false; + + if (arg == "default"){ + line = ref_line = line; + return false; + } + + try { + int pos; + + if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){ + ref_line = System.Int32.Parse (arg.Substring (0, pos)); + pos++; + + char [] quotes = { '\"' }; + + ref_name = arg.Substring (pos). Trim(quotes); + } else { + ref_line = System.Int32.Parse (arg); + } + } catch { + return false; + } + + return true; + } + + // + // Handles #define and #undef + // + void PreProcessDefinition (bool is_define, string arg) + { + if (arg == "" || arg == "true" || arg == "false"){ + Report.Error(1001, Location, "Missing identifer to pre-processor directive"); + return; + } + + if (is_define){ + if (defines == null) + defines = new Hashtable (); + defines [arg] = 1; + } else { + if (defines == null) + return; + if (defines.Contains (arg)) + defines.Remove (arg); + } + } + + bool eval_val (string s) + { + if (s == "true") + return true; + if (s == "false") + return false; + + if (defines == null) + return false; + if (defines.Contains (s)) + return true; + + return false; + } + + bool pp_primary (ref string s) + { + s = s.Trim (); + int len = s.Length; + + if (len > 0){ + char c = s [0]; + + if (c == '('){ + s = s.Substring (1); + bool val = pp_expr (ref s); + if (s.Length > 0 && s [0] == ')') + return val; + Error_InvalidDirective (); + return false; + } + + if (Char.IsLetter (c) || c == '_'){ + int j = 1; + + while (j < len){ + c = s [j]; + + if (Char.IsLetter (c) || Char.IsDigit (c) || c == '_'){ + j++; + continue; + } + bool v = eval_val (s.Substring (0, j)); + s = s.Substring (j); + return v; + } + bool vv = eval_val (s); + s = ""; + return vv; + } + } + Error_InvalidDirective (); + return false; + } + + bool pp_unary (ref string s) + { + s = s.Trim (); + int len = s.Length; + + if (len > 0){ + if (s [0] == '!'){ + if (len > 1 && s [1] == '='){ + Error_InvalidDirective (); + return false; + } + s = s.Substring (1); + return ! pp_primary (ref s); + } else + return pp_primary (ref s); + } else { + Error_InvalidDirective (); + return false; + } + } + + bool pp_eq (ref string s) + { + bool va = pp_unary (ref s); + + s = s.Trim (); + int len = s.Length; + if (len > 0){ + if (s [0] == '='){ + if (len > 2 && s [1] == '='){ + s = s.Substring (2); + return va == pp_unary (ref s); + } else { + Error_InvalidDirective (); + return false; + } + } else if (s [0] == '!' && len > 1 && s [1] == '='){ + s = s.Substring (2); + + return va != pp_unary (ref s); + + } + } + + return va; + + } + + bool pp_and (ref string s) + { + bool va = pp_eq (ref s); + + s = s.Trim (); + int len = s.Length; + if (len > 0){ + if (s [0] == '&'){ + if (len > 2 && s [1] == '&'){ + s = s.Substring (2); + return va && pp_eq (ref s); + } else { + Error_InvalidDirective (); + return false; + } + } + } + return va; + } + + // + // Evaluates an expression for `#if' or `#elif' + // + bool pp_expr (ref string s) + { + bool va = pp_and (ref s); + + s = s.Trim (); + int len = s.Length; + if (len > 0){ + if (s [0] == '|'){ + if (len > 2 && s [1] == '|'){ + s = s.Substring (2); + return va || pp_and (ref s); + } else { + Error_InvalidDirective (); + return false; + } + } else { + Error_InvalidDirective (); + return false; + } + } + + return va; + } + + bool eval (string s) + { + bool v = pp_expr (ref s); + + return v; + } + + void Error_InvalidDirective () + { + Report.Error (1517, Location, "Invalid pre-processor directive"); + } + + void Error_UnexpectedDirective (string extra) + { + Report.Error ( + 1028, Location, + "Unexpected processor directive (" + extra + ")"); + } + + void Error_TokensSeen () + { + Report.Error ( + 1032, Location, + "Cannot define or undefine pre-processor symbols after a token in the file"); + } + + // + // if true, then the code continues processing the code + // if false, the code stays in a loop until another directive is + // reached. + // + bool handle_preprocessing_directive (bool caller_is_taking) + { + char [] blank = { ' ', '\t' }; + string cmd, arg; + + get_cmd_arg (out cmd, out arg); + + // + // The first group of pre-processing instructions is always processed + // + switch (cmd){ + case "line": + if (!PreProcessLine (arg)) + Report.Error ( + 1576, Location, + "Argument to #line directive is missing or invalid"); + return true; + + case "region": + arg = "true"; + goto case "if"; + + case "endregion": + goto case "endif"; + + case "if": + if (arg == ""){ + Error_InvalidDirective (); + return true; + } + bool taking = false; + if (ifstack == null) + ifstack = new Stack (); + + if (ifstack.Count == 0){ + taking = true; + } else { + int state = (int) ifstack.Peek (); + if ((state & TAKING) != 0) + taking = true; + } + + if (eval (arg) && taking){ + ifstack.Push (TAKING | TAKEN_BEFORE | PARENT_TAKING); + return true; + } else { + ifstack.Push (taking ? PARENT_TAKING : 0); + return false; + } + + case "endif": + if (ifstack == null || ifstack.Count == 0){ + Error_UnexpectedDirective ("no #if for this #endif"); + return true; + } else { + ifstack.Pop (); + if (ifstack.Count == 0) + return true; + else { + int state = (int) ifstack.Peek (); + + if ((state & TAKING) != 0) + return true; + else + return false; + } + } + + case "elif": + if (ifstack == null || ifstack.Count == 0){ + Error_UnexpectedDirective ("no #if for this #elif"); + return true; + } else { + int state = (int) ifstack.Peek (); + + if ((state & ELSE_SEEN) != 0){ + Error_UnexpectedDirective ("#elif not valid after #else"); + return true; + } + + if ((state & (TAKEN_BEFORE | TAKING)) != 0) + return false; + + if (eval (arg) && ((state & PARENT_TAKING) != 0)){ + state = (int) ifstack.Pop (); + ifstack.Push (state | TAKING | TAKEN_BEFORE); + return true; + } else + return false; + } + + case "else": + if (ifstack == null || ifstack.Count == 0){ + Report.Error ( + 1028, Location, + "Unexpected processor directive (no #if for this #else)"); + return true; + } else { + int state = (int) ifstack.Peek (); + + if ((state & ELSE_SEEN) != 0){ + Error_UnexpectedDirective ("#else within #else"); + return true; + } + + ifstack.Pop (); + ifstack.Push (state | ELSE_SEEN); + + if ((state & TAKEN_BEFORE) == 0){ + if ((state & PARENT_TAKING) != 0) + return true; + else + return false; + } + return false; + } + } + + // + // These are only processed if we are in a `taking' block + // + if (!caller_is_taking) + return false; + + switch (cmd){ + case "define": + if (any_token_seen){ + Error_TokensSeen (); + return true; + } + PreProcessDefinition (true, arg); + return true; + + case "undef": + if (any_token_seen){ + Error_TokensSeen (); + return true; + } + PreProcessDefinition (false, arg); + return true; + + case "error": + Report.Error (1029, Location, "#error: '" + arg + "'"); + return true; + + case "warning": + Report.Warning (1030, Location, "#warning: '" + arg + "'"); + return true; + } + + Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")"); + return true; + } public int xtoken () { @@ -731,42 +1321,52 @@ namespace CIR int c; val = null; + // optimization: eliminate col and implement #directive semantic correctly. for (;(c = getChar ()) != -1; col++) { - - if (is_identifier_start_character ((char) c)){ - System.Text.StringBuilder id = new System.Text.StringBuilder (); + if (Char.IsLetter ((char)c) || c == '_'){ string ids; - - id.Append ((char) c); + + id_builder.Length = 0; + tokens_seen = true; + id_builder.Append ((char) c); while ((c = peekChar ()) != -1) { if (is_identifier_part_character ((char) c)){ - id.Append ((char)getChar ()); + id_builder.Append ((char)getChar ()); col++; } else break; } - ids = id.ToString (); + ids = id_builder.ToString (); if (!is_keyword (ids) || allow_keyword_as_ident) { val = ids; + if (ids.Length > 512){ + Report.Error ( + 645, Location, + "Identifier too long (limit is 512 chars)"); + } + allow_keyword_as_ident = false; return Token.IDENTIFIER; } // true, false and null are in the hash anyway. - return getKeyword (ids); + return GetKeyword (ids); } if (c == '.'){ + tokens_seen = true; if (Char.IsDigit ((char) peekChar ())) return is_number (c); return Token.DOT; } - if (Char.IsDigit ((char) c)) + if (Char.IsDigit ((char) c)){ + tokens_seen = true; return is_number (c); + } // Handle double-slash comments. if (c == '/'){ @@ -778,6 +1378,9 @@ namespace CIR col++; line++; ref_line++; + col = 0; + any_token_seen |= tokens_seen; + tokens_seen = false; continue; } else if (d == '*'){ getChar (); @@ -791,42 +1394,54 @@ namespace CIR if (d == '\n'){ line++; ref_line++; + col = 0; + any_token_seen |= tokens_seen; + tokens_seen = false; } - col++; } continue; } } /* For now, ignore pre-processor commands */ - if (col == 1 && c == '#'){ - System.Text.StringBuilder s = new System.Text.StringBuilder (); + // FIXME: In C# the '#' is not limited to appear + // on the first column. + if (c == '#' && !tokens_seen){ + bool cont = true; - while ((c = getChar ()) != -1 && (c != '\n')){ - s.Append ((char) c); - } - if (String.Compare (s.ToString (), 0, "line", 0, 4) == 0){ - string arg = s.ToString ().Substring (5); - int pos; - - if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){ - ref_line = System.Int32.Parse (arg.Substring (0, pos)); - pos++; - - char [] quotes = { '\"' }; + start_again: + + cont = handle_preprocessing_directive (cont); - ref_name = arg.Substring (pos); - ref_name.TrimStart (quotes); - ref_name.TrimEnd (quotes); - } else - ref_line = System.Int32.Parse (arg); + if (cont){ + col = 0; + continue; } - line++; - ref_line++; + col = 1; + + bool skipping = false; + for (;(c = getChar ()) != -1; col++){ + if (c == '\n'){ + col = 0; + line++; + ref_line++; + skipping = false; + } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r') + continue; + else if (c != '#') + skipping = true; + if (c == '#' && !skipping) + goto start_again; + } + any_token_seen |= tokens_seen; + tokens_seen = false; + if (c == -1) + Report.Error (1027, Location, "#endif expected"); continue; } if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){ + tokens_seen = true; if (doread){ getChar (); col++; @@ -835,25 +1450,37 @@ namespace CIR } if (c == '"'){ - System.Text.StringBuilder s = new System.Text.StringBuilder (); - + string_builder.Length = 0; + + tokens_seen = true; + while ((c = getChar ()) != -1){ if (c == '"'){ - val = s.ToString (); + if (allow_keyword_as_ident && peekChar () == '"'){ + string_builder.Append ((char) c); + getChar (); + continue; + } + allow_keyword_as_ident = false; + val = string_builder.ToString (); return Token.LITERAL_STRING; } - c = escape (c); - if (c == -1) - return Token.ERROR; - s.Append ((char) c); + if (!allow_keyword_as_ident){ + c = escape (c); + if (c == -1) + return Token.ERROR; + } + string_builder.Append ((char) c); } } if (c == '\''){ c = getChar (); + tokens_seen = true; if (c == '\''){ - error_details = "CS1011: Empty character literal"; + error_details = "Empty character literal"; + Report.Error (1011, Location, error_details); return Token.ERROR; } c = escape (c); @@ -862,8 +1489,11 @@ namespace CIR val = new System.Char (); val = (char) c; c = getChar (); + if (c != '\''){ - error_details = "CS1012: Too many characters in character literal"; + error_details = "Too many characters in character literal"; + Report.Error (1012, Location, error_details); + // Try to recover, read until newline or next "'" while ((c = getChar ()) != -1){ if (c == '\n' || c == '\'') @@ -880,16 +1510,19 @@ namespace CIR line++; ref_line++; col = 0; + any_token_seen |= tokens_seen; + tokens_seen = false; continue; } + if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r'){ if (c == '\t') col = (((col + 8) / 8) * 8) - 1; - continue; } if (c == '@'){ + tokens_seen = true; allow_keyword_as_ident = true; continue; } @@ -899,35 +1532,10 @@ namespace CIR return Token.ERROR; } + if (ifstack != null && ifstack.Count > 1) + Report.Error (1027, Location, "#endif expected"); return Token.EOF; } } - - public class Location { - public readonly string Name; - public readonly int Col; - public readonly int Row; - - public Location (string name, int col, int row) - { - Name = name; - Col = col; - Row = row; - } - - // - // Whether the Location is Null - // - static public bool IsNull (Location l) - { - return l == null; - } - - static public Location Null { - get { - return null; - } - } - } - } +