Fix typos.
[mono.git] / mcs / mcs / cs-tokenizer.cs
index 8ca476f7e1fec4bc8d7a67f06ecb22aceeb2da00..582266be494d938b444181cba1042d42c601bf5e 100755 (executable)
@@ -10,7 +10,9 @@
 //\r
 \r
 /*\r
-  Make sure we accept the proper Unicode ranges, per the spec.\r
+ * TODO:\r
+ *   Make sure we accept the proper Unicode ranges, per the spec.\r
+ *   Report error 1032\r
 */\r
 \r
 using System;\r
@@ -35,6 +37,19 @@ namespace Mono.CSharp
                public int current_token;\r
                bool handle_get_set = false;\r
                bool handle_remove_add = false;\r
+               bool handle_assembly = false;\r
+\r
+               //\r
+               // Whether tokens have been seen on this line\r
+               //\r
+               bool tokens_seen = false;\r
+\r
+               //\r
+               // Whether a token has been seen on the file\r
+               // This is needed because `define' is not allowed to be used\r
+               // after a token has been seen.\r
+               //\r
+               bool any_token_seen = false;\r
                \r
                //\r
                // Returns a verbose representation of the current location\r
@@ -66,6 +81,16 @@ namespace Mono.CSharp
                        }\r
                 }\r
 \r
+               public bool AssemblyTargetParsing {\r
+                       get {\r
+                               return handle_assembly;\r
+                       }\r
+\r
+                       set {\r
+                               handle_assembly = value;\r
+                       }\r
+               }\r
+\r
                public bool EventParsing {\r
                        get {\r
                                return handle_remove_add;\r
@@ -86,7 +111,6 @@ namespace Mono.CSharp
                //\r
                // Values for the associated token returned\r
                //\r
-               System.Text.StringBuilder number;\r
                int putback_char;\r
                Object val;\r
 \r
@@ -104,6 +128,10 @@ namespace Mono.CSharp
                // pre-processor if stack state:\r
                //\r
                Stack ifstack;\r
+\r
+               static System.Text.StringBuilder id_builder;\r
+               static System.Text.StringBuilder string_builder;\r
+               static System.Text.StringBuilder number_builder;\r
                \r
                //\r
                // Details about the error encoutered by the tokenizer\r
@@ -135,6 +163,7 @@ namespace Mono.CSharp
                        keywords.Add ("abstract", Token.ABSTRACT);\r
                        keywords.Add ("as", Token.AS);\r
                        keywords.Add ("add", Token.ADD);\r
+                       keywords.Add ("assembly", Token.ASSEMBLY);\r
                        keywords.Add ("base", Token.BASE);\r
                        keywords.Add ("bool", Token.BOOL);\r
                        keywords.Add ("break", Token.BREAK);\r
@@ -193,6 +222,7 @@ namespace Mono.CSharp
                        keywords.Add ("set", Token.SET);\r
                        keywords.Add ("short", Token.SHORT);\r
                        keywords.Add ("sizeof", Token.SIZEOF);\r
+                       keywords.Add ("stackalloc", Token.STACKALLOC);\r
                        keywords.Add ("static", Token.STATIC);\r
                        keywords.Add ("string", Token.STRING);\r
                        keywords.Add ("struct", Token.STRUCT);\r
@@ -210,6 +240,7 @@ namespace Mono.CSharp
                        keywords.Add ("using", Token.USING);\r
                        keywords.Add ("virtual", Token.VIRTUAL);\r
                        keywords.Add ("void", Token.VOID);\r
+                       keywords.Add ("volatile", Token.VOLATILE);\r
                        keywords.Add ("while", Token.WHILE);\r
                }\r
 \r
@@ -222,6 +253,9 @@ namespace Mono.CSharp
                        csharp_format_info = new NumberFormatInfo ();\r
                        csharp_format_info.CurrencyDecimalSeparator = ".";\r
                        styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint;\r
+                       id_builder = new System.Text.StringBuilder ();\r
+                       string_builder = new System.Text.StringBuilder ();\r
+                       number_builder = new System.Text.StringBuilder ();\r
                }\r
 \r
                bool is_keyword (string name)\r
@@ -233,6 +267,8 @@ namespace Mono.CSharp
                                return false;\r
                        if (handle_remove_add == false && (name == "remove" || name == "add"))\r
                                return false;\r
+                       if (handle_assembly == false && (name == "assembly"))\r
+                               return false;\r
                        return res;\r
                }\r
 \r
@@ -258,8 +294,12 @@ namespace Mono.CSharp
                                foreach (string def in defs)\r
                                        defines [def] = true;\r
                        }\r
-                       \r
-                       Location.Push (fname);\r
+\r
+                       //\r
+                       // FIXME: This could be `Location.Push' but we have to\r
+                       // find out why the MS compiler allows this\r
+                       //\r
+                       Mono.CSharp.Location.Push (fname);\r
                }\r
 \r
                bool is_identifier_start_character (char c)\r
@@ -274,7 +314,6 @@ namespace Mono.CSharp
 \r
                int is_punct (char c, ref bool doread)\r
                {\r
-                       int idx = "{}[](),:;~+-*/%&|^!=<>?".IndexOf (c);\r
                        int d;\r
                        int t;\r
 \r
@@ -323,7 +362,7 @@ namespace Mono.CSharp
                                else if (d == '=')\r
                                        t = Token.OP_SUB_ASSIGN;\r
                                else if (d == '>')\r
-                                       return Token.OP_PTR;\r
+                                       t = Token.OP_PTR;\r
                                else\r
                                        return Token.MINUS;\r
                                doread = true;\r
@@ -442,30 +481,36 @@ namespace Mono.CSharp
                        bool seen_digits = false;\r
                        \r
                        if (c != -1)\r
-                               number.Append ((char) c);\r
+                               number_builder.Append ((char) c);\r
                        \r
                        while ((d = peekChar ()) != -1){\r
                                if (Char.IsDigit ((char)d)){\r
-                                       number.Append ((char) d);\r
+                                       number_builder.Append ((char) d);\r
                                        getChar ();\r
                                        seen_digits = true;\r
                                } else\r
                                        break;\r
                        }\r
+                       \r
                        return seen_digits;\r
                }\r
 \r
+               bool is_hex (char e)\r
+               {\r
+                       return Char.IsDigit (e) || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');\r
+               }\r
+               \r
                void hex_digits (int c)\r
                {\r
                        int d;\r
 \r
                        if (c != -1)\r
-                               number.Append ((char) c);\r
+                               number_builder.Append ((char) c);\r
                        while ((d = peekChar ()) != -1){\r
                                char e = Char.ToUpper ((char) d);\r
                                \r
-                               if (Char.IsDigit (e) || (e >= 'A' && e <= 'F')){\r
-                                       number.Append ((char) e);\r
+                               if (is_hex (e)){\r
+                                       number_builder.Append ((char) e);\r
                                        getChar ();\r
                                } else\r
                                        break;\r
@@ -489,7 +534,6 @@ namespace Mono.CSharp
                        default:\r
                                return Token.NONE;\r
                        }\r
-                       getChar ();\r
                        return t;\r
                }\r
 \r
@@ -579,13 +623,13 @@ namespace Mono.CSharp
                //\r
                int adjust_int (int c)\r
                {\r
-                       ulong ul = System.UInt64.Parse (number.ToString ());\r
+                       ulong ul = System.UInt64.Parse (number_builder.ToString ());\r
                        return integer_type_suffix (ul, c);\r
                }\r
 \r
                int adjust_real (int t)\r
                {\r
-                       string s = number.ToString ();\r
+                       string s = number_builder.ToString ();\r
 \r
                        switch (t){\r
                        case Token.LITERAL_DECIMAL:\r
@@ -620,10 +664,9 @@ namespace Mono.CSharp
                int is_number (int c)\r
                {\r
                        bool is_real = false;\r
-                       number = new System.Text.StringBuilder ();\r
                        int type;\r
 \r
-                       number.Length = 0;\r
+                       number_builder.Length = 0;\r
 \r
                        if (Char.IsDigit ((char)c)){\r
                                if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){\r
@@ -631,7 +674,7 @@ namespace Mono.CSharp
                                        getChar ();\r
                                        hex_digits (-1);\r
 \r
-                                       string s = number.ToString ();\r
+                                       string s = number_builder.ToString ();\r
 \r
                                        ul = System.UInt64.Parse (s, NumberStyles.HexNumber);\r
                                        return integer_type_suffix (ul, peekChar ());\r
@@ -647,49 +690,89 @@ namespace Mono.CSharp
                        if (c == '.'){\r
                                if (decimal_digits ('.')){\r
                                        is_real = true;\r
-                                       c = peekChar ();\r
+                                       c = getChar ();\r
                                } else {\r
                                        putback ('.');\r
-                                       number.Length -= 1;\r
+                                       number_builder.Length -= 1;\r
                                        return adjust_int (-1);\r
                                }\r
                        }\r
                        \r
                        if (c == 'e' || c == 'E'){\r
                                is_real = true;\r
-                               number.Append ("e");\r
-                               getChar ();\r
+                               number_builder.Append ("e");\r
+                               c = getChar ();\r
                                \r
-                               c = peekChar ();\r
                                if (c == '+'){\r
-                                       number.Append ((char) c);\r
-                                       getChar ();\r
-                                       c = peekChar ();\r
+                                       number_builder.Append ((char) c);\r
+                                       c = getChar ();\r
                                } else if (c == '-'){\r
-                                       number.Append ((char) c);\r
-                                       getChar ();\r
-                                       c = peekChar ();\r
+                                       number_builder.Append ((char) c);\r
+                                       c = getChar ();\r
                                }\r
                                decimal_digits (-1);\r
-                               c = peekChar ();\r
+                               c = getChar ();\r
                        }\r
 \r
                        type = real_type_suffix (c);\r
                        if (type == Token.NONE && !is_real){\r
                                putback (c);\r
                                return adjust_int (c);\r
-                       } else\r
+                       } else \r
                                is_real = true;\r
 \r
+                       if (type == Token.NONE){\r
+                               putback (c);\r
+                       }\r
+                       \r
                        if (is_real)\r
                                return adjust_real (type);\r
 \r
                        Console.WriteLine ("This should not be reached");\r
                        throw new Exception ("Is Number should never reach this point");\r
                }\r
+\r
+               //\r
+               // Accepts exactly count (4 or 8) hex, no more no less\r
+               //\r
+               int getHex (int count, out bool error)\r
+               {\r
+                       int [] buffer = new int [8];\r
+                       int i;\r
+                       int total = 0;\r
+                       int c;\r
+                       char e;\r
+                       int top = count != -1 ? count : 4;\r
                        \r
+                       getChar ();\r
+                       error = false;\r
+                       for (i = 0; i < top; i++){\r
+                               c = getChar ();\r
+                               e = Char.ToUpper ((char) c);\r
+                               \r
+                               if (!is_hex (e)){\r
+                                       error = true;\r
+                                       return 0;\r
+                               }\r
+                               if (Char.IsDigit (e))\r
+                                       c = (int) e - (int) '0';\r
+                               else\r
+                                       c = (int) e - (int) 'A';\r
+                               total = (total * 16) + c;\r
+                               if (count == -1){\r
+                                       int p = peekChar ();\r
+                                       if (p == -1)\r
+                                               break;\r
+                                       if (!is_hex ((char)p))\r
+                                               break;\r
+                               }\r
+                       }\r
+                       return total;\r
+               }\r
+\r
                int escape (int c)\r
                {\r
+                       bool error;\r
                        int d;\r
                        int v;\r
 \r
@@ -709,7 +792,7 @@ namespace Mono.CSharp
                        case 'v':\r
                                v = '\v'; break;\r
                        case 'r':\r
-                               v = 'c'; break;\r
+                               v = '\r'; break;\r
                        case '\\':\r
                                v = '\\'; break;\r
                        case 'f':\r
@@ -720,8 +803,23 @@ namespace Mono.CSharp
                                v = '"'; break;\r
                        case '\'':\r
                                v = '\''; break;\r
+                       case 'x':\r
+                               v = getHex (-1, out error);\r
+                               if (error)\r
+                                       goto default;\r
+                               return v;\r
+                       case 'u':\r
+                               v = getHex (4, out error);\r
+                               if (error)\r
+                                       goto default;\r
+                               return v;\r
+                       case 'U':\r
+                               v = getHex (8, out error);\r
+                               if (error)\r
+                                       goto default;\r
+                               return v;\r
                        default:\r
-                               error_details = "cs1009: Unrecognized escape sequence " + (char)d;\r
+                               Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);\r
                                return -1;\r
                        }\r
                        getChar ();\r
@@ -781,11 +879,15 @@ namespace Mono.CSharp
                {\r
                        int c;\r
                        \r
+                       tokens_seen = false;\r
                        arg = "";\r
                        static_cmd_arg.Length = 0;\r
                                \r
-                       while ((c = getChar ()) != -1 && (c != '\n') && (c != ' '))\r
+                       while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t')){\r
+                               if (c == '\r')\r
+                                       continue;\r
                                static_cmd_arg.Append ((char) c);\r
+                       }\r
 \r
                        cmd = static_cmd_arg.ToString ();\r
 \r
@@ -796,7 +898,7 @@ namespace Mono.CSharp
                        }\r
 \r
                        // skip over white space\r
-                       while ((c = getChar ()) != -1 && (c != '\n') && (c == ' '))\r
+                       while ((c = getChar ()) != -1 && (c != '\n') && ((c == ' ') || (c == '\t')))\r
                                ;\r
 \r
                        if (c == '\n'){\r
@@ -808,8 +910,11 @@ namespace Mono.CSharp
                        static_cmd_arg.Length = 0;\r
                        static_cmd_arg.Append ((char) c);\r
                        \r
-                       while ((c = getChar ()) != -1 && (c != '\n'))\r
+                       while ((c = getChar ()) != -1 && (c != '\n')){\r
+                               if (c == '\r')\r
+                                       continue;\r
                                static_cmd_arg.Append ((char) c);\r
+                       }\r
 \r
                        if (c == '\n'){\r
                                line++;\r
@@ -840,9 +945,7 @@ namespace Mono.CSharp
                                        \r
                                        char [] quotes = { '\"' };\r
                                        \r
-                                       ref_name = arg.Substring (pos);\r
-                                       ref_name.TrimStart (quotes);\r
-                                       ref_name.TrimEnd (quotes);\r
+                                       ref_name = arg.Substring (pos). Trim(quotes);\r
                                } else {\r
                                        ref_line = System.Int32.Parse (arg);\r
                                }\r
@@ -889,34 +992,183 @@ namespace Mono.CSharp
 \r
                        return false;\r
                }\r
+\r
+               bool pp_primary (ref string s)\r
+               {\r
+                       s = s.Trim ();\r
+                       int len = s.Length;\r
+\r
+                       if (len > 0){\r
+                               char c = s [0];\r
+                               \r
+                               if (c == '('){\r
+                                       s = s.Substring (1);\r
+                                       bool val = pp_expr (ref s);\r
+                                       if (s.Length > 0 && s [0] == ')')\r
+                                               return val;\r
+                                       Error_InvalidDirective ();\r
+                                       return false;\r
+                               }\r
+                               \r
+                               if (Char.IsLetter (c) || c == '_'){\r
+                                       int j = 1;\r
+\r
+                                       while (j < len){\r
+                                               c = s [j];\r
+                                               \r
+                                               if (Char.IsLetter (c) || Char.IsDigit (c) || c == '_'){\r
+                                                       j++;\r
+                                                       continue;\r
+                                               }\r
+                                               bool v = eval_val (s.Substring (0, j));\r
+                                               s = s.Substring (j);\r
+                                               return v;\r
+                                       }\r
+                                       bool vv = eval_val (s);\r
+                                       s = "";\r
+                                       return vv;\r
+                               }\r
+                       }\r
+                       Error_InvalidDirective ();\r
+                       return false;\r
+               }\r
+               \r
+               bool pp_unary (ref string s)\r
+               {\r
+                       s = s.Trim ();\r
+                       int len = s.Length;\r
+\r
+                       if (len > 0){\r
+                               if (s [0] == '!'){\r
+                                       if (len > 1 && s [1] == '='){\r
+                                               Error_InvalidDirective ();\r
+                                               return false;\r
+                                       }\r
+                                       s = s.Substring (1);\r
+                                       return ! pp_primary (ref s);\r
+                               } else\r
+                                       return pp_primary (ref s);\r
+                       } else {\r
+                               Error_InvalidDirective ();\r
+                               return false;\r
+                       }\r
+               }\r
+               \r
+               bool pp_eq (ref string s)\r
+               {\r
+                       bool va = pp_unary (ref s);\r
+\r
+                       s = s.Trim ();\r
+                       int len = s.Length;\r
+                       if (len > 0){\r
+                               if (s [0] == '='){\r
+                                       if (len > 2 && s [1] == '='){\r
+                                               s = s.Substring (2);\r
+                                               return va == pp_unary (ref s);\r
+                                       } else {\r
+                                               Error_InvalidDirective ();\r
+                                               return false;\r
+                                       }\r
+                               } else if (s [0] == '!' && len > 1 && s [1] == '='){\r
+                                       s = s.Substring (2);\r
+\r
+                                       return va != pp_unary (ref s);\r
+\r
+                               }\r
+                       }\r
+\r
+                       return va;\r
+                               \r
+               }\r
+               \r
+               bool pp_and (ref string s)\r
+               {\r
+                       bool va = pp_eq (ref s);\r
+\r
+                       s = s.Trim ();\r
+                       int len = s.Length;\r
+                       if (len > 0){\r
+                               if (s [0] == '&'){\r
+                                       if (len > 2 && s [1] == '&'){\r
+                                               s = s.Substring (2);\r
+                                               return va && pp_eq (ref s);\r
+                                       } else {\r
+                                               Error_InvalidDirective ();\r
+                                               return false;\r
+                                       }\r
+                               } \r
+                       }\r
+                       return va;\r
+               }\r
                \r
                //\r
                // Evaluates an expression for `#if' or `#elif'\r
                //\r
+               bool pp_expr (ref string s)\r
+               {\r
+                       bool va = pp_and (ref s);\r
+\r
+                       s = s.Trim ();\r
+                       int len = s.Length;\r
+                       if (len > 0){\r
+                               if (s [0] == '|'){\r
+                                       if (len > 2 && s [1] == '|'){\r
+                                               s = s.Substring (2);\r
+                                               return va || pp_and (ref s);\r
+                                       } else {\r
+                                               Error_InvalidDirective ();\r
+                                               return false;\r
+                                       }\r
+                               } else {\r
+                                       Error_InvalidDirective ();\r
+                                       return false;\r
+                               }\r
+                       }\r
+\r
+                       return va;\r
+               }\r
+\r
                bool eval (string s)\r
                {\r
-                       return eval_val (s);\r
+                       bool v = pp_expr (ref s);\r
+\r
+                       return v;\r
+               }\r
+               \r
+               void Error_InvalidDirective ()\r
+               {\r
+                       Report.Error (1517, Location, "Invalid pre-processor directive");\r
                }\r
 \r
-               void report1028 (string extra)\r
+               void Error_UnexpectedDirective (string extra)\r
                {\r
                        Report.Error (\r
                                1028, Location,\r
                                "Unexpected processor directive (" + extra + ")");\r
                }\r
+\r
+               void Error_TokensSeen ()\r
+               {\r
+                       Report.Error (\r
+                               1032, Location,\r
+                               "Cannot define or undefine pre-processor symbols after a token in the file");\r
+               }\r
                \r
                //\r
                // if true, then the code continues processing the code\r
                // if false, the code stays in a loop until another directive is\r
                // reached.\r
                //\r
-               bool handle_preprocessing_directive ()\r
+               bool handle_preprocessing_directive (bool caller_is_taking)\r
                {\r
                        char [] blank = { ' ', '\t' };\r
                        string cmd, arg;\r
                        \r
                        get_cmd_arg (out cmd, out arg);\r
 \r
+                       //\r
+                       // The first group of pre-processing instructions is always processed\r
+                       //\r
                        switch (cmd){\r
                        case "line":\r
                                if (!PreProcessLine (arg))\r
@@ -925,22 +1177,6 @@ namespace Mono.CSharp
                                                "Argument to #line directive is missing or invalid");\r
                                return true;\r
 \r
-                       case "define":\r
-                               PreProcessDefinition (true, arg);\r
-                               return true;\r
-\r
-                       case "undef":\r
-                               PreProcessDefinition (false, arg);\r
-                               return true;\r
-\r
-                       case "error":\r
-                               Report.Error (1029, Location, "#error: '" + arg + "'");\r
-                               return true;\r
-\r
-                       case "warning":\r
-                               Report.Warning (1030, Location, "#warning: '" + arg + "'");\r
-                               return true;\r
-\r
                        case "region":\r
                                arg = "true";\r
                                goto case "if";\r
@@ -950,7 +1186,7 @@ namespace Mono.CSharp
                                \r
                        case "if":\r
                                if (arg == ""){\r
-                                       Report.Error (1517, Location, "Invalid pre-processor directive");\r
+                                       Error_InvalidDirective ();\r
                                        return true;\r
                                }\r
                                bool taking = false;\r
@@ -975,7 +1211,7 @@ namespace Mono.CSharp
                                \r
                        case "endif":\r
                                if (ifstack == null || ifstack.Count == 0){\r
-                                       report1028 ("no #if for this #endif");\r
+                                       Error_UnexpectedDirective ("no #if for this #endif");\r
                                        return true;\r
                                } else {\r
                                        ifstack.Pop ();\r
@@ -993,13 +1229,13 @@ namespace Mono.CSharp
 \r
                        case "elif":\r
                                if (ifstack == null || ifstack.Count == 0){\r
-                                       report1028 ("no #if for this #elif");\r
+                                       Error_UnexpectedDirective ("no #if for this #elif");\r
                                        return true;\r
                                } else {\r
                                        int state = (int) ifstack.Peek ();\r
 \r
                                        if ((state & ELSE_SEEN) != 0){\r
-                                               report1028 ("#elif not valid after #else");\r
+                                               Error_UnexpectedDirective ("#elif not valid after #else");\r
                                                return true;\r
                                        }\r
 \r
@@ -1024,7 +1260,7 @@ namespace Mono.CSharp
                                        int state = (int) ifstack.Peek ();\r
 \r
                                        if ((state & ELSE_SEEN) != 0){\r
-                                               report1028 ("#else within #else");\r
+                                               Error_UnexpectedDirective ("#else within #else");\r
                                                return true;\r
                                        }\r
 \r
@@ -1040,8 +1276,40 @@ namespace Mono.CSharp
                                        return false;\r
                                }\r
                        }\r
-                       \r
-                       Report.Error (1024, "Preprocessor directive expected");\r
+\r
+                       //\r
+                       // These are only processed if we are in a `taking' block\r
+                       //\r
+                       if (!caller_is_taking)\r
+                               return false;\r
+                                       \r
+                       switch (cmd){\r
+                       case "define":\r
+                               if (any_token_seen){\r
+                                       Error_TokensSeen ();\r
+                                       return true;\r
+                               }\r
+                               PreProcessDefinition (true, arg);\r
+                               return true;\r
+\r
+                       case "undef":\r
+                               if (any_token_seen){\r
+                                       Error_TokensSeen ();\r
+                                       return true;\r
+                               }\r
+                               PreProcessDefinition (false, arg);\r
+                               return true;\r
+\r
+                       case "error":\r
+                               Report.Error (1029, Location, "#error: '" + arg + "'");\r
+                               return true;\r
+\r
+                       case "warning":\r
+                               Report.Warning (1030, Location, "#warning: '" + arg + "'");\r
+                               return true;\r
+                       }\r
+\r
+                       Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");\r
                        return true;\r
                }\r
                \r
@@ -1055,25 +1323,31 @@ namespace Mono.CSharp
                        val = null;\r
                        // optimization: eliminate col and implement #directive semantic correctly.\r
                        for (;(c = getChar ()) != -1; col++) {\r
-                       \r
-                               if (is_identifier_start_character ((char) c)){\r
-                                       System.Text.StringBuilder id = new System.Text.StringBuilder ();\r
+                               if (Char.IsLetter ((char)c) || c == '_'){\r
                                        string ids;\r
-                                       \r
-                                       id.Append ((char) c);\r
+\r
+                                       id_builder.Length = 0;\r
+                                       tokens_seen = true;\r
+                                       id_builder.Append ((char) c);\r
                                        \r
                                        while ((c = peekChar ()) != -1) {\r
                                                if (is_identifier_part_character ((char) c)){\r
-                                                       id.Append ((char)getChar ());\r
+                                                       id_builder.Append ((char)getChar ());\r
                                                        col++;\r
                                                } else \r
                                                        break;\r
                                        }\r
                                        \r
-                                       ids = id.ToString ();\r
+                                       ids = id_builder.ToString ();\r
 \r
                                        if (!is_keyword (ids) || allow_keyword_as_ident) {\r
                                                val = ids;\r
+                                               if (ids.Length > 512){\r
+                                                       Report.Error (\r
+                                                               645, Location,\r
+                                                               "Identifier too long (limit is 512 chars)");\r
+                                               }\r
+                                               allow_keyword_as_ident = false;\r
                                                return Token.IDENTIFIER;\r
                                        }\r
 \r
@@ -1083,13 +1357,16 @@ namespace Mono.CSharp
                                }\r
 \r
                                if (c == '.'){\r
+                                       tokens_seen = true;\r
                                        if (Char.IsDigit ((char) peekChar ()))\r
                                                return is_number (c);\r
                                        return Token.DOT;\r
                                }\r
                                \r
-                               if (Char.IsDigit ((char) c))\r
+                               if (Char.IsDigit ((char) c)){\r
+                                       tokens_seen = true;\r
                                        return is_number (c);\r
+                               }\r
 \r
                                // Handle double-slash comments.\r
                                if (c == '/'){\r
@@ -1102,6 +1379,8 @@ namespace Mono.CSharp
                                                line++;\r
                                                ref_line++;\r
                                                col = 0;\r
+                                               any_token_seen |= tokens_seen;\r
+                                               tokens_seen = false;\r
                                                continue;\r
                                        } else if (d == '*'){\r
                                                getChar ();\r
@@ -1116,6 +1395,8 @@ namespace Mono.CSharp
                                                                line++;\r
                                                                ref_line++;\r
                                                                col = 0;\r
+                                                               any_token_seen |= tokens_seen;\r
+                                                               tokens_seen = false;\r
                                                        }\r
                                                }\r
                                                continue;\r
@@ -1125,10 +1406,12 @@ namespace Mono.CSharp
                                /* For now, ignore pre-processor commands */\r
                                // FIXME: In C# the '#' is not limited to appear\r
                                // on the first column.\r
-                               if (col <= 1 && c == '#'){\r
+                               if (c == '#' && !tokens_seen){\r
+                                       bool cont = true;\r
+                                       \r
                                start_again:\r
                                        \r
-                                       bool cont = handle_preprocessing_directive ();\r
+                                       cont = handle_preprocessing_directive (cont);\r
 \r
                                        if (cont){\r
                                                col = 0;\r
@@ -1136,21 +1419,29 @@ namespace Mono.CSharp
                                        }\r
                                        col = 1;\r
 \r
+                                       bool skipping = false;\r
                                        for (;(c = getChar ()) != -1; col++){\r
                                                if (c == '\n'){\r
                                                        col = 0;\r
                                                        line++;\r
                                                        ref_line++;\r
-                                               } else if (col == 1 && c == '#'){\r
+                                                       skipping = false;\r
+                                               } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r')\r
+                                                       continue;\r
+                                               else if (c != '#')\r
+                                                       skipping = true;\r
+                                               if (c == '#' && !skipping)\r
                                                        goto start_again;\r
-                                               }\r
                                        }\r
+                                       any_token_seen |= tokens_seen;\r
+                                       tokens_seen = false;\r
                                        if (c == -1)\r
                                                Report.Error (1027, Location, "#endif expected");\r
                                        continue;\r
                                }\r
                                \r
                                if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){\r
+                                       tokens_seen = true;\r
                                        if (doread){\r
                                                getChar ();\r
                                                col++;\r
@@ -1159,25 +1450,37 @@ namespace Mono.CSharp
                                }\r
                                \r
                                if (c == '"'){\r
-                                       System.Text.StringBuilder s = new System.Text.StringBuilder ();\r
-\r
+                                       string_builder.Length = 0;\r
+                                       \r
+                                       tokens_seen = true;\r
+                                       \r
                                        while ((c = getChar ()) != -1){\r
                                                if (c == '"'){\r
-                                                       val = s.ToString ();\r
+                                                       if (allow_keyword_as_ident && peekChar () == '"'){\r
+                                                               string_builder.Append ((char) c);\r
+                                                               getChar ();\r
+                                                               continue;\r
+                                                       } \r
+                                                       allow_keyword_as_ident = false;\r
+                                                       val = string_builder.ToString ();\r
                                                        return Token.LITERAL_STRING;\r
                                                }\r
 \r
-                                               c = escape (c);\r
-                                               if (c == -1)\r
-                                                       return Token.ERROR;\r
-                                               s.Append ((char) c);\r
+                                               if (!allow_keyword_as_ident){\r
+                                                       c = escape (c);\r
+                                                       if (c == -1)\r
+                                                               return Token.ERROR;\r
+                                               }\r
+                                               string_builder.Append ((char) c);\r
                                        }\r
                                }\r
 \r
                                if (c == '\''){\r
                                        c = getChar ();\r
+                                       tokens_seen = true;\r
                                        if (c == '\''){\r
-                                               error_details = "CS1011: Empty character literal";\r
+                                               error_details = "Empty character literal";\r
+                                               Report.Error (1011, Location, error_details);\r
                                                return Token.ERROR;\r
                                        }\r
                                        c = escape (c);\r
@@ -1186,8 +1489,11 @@ namespace Mono.CSharp
                                        val = new System.Char ();\r
                                        val = (char) c;\r
                                        c = getChar ();\r
+\r
                                        if (c != '\''){\r
-                                               error_details = "CS1012: Too many characters in character literal";\r
+                                               error_details = "Too many characters in character literal";\r
+                                               Report.Error (1012, Location, error_details);\r
+\r
                                                // Try to recover, read until newline or next "'"\r
                                                while ((c = getChar ()) != -1){\r
                                                        if (c == '\n' || c == '\'')\r
@@ -1204,16 +1510,19 @@ namespace Mono.CSharp
                                        line++;\r
                                        ref_line++;\r
                                        col = 0;\r
+                                       any_token_seen |= tokens_seen;\r
+                                       tokens_seen = false;\r
                                        continue;\r
                                }\r
+\r
                                if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r'){\r
                                        if (c == '\t')\r
                                                col = (((col + 8) / 8) * 8) - 1;\r
-                                       \r
                                        continue;\r
                                }\r
 \r
                                if (c == '@'){\r
+                                       tokens_seen = true;\r
                                        allow_keyword_as_ident = true;\r
                                        continue;\r
                                }\r