2002-08-19 Martin Baulig <martin@gnome.org>
[mono.git] / mcs / mcs / cs-tokenizer.cs
index 6df480d56b2cd37ef9e4ce820a4ac85020af8ffc..5cecc98b7a77df8e6d41357b4fd16936a5c8b744 100755 (executable)
@@ -37,6 +37,19 @@ namespace Mono.CSharp
                public int current_token;\r
                bool handle_get_set = false;\r
                bool handle_remove_add = false;\r
+               bool handle_assembly = false;\r
+\r
+               //\r
+               // Whether tokens have been seen on this line\r
+               //\r
+               bool tokens_seen = false;\r
+\r
+               //\r
+               // Whether a token has been seen on the file\r
+               // This is needed because `define' is not allowed to be used\r
+               // after a token has been seen.\r
+               //\r
+               bool any_token_seen = false;\r
                \r
                //\r
                // Returns a verbose representation of the current location\r
@@ -68,6 +81,16 @@ namespace Mono.CSharp
                        }\r
                 }\r
 \r
+               public bool AssemblyTargetParsing {\r
+                       get {\r
+                               return handle_assembly;\r
+                       }\r
+\r
+                       set {\r
+                               handle_assembly = value;\r
+                       }\r
+               }\r
+\r
                public bool EventParsing {\r
                        get {\r
                                return handle_remove_add;\r
@@ -88,7 +111,6 @@ namespace Mono.CSharp
                //\r
                // Values for the associated token returned\r
                //\r
-               System.Text.StringBuilder number;\r
                int putback_char;\r
                Object val;\r
 \r
@@ -106,6 +128,10 @@ namespace Mono.CSharp
                // pre-processor if stack state:\r
                //\r
                Stack ifstack;\r
+\r
+               static System.Text.StringBuilder id_builder;\r
+               static System.Text.StringBuilder string_builder;\r
+               static System.Text.StringBuilder number_builder;\r
                \r
                //\r
                // Details about the error encoutered by the tokenizer\r
@@ -137,6 +163,7 @@ namespace Mono.CSharp
                        keywords.Add ("abstract", Token.ABSTRACT);\r
                        keywords.Add ("as", Token.AS);\r
                        keywords.Add ("add", Token.ADD);\r
+                       keywords.Add ("assembly", Token.ASSEMBLY);\r
                        keywords.Add ("base", Token.BASE);\r
                        keywords.Add ("bool", Token.BOOL);\r
                        keywords.Add ("break", Token.BREAK);\r
@@ -195,6 +222,7 @@ namespace Mono.CSharp
                        keywords.Add ("set", Token.SET);\r
                        keywords.Add ("short", Token.SHORT);\r
                        keywords.Add ("sizeof", Token.SIZEOF);\r
+                       keywords.Add ("stackalloc", Token.STACKALLOC);\r
                        keywords.Add ("static", Token.STATIC);\r
                        keywords.Add ("string", Token.STRING);\r
                        keywords.Add ("struct", Token.STRUCT);\r
@@ -222,9 +250,12 @@ namespace Mono.CSharp
                static Tokenizer ()\r
                {\r
                        InitTokens ();\r
-                       csharp_format_info = new NumberFormatInfo ();\r
-                       csharp_format_info.CurrencyDecimalSeparator = ".";\r
-                       styles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint;\r
+                       csharp_format_info = NumberFormatInfo.InvariantInfo;\r
+                       styles = NumberStyles.Float;\r
+                       \r
+                       id_builder = new System.Text.StringBuilder ();\r
+                       string_builder = new System.Text.StringBuilder ();\r
+                       number_builder = new System.Text.StringBuilder ();\r
                }\r
 \r
                bool is_keyword (string name)\r
@@ -236,6 +267,8 @@ namespace Mono.CSharp
                                return false;\r
                        if (handle_remove_add == false && (name == "remove" || name == "add"))\r
                                return false;\r
+                       if (handle_assembly == false && (name == "assembly"))\r
+                               return false;\r
                        return res;\r
                }\r
 \r
@@ -249,6 +282,16 @@ namespace Mono.CSharp
                                return new Location (ref_line);\r
                        }\r
                }\r
+\r
+               void define (string def)\r
+               {\r
+                       if (!RootContext.AllDefines.Contains (def)){\r
+                               RootContext.AllDefines [def] = true;\r
+                       }\r
+                       if (defines.Contains (def))\r
+                               return;\r
+                       defines [def] = true;\r
+               }\r
                \r
                public Tokenizer (System.IO.Stream input, string fname, ArrayList defs)\r
                {\r
@@ -259,7 +302,7 @@ namespace Mono.CSharp
                        if (defs != null){\r
                                defines = new Hashtable ();\r
                                foreach (string def in defs)\r
-                                       defines [def] = true;\r
+                                       define (def);\r
                        }\r
 \r
                        //\r
@@ -329,7 +372,7 @@ namespace Mono.CSharp
                                else if (d == '=')\r
                                        t = Token.OP_SUB_ASSIGN;\r
                                else if (d == '>')\r
-                                       return Token.OP_PTR;\r
+                                       t = Token.OP_PTR;\r
                                else\r
                                        return Token.MINUS;\r
                                doread = true;\r
@@ -448,11 +491,11 @@ namespace Mono.CSharp
                        bool seen_digits = false;\r
                        \r
                        if (c != -1)\r
-                               number.Append ((char) c);\r
+                               number_builder.Append ((char) c);\r
                        \r
                        while ((d = peekChar ()) != -1){\r
                                if (Char.IsDigit ((char)d)){\r
-                                       number.Append ((char) d);\r
+                                       number_builder.Append ((char) d);\r
                                        getChar ();\r
                                        seen_digits = true;\r
                                } else\r
@@ -462,17 +505,22 @@ namespace Mono.CSharp
                        return seen_digits;\r
                }\r
 \r
+               bool is_hex (char e)\r
+               {\r
+                       return Char.IsDigit (e) || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');\r
+               }\r
+               \r
                void hex_digits (int c)\r
                {\r
                        int d;\r
 \r
                        if (c != -1)\r
-                               number.Append ((char) c);\r
+                               number_builder.Append ((char) c);\r
                        while ((d = peekChar ()) != -1){\r
                                char e = Char.ToUpper ((char) d);\r
                                \r
-                               if (Char.IsDigit (e) || (e >= 'A' && e <= 'F')){\r
-                                       number.Append ((char) e);\r
+                               if (is_hex (e)){\r
+                                       number_builder.Append ((char) e);\r
                                        getChar ();\r
                                } else\r
                                        break;\r
@@ -585,36 +633,51 @@ namespace Mono.CSharp
                //\r
                int adjust_int (int c)\r
                {\r
-                       ulong ul = System.UInt64.Parse (number.ToString ());\r
-                       return integer_type_suffix (ul, c);\r
+                       try {\r
+                               ulong ul = System.UInt64.Parse (number_builder.ToString ());\r
+                               return integer_type_suffix (ul, c);\r
+                       } catch (OverflowException) {\r
+                               error_details = "Integral constant is too large";\r
+                               Report.Error (1021, Location, error_details);\r
+                               val = 0ul;\r
+                               return Token.LITERAL_INTEGER;\r
+                       }\r
                }\r
-\r
+               \r
                int adjust_real (int t)\r
                {\r
-                       string s = number.ToString ();\r
+                       string s = number_builder.ToString ();\r
 \r
                        switch (t){\r
                        case Token.LITERAL_DECIMAL:\r
-                               val = new System.Decimal ();\r
-                               val = System.Decimal.Parse (\r
-                                       s, styles, csharp_format_info);\r
-                               break;\r
-                       case Token.LITERAL_DOUBLE:\r
-                               val = new System.Double ();\r
-                               val = System.Double.Parse (\r
-                                       s, styles, csharp_format_info);\r
+                               try {\r
+                                       val = System.Decimal.Parse (s, styles, csharp_format_info);\r
+                               } catch (OverflowException) {\r
+                                       val = 0m;     \r
+                                       error_details = "Floating-point constant is outside the range of the type 'decimal'";\r
+                                       Report.Error(594, Location, error_details);\r
+                               }\r
                                break;\r
                        case Token.LITERAL_FLOAT:\r
-                               val = new System.Double ();\r
-                               val = (float) System.Double.Parse (\r
-                                       s, styles, csharp_format_info);\r
+                               try {\r
+                                       val = (float) System.Double.Parse (s, styles, csharp_format_info);\r
+                               } catch (OverflowException) {\r
+                                       val = 0.0f;     \r
+                                       error_details = "Floating-point constant is outside the range of the type 'float'";\r
+                                       Report.Error(594, Location, error_details);\r
+                               }\r
                                break;\r
-\r
+                               \r
+                       case Token.LITERAL_DOUBLE:\r
                        case Token.NONE:\r
-                               val = new System.Double ();\r
-                               val = System.Double.Parse (\r
-                                       s, styles, csharp_format_info);\r
                                t = Token.LITERAL_DOUBLE;\r
+                               try {\r
+                                       val = System.Double.Parse (s, styles, csharp_format_info);\r
+                               } catch (OverflowException) {\r
+                                       val = 0.0;     \r
+                                       error_details = "Floating-point constant is outside the range of the type 'double'";\r
+                                       Report.Error(594, Location, error_details);\r
+                               }\r
                                break;\r
                        }\r
                        return t;\r
@@ -626,10 +689,9 @@ namespace Mono.CSharp
                int is_number (int c)\r
                {\r
                        bool is_real = false;\r
-                       number = new System.Text.StringBuilder ();\r
                        int type;\r
 \r
-                       number.Length = 0;\r
+                       number_builder.Length = 0;\r
 \r
                        if (Char.IsDigit ((char)c)){\r
                                if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){\r
@@ -637,7 +699,7 @@ namespace Mono.CSharp
                                        getChar ();\r
                                        hex_digits (-1);\r
 \r
-                                       string s = number.ToString ();\r
+                                       string s = number_builder.ToString ();\r
 \r
                                        ul = System.UInt64.Parse (s, NumberStyles.HexNumber);\r
                                        return integer_type_suffix (ul, peekChar ());\r
@@ -656,23 +718,20 @@ namespace Mono.CSharp
                                        c = getChar ();\r
                                } else {\r
                                        putback ('.');\r
-                                       number.Length -= 1;\r
+                                       number_builder.Length -= 1;\r
                                        return adjust_int (-1);\r
                                }\r
                        }\r
                        \r
                        if (c == 'e' || c == 'E'){\r
                                is_real = true;\r
-                               number.Append ("e");\r
+                               number_builder.Append ("e");\r
                                c = getChar ();\r
                                \r
-                               if (c == '+'){\r
-                                       number.Append ((char) c);\r
-                                       c = getChar ();\r
-                               } else if (c == '-'){\r
-                                       number.Append ((char) c);\r
-                                       c = getChar ();\r
-                               }\r
+                               if (c == '+')\r
+                                       number_builder.Append ((char) c);\r
+                               else if (c == '-')\r
+                                       number_builder.Append ((char) c);\r
                                decimal_digits (-1);\r
                                c = getChar ();\r
                        }\r
@@ -694,9 +753,48 @@ namespace Mono.CSharp
                        Console.WriteLine ("This should not be reached");\r
                        throw new Exception ("Is Number should never reach this point");\r
                }\r
+\r
+               //\r
+               // Accepts exactly count (4 or 8) hex, no more no less\r
+               //\r
+               int getHex (int count, out bool error)\r
+               {\r
+                       int [] buffer = new int [8];\r
+                       int i;\r
+                       int total = 0;\r
+                       int c;\r
+                       char e;\r
+                       int top = count != -1 ? count : 4;\r
                        \r
+                       getChar ();\r
+                       error = false;\r
+                       for (i = 0; i < top; i++){\r
+                               c = getChar ();\r
+                               e = Char.ToUpper ((char) c);\r
+                               \r
+                               if (!is_hex (e)){\r
+                                       error = true;\r
+                                       return 0;\r
+                               }\r
+                               if (Char.IsDigit (e))\r
+                                       c = (int) e - (int) '0';\r
+                               else\r
+                                       c = (int) e - (int) 'A' + 10;\r
+                               total = (total * 16) + c;\r
+                               if (count == -1){\r
+                                       int p = peekChar ();\r
+                                       if (p == -1)\r
+                                               break;\r
+                                       if (!is_hex ((char)p))\r
+                                               break;\r
+                               }\r
+                       }\r
+                       return total;\r
+               }\r
+\r
                int escape (int c)\r
                {\r
+                       bool error;\r
                        int d;\r
                        int v;\r
 \r
@@ -727,8 +825,23 @@ namespace Mono.CSharp
                                v = '"'; break;\r
                        case '\'':\r
                                v = '\''; break;\r
+                       case 'x':\r
+                               v = getHex (-1, out error);\r
+                               if (error)\r
+                                       goto default;\r
+                               return v;\r
+                       case 'u':\r
+                               v = getHex (4, out error);\r
+                               if (error)\r
+                                       goto default;\r
+                               return v;\r
+                       case 'U':\r
+                               v = getHex (8, out error);\r
+                               if (error)\r
+                                       goto default;\r
+                               return v;\r
                        default:\r
-                               error_details = "cs1009: Unrecognized escape sequence " + (char)d;\r
+                               Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);\r
                                return -1;\r
                        }\r
                        getChar ();\r
@@ -788,10 +901,11 @@ namespace Mono.CSharp
                {\r
                        int c;\r
                        \r
+                       tokens_seen = false;\r
                        arg = "";\r
                        static_cmd_arg.Length = 0;\r
                                \r
-                       while ((c = getChar ()) != -1 && (c != '\n') && ((c != ' ') || c == '\t')){\r
+                       while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t')){\r
                                if (c == '\r')\r
                                        continue;\r
                                static_cmd_arg.Append ((char) c);\r
@@ -853,9 +967,7 @@ namespace Mono.CSharp
                                        \r
                                        char [] quotes = { '\"' };\r
                                        \r
-                                       ref_name = arg.Substring (pos);\r
-                                       ref_name.TrimStart (quotes);\r
-                                       ref_name.TrimEnd (quotes);\r
+                                       ref_name = arg.Substring (pos). Trim(quotes);\r
                                } else {\r
                                        ref_line = System.Int32.Parse (arg);\r
                                }\r
@@ -876,10 +988,23 @@ namespace Mono.CSharp
                                return;\r
                        }\r
 \r
+                       char[] whitespace = { ' ', '\t' };\r
+                       if (arg.IndexOfAny (whitespace) != -1){\r
+                               Report.Error(1025, Location, "Single-line comment or end-of-line expected");\r
+                               return;\r
+                       }\r
+\r
+                       foreach (char c in arg){\r
+                               if (!Char.IsLetter (c) && (c != '_')){\r
+                                       Report.Error(1001, Location, "Identifier expected");\r
+                                       return;\r
+                               }\r
+                       }\r
+\r
                        if (is_define){\r
                                if (defines == null)\r
                                        defines = new Hashtable ();\r
-                               defines [arg] = 1;\r
+                               define (arg);\r
                        } else {\r
                                if (defines == null)\r
                                        return;\r
@@ -905,7 +1030,7 @@ namespace Mono.CSharp
 \r
                bool pp_primary (ref string s)\r
                {\r
-                       s.Trim ();\r
+                       s = s.Trim ();\r
                        int len = s.Length;\r
 \r
                        if (len > 0){\r
@@ -914,19 +1039,21 @@ namespace Mono.CSharp
                                if (c == '('){\r
                                        s = s.Substring (1);\r
                                        bool val = pp_expr (ref s);\r
-                                       if (s.Length > 0 && s [0] == ')')\r
+                                       if (s.Length > 0 && s [0] == ')'){\r
+                                               s = s.Substring (1);\r
                                                return val;\r
-                                       report1517 ();\r
+                                       }\r
+                                       Error_InvalidDirective ();\r
                                        return false;\r
                                }\r
                                \r
-                               if (Char.IsLetter (c) || c == '_'){\r
+                               if (is_identifier_start_character(c)){\r
                                        int j = 1;\r
 \r
                                        while (j < len){\r
                                                c = s [j];\r
                                                \r
-                                               if (Char.IsLetter (c) || Char.IsDigit (c) || c == '_'){\r
+                                               if (is_identifier_part_character(c)){\r
                                                        j++;\r
                                                        continue;\r
                                                }\r
@@ -939,7 +1066,7 @@ namespace Mono.CSharp
                                        return vv;\r
                                }\r
                        }\r
-                       report1517 ();\r
+                       Error_InvalidDirective ();\r
                        return false;\r
                }\r
                \r
@@ -951,7 +1078,7 @@ namespace Mono.CSharp
                        if (len > 0){\r
                                if (s [0] == '!'){\r
                                        if (len > 1 && s [1] == '='){\r
-                                               report1517 ();\r
+                                               Error_InvalidDirective ();\r
                                                return false;\r
                                        }\r
                                        s = s.Substring (1);\r
@@ -959,7 +1086,7 @@ namespace Mono.CSharp
                                } else\r
                                        return pp_primary (ref s);\r
                        } else {\r
-                               report1517 ();\r
+                               Error_InvalidDirective ();\r
                                return false;\r
                        }\r
                }\r
@@ -976,7 +1103,7 @@ namespace Mono.CSharp
                                                s = s.Substring (2);\r
                                                return va == pp_unary (ref s);\r
                                        } else {\r
-                                               report1517 ();\r
+                                               Error_InvalidDirective ();\r
                                                return false;\r
                                        }\r
                                } else if (s [0] == '!' && len > 1 && s [1] == '='){\r
@@ -1003,7 +1130,7 @@ namespace Mono.CSharp
                                                s = s.Substring (2);\r
                                                return va && pp_eq (ref s);\r
                                        } else {\r
-                                               report1517 ();\r
+                                               Error_InvalidDirective ();\r
                                                return false;\r
                                        }\r
                                } \r
@@ -1021,18 +1148,17 @@ namespace Mono.CSharp
                        s = s.Trim ();\r
                        int len = s.Length;\r
                        if (len > 0){\r
-                               if (s [0] == '|'){\r
+                               char c = s [0];\r
+                               \r
+                               if (c == '|'){\r
                                        if (len > 2 && s [1] == '|'){\r
                                                s = s.Substring (2);\r
                                                return va || pp_and (ref s);\r
                                        } else {\r
-                                               report1517 ();\r
+                                               Error_InvalidDirective ();\r
                                                return false;\r
                                        }\r
-                               } else {\r
-                                       report1517 ();\r
-                                       return false;\r
-                               }\r
+                               } \r
                        }\r
 \r
                        return va;\r
@@ -1041,34 +1167,54 @@ namespace Mono.CSharp
                bool eval (string s)\r
                {\r
                        bool v = pp_expr (ref s);\r
+                       s = s.Trim ();\r
+                       if (s.Length != 0){\r
+                               Error_InvalidDirective ();\r
+                               return false;\r
+                       }\r
 \r
                        return v;\r
                }\r
                \r
-               void report1517 ()\r
+               void Error_InvalidDirective ()\r
                {\r
                        Report.Error (1517, Location, "Invalid pre-processor directive");\r
                }\r
 \r
-               void report1028 (string extra)\r
+               void Error_UnexpectedDirective (string extra)\r
                {\r
                        Report.Error (\r
                                1028, Location,\r
                                "Unexpected processor directive (" + extra + ")");\r
                }\r
+\r
+               void Error_TokensSeen ()\r
+               {\r
+                       Report.Error (\r
+                               1032, Location,\r
+                               "Cannot define or undefine pre-processor symbols after a token in the file");\r
+               }\r
                \r
                //\r
                // if true, then the code continues processing the code\r
                // if false, the code stays in a loop until another directive is\r
                // reached.\r
                //\r
-               bool handle_preprocessing_directive ()\r
+               bool handle_preprocessing_directive (bool caller_is_taking)\r
                {\r
                        char [] blank = { ' ', '\t' };\r
                        string cmd, arg;\r
                        \r
                        get_cmd_arg (out cmd, out arg);\r
 \r
+                       // Eat any trailing whitespaces and single-line comments\r
+                       if (arg.IndexOf ("//") != -1)\r
+                               arg = arg.Substring (0, arg.IndexOf ("//"));\r
+                       arg = arg.TrimEnd (' ', '\t');\r
+\r
+                       //\r
+                       // The first group of pre-processing instructions is always processed\r
+                       //\r
                        switch (cmd){\r
                        case "line":\r
                                if (!PreProcessLine (arg))\r
@@ -1077,22 +1223,6 @@ namespace Mono.CSharp
                                                "Argument to #line directive is missing or invalid");\r
                                return true;\r
 \r
-                       case "define":\r
-                               PreProcessDefinition (true, arg);\r
-                               return true;\r
-\r
-                       case "undef":\r
-                               PreProcessDefinition (false, arg);\r
-                               return true;\r
-\r
-                       case "error":\r
-                               Report.Error (1029, Location, "#error: '" + arg + "'");\r
-                               return true;\r
-\r
-                       case "warning":\r
-                               Report.Warning (1030, Location, "#warning: '" + arg + "'");\r
-                               return true;\r
-\r
                        case "region":\r
                                arg = "true";\r
                                goto case "if";\r
@@ -1102,7 +1232,7 @@ namespace Mono.CSharp
                                \r
                        case "if":\r
                                if (arg == ""){\r
-                                       report1517 ();\r
+                                       Error_InvalidDirective ();\r
                                        return true;\r
                                }\r
                                bool taking = false;\r
@@ -1127,7 +1257,7 @@ namespace Mono.CSharp
                                \r
                        case "endif":\r
                                if (ifstack == null || ifstack.Count == 0){\r
-                                       report1028 ("no #if for this #endif");\r
+                                       Error_UnexpectedDirective ("no #if for this #endif");\r
                                        return true;\r
                                } else {\r
                                        ifstack.Pop ();\r
@@ -1145,13 +1275,13 @@ namespace Mono.CSharp
 \r
                        case "elif":\r
                                if (ifstack == null || ifstack.Count == 0){\r
-                                       report1028 ("no #if for this #elif");\r
+                                       Error_UnexpectedDirective ("no #if for this #elif");\r
                                        return true;\r
                                } else {\r
                                        int state = (int) ifstack.Peek ();\r
 \r
                                        if ((state & ELSE_SEEN) != 0){\r
-                                               report1028 ("#elif not valid after #else");\r
+                                               Error_UnexpectedDirective ("#elif not valid after #else");\r
                                                return true;\r
                                        }\r
 \r
@@ -1176,7 +1306,7 @@ namespace Mono.CSharp
                                        int state = (int) ifstack.Peek ();\r
 \r
                                        if ((state & ELSE_SEEN) != 0){\r
-                                               report1028 ("#else within #else");\r
+                                               Error_UnexpectedDirective ("#else within #else");\r
                                                return true;\r
                                        }\r
 \r
@@ -1192,60 +1322,132 @@ namespace Mono.CSharp
                                        return false;\r
                                }\r
                        }\r
-                       \r
-                       Report.Error (1024, "Preprocessor directive expected (got: " + cmd + ")");\r
+\r
+                       //\r
+                       // These are only processed if we are in a `taking' block\r
+                       //\r
+                       if (!caller_is_taking)\r
+                               return false;\r
+                                       \r
+                       switch (cmd){\r
+                       case "define":\r
+                               if (any_token_seen){\r
+                                       Error_TokensSeen ();\r
+                                       return true;\r
+                               }\r
+                               PreProcessDefinition (true, arg);\r
+                               return true;\r
+\r
+                       case "undef":\r
+                               if (any_token_seen){\r
+                                       Error_TokensSeen ();\r
+                                       return true;\r
+                               }\r
+                               PreProcessDefinition (false, arg);\r
+                               return true;\r
+\r
+                       case "error":\r
+                               Report.Error (1029, Location, "#error: '" + arg + "'");\r
+                               return true;\r
+\r
+                       case "warning":\r
+                               Report.Warning (1030, Location, "#warning: '" + arg + "'");\r
+                               return true;\r
+                       }\r
+\r
+                       Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");\r
                        return true;\r
+\r
+               }\r
+\r
+               private int consume_string(bool quoted) \r
+               {\r
+                       int c;\r
+                       string_builder.Length = 0;\r
+                                                               \r
+                       while ((c = getChar ()) != -1){\r
+                               if (c == '"'){\r
+                                       if (quoted && peekChar () == '"'){\r
+                                               string_builder.Append ((char) c);\r
+                                               getChar ();\r
+                                               continue;\r
+                                       } else {\r
+                                               val = string_builder.ToString ();\r
+                                               return Token.LITERAL_STRING;\r
+                                       }\r
+                               }\r
+\r
+                               if (c == '\n' && !quoted) {\r
+                                       Report.Error(1010, Location, "Newline in constant");\r
+                               }\r
+\r
+                               if (!quoted){\r
+                                       c = escape (c);\r
+                                       if (c == -1)\r
+                                               return Token.ERROR;\r
+                               }\r
+                               string_builder.Append ((char) c);\r
+                       }\r
+\r
+                       Report.Error (1039, Location, "Unterminated string literal");\r
+                       return Token.EOF;\r
+               }\r
+\r
+               private int consume_identifier(int c, bool quoted) \r
+               {\r
+                       id_builder.Length = 0;\r
+\r
+                       id_builder.Append ((char) c);\r
+                                       \r
+                       while ((c = peekChar ()) != -1) {\r
+                               if (is_identifier_part_character ((char) c)){\r
+                                       id_builder.Append ((char)getChar ());\r
+                                       col++;\r
+                               } else \r
+                                       break;\r
+                       }\r
+                                       \r
+                       string ids = id_builder.ToString ();\r
+\r
+                       if (!is_keyword (ids) || quoted) {\r
+                               val = ids;\r
+                               if (ids.Length > 512){\r
+                                       Report.Error (\r
+                                               645, Location,\r
+                                               "Identifier too long (limit is 512 chars)");\r
+                               }\r
+                               return Token.IDENTIFIER;\r
+                       }\r
+\r
+                       // true, false and null are in the hash anyway.\r
+                       return GetKeyword (ids);\r
                }\r
                \r
                public int xtoken ()\r
                {\r
                        int t;\r
-                       bool allow_keyword_as_ident = false;\r
                        bool doread = false;\r
                        int c;\r
 \r
                        val = null;\r
                        // optimization: eliminate col and implement #directive semantic correctly.\r
                        for (;(c = getChar ()) != -1; col++) {\r
-                               if (is_identifier_start_character ((char) c)){\r
-                                       System.Text.StringBuilder id = new System.Text.StringBuilder ();\r
-                                       string ids;\r
-                                       \r
-                                       id.Append ((char) c);\r
-                                       \r
-                                       while ((c = peekChar ()) != -1) {\r
-                                               if (is_identifier_part_character ((char) c)){\r
-                                                       id.Append ((char)getChar ());\r
-                                                       col++;\r
-                                               } else \r
-                                                       break;\r
-                                       }\r
-                                       \r
-                                       ids = id.ToString ();\r
-\r
-                                       if (!is_keyword (ids) || allow_keyword_as_ident) {\r
-                                               val = ids;\r
-                                               if (ids.Length > 512){\r
-                                                       Report.Error (\r
-                                                               645, Location,\r
-                                                               "Identifier too long (limit is 512 chars)");\r
-                                               }\r
-                                               return Token.IDENTIFIER;\r
-                                       }\r
-\r
-                                       // true, false and null are in the hash anyway.\r
-                                       return GetKeyword (ids);\r
-\r
+                               if (is_identifier_start_character((char)c)){\r
+                                       tokens_seen = true;\r
+                                       return consume_identifier(c, false);\r
                                }\r
 \r
                                if (c == '.'){\r
+                                       tokens_seen = true;\r
                                        if (Char.IsDigit ((char) peekChar ()))\r
                                                return is_number (c);\r
                                        return Token.DOT;\r
                                }\r
                                \r
-                               if (Char.IsDigit ((char) c))\r
+                               if (Char.IsDigit ((char) c)){\r
+                                       tokens_seen = true;\r
                                        return is_number (c);\r
+                               }\r
 \r
                                // Handle double-slash comments.\r
                                if (c == '/'){\r
@@ -1258,6 +1460,8 @@ namespace Mono.CSharp
                                                line++;\r
                                                ref_line++;\r
                                                col = 0;\r
+                                               any_token_seen |= tokens_seen;\r
+                                               tokens_seen = false;\r
                                                continue;\r
                                        } else if (d == '*'){\r
                                                getChar ();\r
@@ -1272,6 +1476,8 @@ namespace Mono.CSharp
                                                                line++;\r
                                                                ref_line++;\r
                                                                col = 0;\r
+                                                               any_token_seen |= tokens_seen;\r
+                                                               tokens_seen = false;\r
                                                        }\r
                                                }\r
                                                continue;\r
@@ -1281,10 +1487,12 @@ namespace Mono.CSharp
                                /* For now, ignore pre-processor commands */\r
                                // FIXME: In C# the '#' is not limited to appear\r
                                // on the first column.\r
-                               if (col <= 1 && c == '#'){\r
+                               if (c == '#' && !tokens_seen){\r
+                                       bool cont = true;\r
+                                       \r
                                start_again:\r
                                        \r
-                                       bool cont = handle_preprocessing_directive ();\r
+                                       cont = handle_preprocessing_directive (cont);\r
 \r
                                        if (cont){\r
                                                col = 0;\r
@@ -1292,21 +1500,29 @@ namespace Mono.CSharp
                                        }\r
                                        col = 1;\r
 \r
+                                       bool skipping = false;\r
                                        for (;(c = getChar ()) != -1; col++){\r
                                                if (c == '\n'){\r
                                                        col = 0;\r
                                                        line++;\r
                                                        ref_line++;\r
-                                               } else if (col == 1 && c == '#'){\r
+                                                       skipping = false;\r
+                                               } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r')\r
+                                                       continue;\r
+                                               else if (c != '#')\r
+                                                       skipping = true;\r
+                                               if (c == '#' && !skipping)\r
                                                        goto start_again;\r
-                                               }\r
                                        }\r
+                                       any_token_seen |= tokens_seen;\r
+                                       tokens_seen = false;\r
                                        if (c == -1)\r
-                                               Report.Error (1027, Location, "#endif expected");\r
+                                               Report.Error (1027, Location, "#endif/#endregion expected");\r
                                        continue;\r
                                }\r
                                \r
                                if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){\r
+                                       tokens_seen = true;\r
                                        if (doread){\r
                                                getChar ();\r
                                                col++;\r
@@ -1314,24 +1530,13 @@ namespace Mono.CSharp
                                        return t;\r
                                }\r
                                \r
-                               if (c == '"'){\r
-                                       System.Text.StringBuilder s = new System.Text.StringBuilder ();\r
-\r
-                                       while ((c = getChar ()) != -1){\r
-                                               if (c == '"'){\r
-                                                       val = s.ToString ();\r
-                                                       return Token.LITERAL_STRING;\r
-                                               }\r
-\r
-                                               c = escape (c);\r
-                                               if (c == -1)\r
-                                                       return Token.ERROR;\r
-                                               s.Append ((char) c);\r
-                                       }\r
+                               if (c == '"') {\r
+                                       return consume_string(false);\r
                                }\r
 \r
                                if (c == '\''){\r
                                        c = getChar ();\r
+                                       tokens_seen = true;\r
                                        if (c == '\''){\r
                                                error_details = "Empty character literal";\r
                                                Report.Error (1011, Location, error_details);\r
@@ -1364,19 +1569,27 @@ namespace Mono.CSharp
                                        line++;\r
                                        ref_line++;\r
                                        col = 0;\r
+                                       any_token_seen |= tokens_seen;\r
+                                       tokens_seen = false;\r
                                        continue;\r
                                }\r
 \r
                                if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r'){\r
                                        if (c == '\t')\r
                                                col = (((col + 8) / 8) * 8) - 1;\r
-\r
                                        continue;\r
                                }\r
 \r
-                               if (c == '@'){\r
-                                       allow_keyword_as_ident = true;\r
-                                       continue;\r
+                               if (c == '@') {\r
+                                       c = getChar();\r
+                                       if (c == '"') {\r
+                                               tokens_seen = true;\r
+                                               return consume_string(true);\r
+                                       } else if (is_identifier_start_character((char) c)){\r
+                                               return consume_identifier(c, true);\r
+                                       } else {\r
+                                               Report.Error(1033, Location, "'@' must be followed by string constant or identifier");\r
+                                       }\r
                                }\r
 \r
                                error_details = ((char)c).ToString ();\r
@@ -1384,8 +1597,9 @@ namespace Mono.CSharp
                                return Token.ERROR;\r
                        }\r
 \r
-                       if (ifstack != null && ifstack.Count > 1)\r
-                               Report.Error (1027, Location, "#endif expected");\r
+                       if (ifstack != null && ifstack.Count >= 1)\r
+                               Report.Error (1027, Location, "#endif/#endregion expected");\r
+\r
                        return Token.EOF;\r
                }\r
        }\r