2002-12-29 Miguel de Icaza <miguel@ximian.com>
[mono.git] / mcs / mcs / cs-tokenizer.cs
index 5148a9dbd9bc32d0d9e4eee781e617b2e0319951..5193fa277ef15ba7b5c6a858d9ae33cfb3b838b6 100755 (executable)
@@ -160,9 +160,14 @@ namespace Mono.CSharp
                //\r
                Stack ifstack;\r
 \r
-               static System.Text.StringBuilder id_builder;\r
                static System.Text.StringBuilder string_builder;\r
-               static System.Text.StringBuilder number_builder;\r
+\r
+               const int max_id_size = 512;\r
+               static char [] id_builder = new char [max_id_size];\r
+\r
+               const int max_number_size = 128;\r
+               static char [] number_builder = new char [max_number_size];\r
+               static int number_pos;\r
                \r
                //\r
                // Details about the error encoutered by the tokenizer\r
@@ -284,9 +289,7 @@ namespace Mono.CSharp
                        csharp_format_info = NumberFormatInfo.InvariantInfo;\r
                        styles = NumberStyles.Float;\r
                        \r
-                       id_builder = new System.Text.StringBuilder ();\r
                        string_builder = new System.Text.StringBuilder ();\r
-                       number_builder = new System.Text.StringBuilder ();\r
                }\r
 \r
                int GetKeyword (string name)\r
@@ -517,21 +520,31 @@ namespace Mono.CSharp
                        return Token.ERROR;\r
                }\r
 \r
+               void Error_NumericConstantTooLong ()\r
+               {\r
+                       Report.Error (1021, Location, "Numeric constant too long");                     \r
+               }\r
+               \r
                bool decimal_digits (int c)\r
                {\r
                        int d;\r
                        bool seen_digits = false;\r
                        \r
-                       if (c != -1)\r
-                               number_builder.Append ((char) c);\r
-\r
+                       if (c != -1){\r
+                               if (number_pos == max_number_size)\r
+                                       Error_NumericConstantTooLong ();\r
+                               number_builder [number_pos++] = (char) c;\r
+                       }\r
+                       \r
                        //\r
                        // We use peekChar2, because decimal_digits needs to do a \r
                        // 2-character look-ahead (5.ToString for example).\r
                        //\r
                        while ((d = peekChar2 ()) != -1){\r
                                if (d >= '0' && d <= '9'){\r
-                                       number_builder.Append ((char) d);\r
+                                       if (number_pos == max_number_size)\r
+                                               Error_NumericConstantTooLong ();\r
+                                       number_builder [number_pos++] = (char) d;\r
                                        getChar ();\r
                                        seen_digits = true;\r
                                } else\r
@@ -551,14 +564,8 @@ namespace Mono.CSharp
                        int d;\r
 \r
                        if (c != -1)\r
-                               number_builder.Append ((char) c);\r
-                       while ((d = peekChar ()) != -1){\r
-                               if (is_hex (d)){\r
-                                       number_builder.Append ((char) d);\r
-                                       getChar ();\r
-                               } else\r
-                                       break;\r
-                       }\r
+                               number_builder [number_pos++] = (char) c;\r
+                       \r
                }\r
                \r
                int real_type_suffix (int c)\r
@@ -668,8 +675,21 @@ namespace Mono.CSharp
                int adjust_int (int c)\r
                {\r
                        try {\r
-                               ulong ul = System.UInt64.Parse (number_builder.ToString ());\r
-                               return integer_type_suffix (ul, c);\r
+                               if (number_pos > 9){\r
+                                       ulong ul = (uint) (number_builder [0] - '0');\r
+\r
+                                       for (int i = 1; i < number_pos; i++){\r
+                                               ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));\r
+                                       }\r
+                                       return integer_type_suffix (ul, c);\r
+                               } else {\r
+                                       uint ui = (uint) (number_builder [0] - '0');\r
+\r
+                                       for (int i = 1; i < number_pos; i++){\r
+                                               ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));\r
+                                       }\r
+                                       return integer_type_suffix (ui, c);\r
+                               }\r
                        } catch (OverflowException) {\r
                                error_details = "Integral constant is too large";\r
                                Report.Error (1021, Location, error_details);\r
@@ -680,7 +700,7 @@ namespace Mono.CSharp
                \r
                int adjust_real (int t)\r
                {\r
-                       string s = number_builder.ToString ();\r
+                       string s = new String (number_builder, 0, number_pos);\r
 \r
                        switch (t){\r
                        case Token.LITERAL_DECIMAL:\r
@@ -717,6 +737,32 @@ namespace Mono.CSharp
                        return t;\r
                }\r
 \r
+               int handle_hex ()\r
+               {\r
+                       int d;\r
+                       ulong ul;\r
+                       \r
+                       getChar ();\r
+                       while ((d = peekChar ()) != -1){\r
+                               if (is_hex (d)){\r
+                                       if (number_pos == 16){\r
+                                               Report.Error (1021, Location, "Integral constant too large");\r
+                                               return Token.ERROR;\r
+                                       }\r
+                                       number_builder [number_pos++] = (char) d;\r
+                                       getChar ();\r
+                               } else\r
+                                       break;\r
+                       }\r
+                       \r
+                       string s = new String (number_builder, 0, number_pos);\r
+                       if (number_pos <= 8)\r
+                               ul = System.UInt32.Parse (s, NumberStyles.HexNumber);\r
+                       else\r
+                               ul = System.UInt64.Parse (s, NumberStyles.HexNumber);\r
+                       return integer_type_suffix (ul, peekChar ());\r
+               }\r
+\r
                //\r
                // Invoked if we know we have .digits or digits\r
                //\r
@@ -725,18 +771,14 @@ namespace Mono.CSharp
                        bool is_real = false;\r
                        int type;\r
 \r
-                       number_builder.Length = 0;\r
+                       number_pos = 0;\r
 \r
                        if (c >= '0' && c <= '9'){\r
-                               if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){\r
-                                       ulong ul;\r
-                                       getChar ();\r
-                                       hex_digits (-1);\r
-\r
-                                       string s = number_builder.ToString ();\r
+                               if (c == '0'){\r
+                                       int peek = peekChar ();\r
 \r
-                                       ul = System.UInt64.Parse (s, NumberStyles.HexNumber);\r
-                                       return integer_type_suffix (ul, peekChar ());\r
+                                       if (peek == 'x' || peek == 'X')\r
+                                               return handle_hex ();\r
                                }\r
                                decimal_digits (c);\r
                                c = getChar ();\r
@@ -752,24 +794,33 @@ namespace Mono.CSharp
                                        c = getChar ();\r
                                } else {\r
                                        putback ('.');\r
-                                       number_builder.Length -= 1;\r
+                                       number_pos--;\r
                                        return adjust_int (-1);\r
                                }\r
                        }\r
                        \r
                        if (c == 'e' || c == 'E'){\r
                                is_real = true;\r
-                               number_builder.Append ("e");\r
+                               if (number_pos == max_number_size)\r
+                                       Error_NumericConstantTooLong ();\r
+                               number_builder [number_pos++] = 'e';\r
                                c = getChar ();\r
                                \r
                                if (c == '+'){\r
-                                       number_builder.Append ((char) c);\r
+                                       if (number_pos == max_number_size)\r
+                                               Error_NumericConstantTooLong ();\r
+                                       number_builder [number_pos++] = '+';\r
                                        c = -1;\r
                                } else if (c == '-') {\r
-                                       number_builder.Append ((char) c);\r
+                                       if (number_pos == max_number_size)\r
+                                               Error_NumericConstantTooLong ();\r
+                                       number_builder [number_pos++] = '-';\r
                                        c = -1;\r
-                               } else \r
-                                       number_builder.Append ('+');\r
+                               } else {\r
+                                       if (number_pos == max_number_size)\r
+                                               Error_NumericConstantTooLong ();\r
+                                       number_builder [number_pos++] = '+';\r
+                               }\r
                                        \r
                                decimal_digits (c);\r
                                c = getChar ();\r
@@ -958,9 +1009,7 @@ namespace Mono.CSharp
                        arg = "";\r
                        static_cmd_arg.Length = 0;\r
                                \r
-                       while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t')){\r
-                               if (c == '\r')\r
-                                       continue;\r
+                       while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){\r
                                static_cmd_arg.Append ((char) c);\r
                        }\r
 \r
@@ -970,31 +1019,34 @@ namespace Mono.CSharp
                                line++;\r
                                ref_line++;\r
                                return;\r
-                       }\r
+                       } else if (c == '\r')\r
+                               col = 0;\r
 \r
                        // skip over white space\r
-                       while ((c = getChar ()) != -1 && (c != '\n') && ((c == ' ') || (c == '\t')))\r
+                       while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))\r
                                ;\r
 \r
                        if (c == '\n'){\r
                                line++;\r
                                ref_line++;\r
                                return;\r
+                       } else if (c == '\r'){\r
+                               col = 0;\r
+                               return;\r
                        }\r
                        \r
                        static_cmd_arg.Length = 0;\r
                        static_cmd_arg.Append ((char) c);\r
                        \r
-                       while ((c = getChar ()) != -1 && (c != '\n')){\r
-                               if (c == '\r')\r
-                                       continue;\r
+                       while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){\r
                                static_cmd_arg.Append ((char) c);\r
                        }\r
 \r
                        if (c == '\n'){\r
                                line++;\r
                                ref_line++;\r
-                       }\r
+                       } else if (c == '\r')\r
+                               col = 0;\r
                        arg = static_cmd_arg.ToString ().Trim ();\r
                }\r
 \r
@@ -1197,7 +1249,6 @@ namespace Mono.CSharp
                bool pp_expr (ref string s)\r
                {\r
                        bool va = pp_and (ref s);\r
-\r
                        s = s.Trim ();\r
                        int len = s.Length;\r
                        if (len > 0){\r
@@ -1206,14 +1257,14 @@ namespace Mono.CSharp
                                if (c == '|'){\r
                                        if (len > 2 && s [1] == '|'){\r
                                                s = s.Substring (2);\r
-                                               return va | pp_and (ref s);\r
+                                               return va | pp_expr (ref s);\r
                                        } else {\r
                                                Error_InvalidDirective ();\r
                                                return false;\r
                                        }\r
                                } \r
                        }\r
-\r
+                       \r
                        return va;\r
                }\r
 \r
@@ -1364,15 +1415,21 @@ namespace Mono.CSharp
                                        }\r
 \r
                                        ifstack.Pop ();\r
-                                       ifstack.Push (state | ELSE_SEEN);\r
 \r
+                                       bool ret;\r
                                        if ((state & TAKEN_BEFORE) == 0){\r
-                                               if ((state & PARENT_TAKING) != 0)\r
-                                                       return true;\r
-                                               else\r
-                                                       return false;\r
-                                       }\r
-                                       return false;\r
+                                               ret = ((state & PARENT_TAKING) != 0);\r
+                                       } else\r
+                                               ret = false;\r
+                                       \r
+                                       if (ret)\r
+                                               state |= TAKING;\r
+                                       else\r
+                                               state &= ~TAKING;\r
+                                       \r
+                                       ifstack.Push (state | ELSE_SEEN);\r
+                                       \r
+                                       return ret;\r
                                }\r
                        }\r
 \r
@@ -1451,15 +1508,21 @@ namespace Mono.CSharp
                        return Token.EOF;\r
                }\r
 \r
-               private int consume_identifier (int c, bool quoted) \r
+               private int consume_identifier (int s, bool quoted) \r
                {\r
-                       id_builder.Length = 0;\r
-\r
-                       id_builder.Append ((char) c);\r
+                       int pos = 1;\r
+                       int c;\r
+                       \r
+                       id_builder [0] = (char) s;\r
                                        \r
                        while ((c = reader.Read ()) != -1) {\r
                                if (is_identifier_part_character ((char) c)){\r
-                                       id_builder.Append ((char)c);\r
+                                       if (pos == max_id_size){\r
+                                               Report.Error (645, Location, "Identifier too long (limit is 512 chars)");\r
+                                               return Token.ERROR;\r
+                                       }\r
+                                       \r
+                                       id_builder [pos++] = (char) c;\r
                                        putback_char = -1;\r
                                        col++;\r
                                } else {\r
@@ -1467,21 +1530,19 @@ namespace Mono.CSharp
                                        break;\r
                                }\r
                        }\r
-                                       \r
-                       string ids = id_builder.ToString ();\r
-                       int keyword = GetKeyword (ids);\r
-                       \r
-                       if (keyword == -1 || quoted){\r
-                               val = ids;\r
-                               if (ids.Length > 512){\r
-                                       Report.Error (\r
-                                               645, Location,\r
-                                               "Identifier too long (limit is 512 chars)");\r
+\r
+                       string ids = new String (id_builder, 0, pos);\r
+\r
+                       if (s >= 'a'){\r
+                               int keyword = GetKeyword (ids);\r
+                               if (keyword == -1 || quoted){\r
+                                       val = ids;\r
+                                       return Token.IDENTIFIER;\r
                                }\r
-                               return Token.IDENTIFIER;\r
+                               return keyword;\r
                        }\r
-\r
-                       return keyword;\r
+                       val = ids;\r
+                       return Token.IDENTIFIER;\r
                }\r
                \r
                public int xtoken ()\r