copied mono-api-diff.cs from mono-2-2 branch so new patch can be applied and history...
[mono.git] / mcs / mcs / cs-tokenizer.cs
index fafb277aea9c64225ebd41392618f816ff1fb157..023649905cec5923c55db59ec6430a5527a89974 100644 (file)
@@ -14,7 +14,7 @@
 
 using System;
 using System.Text;
-using System.Collections;
+using System.Collections.Generic;
 using System.IO;
 using System.Globalization;
 using System.Reflection;
@@ -40,9 +40,114 @@ namespace Mono.CSharp
                        }
                }
 
+               sealed class IdentifiersComparer : IEqualityComparer<char[]>
+               {
+                       readonly int length;
+
+                       public IdentifiersComparer (int length)
+                       {
+                               this.length = length;
+                       }
+
+                       public bool Equals (char[] x, char[] y)
+                       {
+                               for (int i = 0; i < length; ++i)
+                                       if (x [i] != y [i])
+                                               return false;
+
+                               return true;
+                       }
+
+                       public int GetHashCode (char[] obj)
+                       {
+                               int h = 0;
+                               for (int i = 0; i < length; ++i)
+                                       h = (h << 5) - h + obj [i];
+
+                               return h;
+                       }
+               }
+
+               //
+               // This class has to be used in the parser only, it reuses token
+               // details after each parse
+               //
+               public class LocatedToken
+               {
+                       int row, column;
+                       string value;
+
+                       static LocatedToken[] buffer;
+                       static int pos;
+
+                       private LocatedToken ()
+                       {
+                       }
+
+                       public static LocatedToken Create (int row, int column)
+                       {
+                               return Create (null, row, column);
+                       }
+                       
+                       public static LocatedToken Create (string value, int row, int column)
+                       {
+                               //
+                               // TODO: I am not very happy about the logic but it's the best
+                               // what I could come up with for now.
+                               // Ideally we should be using just tiny buffer (256 elements) which
+                               // is enough to hold all details for currect stack and recycle elements
+                               // poped from the stack but there is a trick needed to recycle
+                               // them properly.
+                               //
+                               LocatedToken entry;
+                               if (pos >= buffer.Length) {
+                                       entry = new LocatedToken ();
+                               } else {
+                                       entry = buffer [pos];
+                                       if (entry == null) {
+                                               entry = new LocatedToken ();
+                                               buffer [pos] = entry;
+                                       }
+
+                                       ++pos;
+                               }
+                               entry.value = value;
+                               entry.row = row;
+                               entry.column = column;
+                               return entry;
+                       }
+
+                       //
+                       // Used for token not required by expression evaluator
+                       //
+                       public static LocatedToken CreateOptional (int row, int col)
+                       {
+#if false
+                               return Create (row, col);
+#endif
+                               return null;
+                       }
+                       
+                       public static void Initialize ()
+                       {
+                               if (buffer == null)
+                                       buffer = new LocatedToken [10000];
+                               pos = 0;
+                       }
+
+                       public Location Location {
+                               get { return new Location (row, column); }
+                       }
+
+                       public string Value {
+                               get { return value; }
+                       }
+               }
+
                SeekableStreamReader reader;
                SourceFile ref_name;
                CompilationUnit file_name;
+               CompilerContext context;
                bool hidden = false;
                int ref_line = 1;
                int line = 1;
@@ -55,7 +160,7 @@ namespace Mono.CSharp
                bool handle_typeof = false;
                bool lambda_arguments_parsing;
                Location current_comment_location = Location.Null;
-               ArrayList escaped_identifiers;
+               List<Location> escaped_identifiers;
                int parsing_generic_less_than;
                
                //
@@ -164,19 +269,19 @@ namespace Mono.CSharp
                // This is used to trigger completion generation on the parser
                public bool CompleteOnEOF;
                
-               void AddEscapedIdentifier (LocatedToken lt)
+               void AddEscapedIdentifier (Location loc)
                {
                        if (escaped_identifiers == null)
-                               escaped_identifiers = new ArrayList ();
+                               escaped_identifiers = new List<Location> ();
 
-                       escaped_identifiers.Add (lt);
+                       escaped_identifiers.Add (loc);
                }
 
                public bool IsEscapedIdentifier (Location loc)
                {
                        if (escaped_identifiers != null) {
-                               foreach (LocatedToken lt in escaped_identifiers)
-                                       if (lt.Location.Equals (loc))
+                               foreach (Location lt in escaped_identifiers)
+                                       if (lt.Equals (loc))
                                                return true;
                        }
 
@@ -187,7 +292,7 @@ namespace Mono.CSharp
                // Class variables
                // 
                static KeywordEntry[][] keywords;
-               static Hashtable keyword_strings;
+               static Dictionary<string, object> keyword_strings;              // TODO: HashSet
                static NumberStyles styles;
                static NumberFormatInfo csharp_format_info;
                
@@ -195,7 +300,7 @@ namespace Mono.CSharp
                // Values for the associated token returned
                //
                internal int putback_char;      // Used by repl only
-               Object val;
+               object val;
 
                //
                // Pre-processor
@@ -208,18 +313,20 @@ namespace Mono.CSharp
                //
                // pre-processor if stack state:
                //
-               Stack ifstack;
+               Stack<int> ifstack;
 
                static System.Text.StringBuilder string_builder;
 
                const int max_id_size = 512;
                static char [] id_builder = new char [max_id_size];
 
-               static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
+               public static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 
                const int max_number_size = 512;
                static char [] number_builder = new char [max_number_size];
                static int number_pos;
+
+               static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
                
                //
                // Details about the error encoutered by the tokenizer
@@ -244,7 +351,8 @@ namespace Mono.CSharp
                // on its own to deamiguate a token in behalf of the
                // parser.
                //
-               Stack position_stack = new Stack (2);
+               Stack<Position> position_stack = new Stack<Position> (2);
+
                class Position {
                        public int position;
                        public int line;
@@ -253,9 +361,10 @@ namespace Mono.CSharp
                        public bool hidden;
                        public int putback_char;
                        public int previous_col;
-                       public Stack ifstack;
+                       public Stack<int> ifstack;
                        public int parsing_generic_less_than;
                        public int current_token;
+                       public object val;
 
                        public Position (Tokenizer t)
                        {
@@ -266,10 +375,16 @@ namespace Mono.CSharp
                                hidden = t.hidden;
                                putback_char = t.putback_char;
                                previous_col = t.previous_col;
-                               if (t.ifstack != null && t.ifstack.Count != 0)
-                                       ifstack = (Stack)t.ifstack.Clone ();
+                               if (t.ifstack != null && t.ifstack.Count != 0) {
+                                       // There is no simple way to clone Stack<T> all
+                                       // methods reverse the order
+                                       var clone = t.ifstack.ToArray ();
+                                       Array.Reverse (clone);
+                                       ifstack = new Stack<int> (clone);
+                               }
                                parsing_generic_less_than = t.parsing_generic_less_than;
                                current_token = t.current_token;
+                               val = t.val;
                        }
                }
                
@@ -280,7 +395,7 @@ namespace Mono.CSharp
 
                public void PopPosition ()
                {
-                       Position p = (Position) position_stack.Pop ();
+                       Position p = position_stack.Pop ();
 
                        reader.Position = p.position;
                        ref_line = p.ref_line;
@@ -292,6 +407,7 @@ namespace Mono.CSharp
                        ifstack = p.ifstack;
                        parsing_generic_less_than = p.parsing_generic_less_than;
                        current_token = p.current_token;
+                       val = p.val;
                }
 
                // Do not reset the position, ignore it.
@@ -302,7 +418,7 @@ namespace Mono.CSharp
                
                static void AddKeyword (string kw, int token)
                {
-                       keyword_strings.Add (kw, kw);
+                       keyword_strings.Add (kw, null);
 
                        int length = kw.Length;
                        if (keywords [length] == null) {
@@ -325,7 +441,7 @@ namespace Mono.CSharp
 
                static void InitTokens ()
                {
-                       keyword_strings = new Hashtable ();
+                       keyword_strings = new Dictionary<string, object> ();
 
                        // 11 is the length of the longest keyword for now
                        keywords = new KeywordEntry [11] [];
@@ -465,11 +581,11 @@ namespace Mono.CSharp
                                for (int i = 1; i < id_len; ++i) {
                                        if (id [i] != kwe.Value [i]) {
                                                res = 0;
+                                               kwe = kwe.Next;
                                                break;
                                        }
                                }
-                               kwe = kwe.Next;
-                       } while (kwe != null && res == 0);
+                       } while (res == 0 && kwe != null);
 
                        if (res == 0)
                                return -1;
@@ -538,7 +654,7 @@ namespace Mono.CSharp
                                                        Report.FeatureIsNotAvailable (Location, "query expressions");
                                                break;
                                        case Token.VOID:
-                                               Expression.Error_VoidInvalidInTheContext (Location);
+                                               Expression.Error_VoidInvalidInTheContext (Location, Report);
                                                break;
                                        default:
                                                PopPosition ();
@@ -618,10 +734,11 @@ namespace Mono.CSharp
                        }
                }
 
-               public Tokenizer (SeekableStreamReader input, CompilationUnit file)
+               public Tokenizer (SeekableStreamReader input, CompilationUnit file, CompilerContext ctx)
                {
                        this.ref_name = file;
                        this.file_name = file;
+                       this.context = ctx;
                        reader = input;
                        
                        putback_char = -1;
@@ -659,13 +776,12 @@ namespace Mono.CSharp
 
                public static bool IsKeyword (string s)
                {
-                       return keyword_strings [s] != null;
+                       return keyword_strings.ContainsKey (s);
                }
 
                //
                // Open parens micro parser. Detects both lambda and cast ambiguity.
-               //
-               
+               //      
                int TokenizeOpenParens ()
                {
                        int ptoken;
@@ -686,12 +802,8 @@ namespace Mono.CSharp
                                        //
                                        // Expression inside parens is lambda, (int i) => 
                                        //
-                                       if (current_token == Token.ARROW) {
-                                               if (RootContext.Version <= LanguageVersion.ISO_2)
-                                                       Report.FeatureIsNotAvailable (Location, "lambda expressions");
-
+                                       if (current_token == Token.ARROW)
                                                return Token.OPEN_PARENS_LAMBDA;
-                                       }
 
                                        //
                                        // Expression inside parens is single type, (int[])
@@ -708,12 +820,7 @@ namespace Mono.CSharp
                                                case Token.BANG:
                                                case Token.TILDE:
                                                case Token.IDENTIFIER:
-                                               case Token.LITERAL_INTEGER:
-                                               case Token.LITERAL_FLOAT:
-                                               case Token.LITERAL_DOUBLE:
-                                               case Token.LITERAL_DECIMAL:
-                                               case Token.LITERAL_CHARACTER:
-                                               case Token.LITERAL_STRING:
+                                               case Token.LITERAL:
                                                case Token.BASE:
                                                case Token.CHECKED:
                                                case Token.DELEGATE:
@@ -956,8 +1063,7 @@ namespace Mono.CSharp
                        case Token.TRUE:
                        case Token.FALSE:
                        case Token.NULL:
-                       case Token.LITERAL_INTEGER:
-                       case Token.LITERAL_STRING:
+                       case Token.LITERAL:
                                return Token.INTERR;
                        }
 
@@ -972,12 +1078,7 @@ namespace Mono.CSharp
                        current_token = Token.NONE;
                        int next_token;
                        switch (xtoken ()) {
-                       case Token.LITERAL_INTEGER:
-                       case Token.LITERAL_STRING:
-                       case Token.LITERAL_CHARACTER:
-                       case Token.LITERAL_DECIMAL:
-                       case Token.LITERAL_DOUBLE:
-                       case Token.LITERAL_FLOAT:
+                       case Token.LITERAL:
                        case Token.TRUE:
                        case Token.FALSE:
                        case Token.NULL:
@@ -1079,24 +1180,18 @@ namespace Mono.CSharp
                        return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
                }
 
-               static int real_type_suffix (int c)
+               static TypeCode real_type_suffix (int c)
                {
-                       int t;
-
                        switch (c){
                        case 'F': case 'f':
-                               t =  Token.LITERAL_FLOAT;
-                               break;
+                               return TypeCode.Single;
                        case 'D': case 'd':
-                               t = Token.LITERAL_DOUBLE;
-                               break;
+                               return TypeCode.Double;
                        case 'M': case 'm':
-                                t= Token.LITERAL_DECIMAL;
-                               break;
+                               return TypeCode.Decimal;
                        default:
-                               return Token.NONE;
+                               return TypeCode.Empty;
                        }
-                       return t;
                }
 
                int integer_type_suffix (ulong ul, int c)
@@ -1123,16 +1218,8 @@ namespace Mono.CSharp
                                                        //
                                                        Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
                                                }
-                                               //
-                                               // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
-                                               // work around that.
-                                               //
-                                               //goto case 'L';
-                                               if (is_long)
-                                                       scanning = false;
-                                               is_long = true;
-                                               get_char ();
-                                               break;
+
+                                               goto case 'L';
 
                                        case 'L': 
                                                if (is_long)
@@ -1150,38 +1237,40 @@ namespace Mono.CSharp
                        }
 
                        if (is_long && is_unsigned){
-                               val = ul;
-                               return Token.LITERAL_INTEGER;
-                       } else if (is_unsigned){
+                               val = new ULongLiteral (ul, Location);
+                               return Token.LITERAL;
+                       }
+                       
+                       if (is_unsigned){
                                // uint if possible, or ulong else.
 
                                if ((ul & 0xffffffff00000000) == 0)
-                                       val = (uint) ul;
+                                       val = new UIntLiteral ((uint) ul, Location);
                                else
-                                       val = ul;
+                                       val = new ULongLiteral (ul, Location);
                        } else if (is_long){
                                // long if possible, ulong otherwise
                                if ((ul & 0x8000000000000000) != 0)
-                                       val = ul;
+                                       val = new ULongLiteral (ul, Location);
                                else
-                                       val = (long) ul;
+                                       val = new LongLiteral ((long) ul, Location);
                        } else {
                                // int, uint, long or ulong in that order
                                if ((ul & 0xffffffff00000000) == 0){
                                        uint ui = (uint) ul;
                                        
                                        if ((ui & 0x80000000) != 0)
-                                               val = ui;
+                                               val = new UIntLiteral (ui, Location);
                                        else
-                                               val = (int) ui;
+                                               val = new IntLiteral ((int) ui, Location);
                                } else {
                                        if ((ul & 0x8000000000000000) != 0)
-                                               val = ul;
+                                               val = new ULongLiteral (ul, Location);
                                        else
-                                               val = (long) ul;
+                                               val = new LongLiteral ((long) ul, Location);
                                }
                        }
-                       return Token.LITERAL_INTEGER;
+                       return Token.LITERAL;
                }
                                
                //
@@ -1210,51 +1299,49 @@ namespace Mono.CSharp
                        } catch (OverflowException) {
                                error_details = "Integral constant is too large";
                                Report.Error (1021, Location, error_details);
-                               val = 0ul;
-                               return Token.LITERAL_INTEGER;
+                               val = new IntLiteral (0, Location);
+                               return Token.LITERAL;
                        }
                        catch (FormatException) {
                                Report.Error (1013, Location, "Invalid number");
-                               val = 0ul;
-                               return Token.LITERAL_INTEGER;
+                               val = new IntLiteral (0, Location);
+                               return Token.LITERAL;
                        }
                }
                
-               int adjust_real (int t)
+               int adjust_real (TypeCode t)
                {
                        string s = new String (number_builder, 0, number_pos);
                        const string error_details = "Floating-point constant is outside the range of type `{0}'";
 
                        switch (t){
-                       case Token.LITERAL_DECIMAL:
+                       case TypeCode.Decimal:
                                try {
-                                       val = System.Decimal.Parse (s, styles, csharp_format_info);
+                                       val = new DecimalLiteral (decimal.Parse (s, styles, csharp_format_info), Location);
                                } catch (OverflowException) {
-                                       val = 0m;     
+                                       val = new DecimalLiteral (0, Location);
                                        Report.Error (594, Location, error_details, "decimal");
                                }
                                break;
-                       case Token.LITERAL_FLOAT:
+                       case TypeCode.Single:
                                try {
-                                       val = float.Parse (s, styles, csharp_format_info);
+                                       val = new FloatLiteral (float.Parse (s, styles, csharp_format_info), Location);
                                } catch (OverflowException) {
-                                       val = 0.0f;     
+                                       val = new FloatLiteral (0, Location);
                                        Report.Error (594, Location, error_details, "float");
                                }
                                break;
-                               
-                       case Token.LITERAL_DOUBLE:
-                       case Token.NONE:
-                               t = Token.LITERAL_DOUBLE;
+                       default:
                                try {
-                                       val = System.Double.Parse (s, styles, csharp_format_info);
+                                       val = new DoubleLiteral (double.Parse (s, styles, csharp_format_info), Location);
                                } catch (OverflowException) {
-                                       val = 0.0;     
+                                       val = new DoubleLiteral (0, Location);
                                        Report.Error (594, Location, error_details, "double");
                                }
                                break;
                        }
-                       return t;
+
+                       return Token.LITERAL;
                }
 
                int handle_hex ()
@@ -1280,13 +1367,13 @@ namespace Mono.CSharp
                        } catch (OverflowException){
                                error_details = "Integral constant is too large";
                                Report.Error (1021, Location, error_details);
-                               val = 0ul;
-                               return Token.LITERAL_INTEGER;
+                               val = new IntLiteral (0, Location);
+                               return Token.LITERAL;
                        }
                        catch (FormatException) {
                                Report.Error (1013, Location, "Invalid number");
-                               val = 0ul;
-                               return Token.LITERAL_INTEGER;
+                               val = new IntLiteral (0, Location);
+                               return Token.LITERAL;
                        }
                        
                        return integer_type_suffix (ul, peek_char ());
@@ -1298,7 +1385,6 @@ namespace Mono.CSharp
                int is_number (int c)
                {
                        bool is_real = false;
-                       int type;
 
                        number_pos = 0;
 
@@ -1355,21 +1441,21 @@ namespace Mono.CSharp
                                c = get_char ();
                        }
 
-                       type = real_type_suffix (c);
-                       if (type == Token.NONE && !is_real){
+                       var type = real_type_suffix (c);
+                       if (type == TypeCode.Empty && !is_real){
                                putback (c);
                                return adjust_int (c);
-                       } else 
-                               is_real = true;
+                       }
+
+                       is_real = true;
 
-                       if (type == Token.NONE){
+                       if (type == TypeCode.Empty){
                                putback (c);
                        }
                        
                        if (is_real)
                                return adjust_real (type);
 
-                       Console.WriteLine ("This should not be reached");
                        throw new Exception ("Is Number should never reach this point");
                }
 
@@ -1574,8 +1660,6 @@ namespace Mono.CSharp
                        return current_token;
                }
 
-               static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
-
                void get_cmd_arg (out string cmd, out string arg)
                {
                        int c;
@@ -1874,18 +1958,18 @@ namespace Mono.CSharp
                                int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
                                foreach (int code in codes) {
                                        if (code != 0)
-                                               Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
+                                               Report.RegisterWarningRegion (Location).WarningDisable (Location, code, Report);
                                }
                                return;
                        }
 
                        if (arg.StartsWith (w_restore)) {
                                int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
-                               Hashtable w_table = Report.warning_ignore_table;
+                               var w_table = Report.warning_ignore_table;
                                foreach (int code in codes) {
-                                       if (w_table != null && w_table.Contains (code))
-                                               Report.Warning (1635, 1, Location, String.Format ("Cannot restore warning `CS{0:0000}' because it was disabled globally", code));
-                                       Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
+                                       if (w_table != null && w_table.ContainsKey (code))
+                                               Report.Warning (1635, 1, Location, "Cannot restore warning `CS{0:0000}' because it was disabled globally", code);
+                                       Report.RegisterWarningRegion (Location).WarningEnable (Location, code, Report);
                                }
                                return;
                        }
@@ -2153,7 +2237,7 @@ namespace Mono.CSharp
                                        Error_UnexpectedDirective ("no #region for this #endregion");
                                        return true;
                                }
-                               int pop = (int) ifstack.Pop ();
+                               int pop = ifstack.Pop ();
                                        
                                if ((pop & REGION) == 0)
                                        Report.Error (1027, Location, "Expected `#endif' directive");
@@ -2162,13 +2246,13 @@ namespace Mono.CSharp
                                
                        case "if":
                                if (ifstack == null)
-                                       ifstack = new Stack (2);
+                                       ifstack = new Stack<int> (2);
 
                                int flags = region_directive ? REGION : 0;
                                if (ifstack.Count == 0){
                                        flags |= PARENT_TAKING;
                                } else {
-                                       int state = (int) ifstack.Peek ();
+                                       int state = ifstack.Peek ();
                                        if ((state & TAKING) != 0) {
                                                flags |= PARENT_TAKING;
                                        }
@@ -2186,7 +2270,7 @@ namespace Mono.CSharp
                                        Error_UnexpectedDirective ("no #if for this #endif");
                                        return true;
                                } else {
-                                       pop = (int) ifstack.Pop ();
+                                       pop = ifstack.Pop ();
                                        
                                        if ((pop & REGION) != 0)
                                                Report.Error (1038, Location, "#endregion directive expected");
@@ -2198,7 +2282,7 @@ namespace Mono.CSharp
                                        if (ifstack.Count == 0)
                                                return true;
 
-                                       int state = (int) ifstack.Peek ();
+                                       int state = ifstack.Peek ();
                                        return (state & TAKING) != 0;
                                }
 
@@ -2207,7 +2291,7 @@ namespace Mono.CSharp
                                        Error_UnexpectedDirective ("no #if for this #elif");
                                        return true;
                                } else {
-                                       int state = (int) ifstack.Pop ();
+                                       int state = ifstack.Pop ();
 
                                        if ((state & REGION) != 0) {
                                                Report.Error (1038, Location, "#endregion directive expected");
@@ -2238,7 +2322,7 @@ namespace Mono.CSharp
                                        Error_UnexpectedDirective ("no #if for this #else");
                                        return true;
                                } else {
-                                       int state = (int) ifstack.Peek ();
+                                       int state = ifstack.Peek ();
 
                                        if ((state & REGION) != 0) {
                                                Report.Error (1038, Location, "#endregion directive expected");
@@ -2337,8 +2421,8 @@ namespace Mono.CSharp
                                                get_char ();
                                                continue;
                                        } else {
-                                               val = string_builder.ToString ();
-                                               return Token.LITERAL_STRING;
+                                               val = new StringLiteral (string_builder.ToString (), Location);
+                                               return Token.LITERAL;
                                        }
                                }
 
@@ -2374,9 +2458,15 @@ namespace Mono.CSharp
                        return res;
                }
 
-               private int consume_identifier (int c, bool quoted) 
+               int consume_identifier (int c, bool quoted) 
                {
+                       //
+                       // This method is very performance sensitive. It accounts
+                       // for approximately 25% of all parser time
+                       //
+
                        int pos = 0;
+                       int column = col;
 
                        if (c == '\\') {
                                int surrogate;
@@ -2388,32 +2478,44 @@ namespace Mono.CSharp
                        }
 
                        id_builder [pos++] = (char) c;
-                       Location loc = Location;
 
-                       while ((c = get_char ()) != -1) {
-                       loop:
-                               if (is_identifier_part_character ((char) c)){
-                                       if (pos == max_id_size){
-                                               Report.Error (645, loc, "Identifier too long (limit is 512 chars)");
-                                               return Token.ERROR;
+                       try {
+                               while (true) {
+                                       c = reader.Read ();
+
+                                       if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
+                                               id_builder [pos++] = (char) c;
+                                               continue;
                                        }
-                                       
-                                       id_builder [pos++] = (char) c;
-                               } else if (c == '\\') {
-                                       int surrogate;
-                                       c = escape (c, out surrogate);
-                                       if (surrogate != 0) {
-                                               if (is_identifier_part_character ((char) c))
-                                                       id_builder [pos++] = (char) c;
-                                               c = surrogate;
+
+                                       if (c < 0x80) {
+                                               if (c == '\\') {
+                                                       int surrogate;
+                                                       c = escape (c, out surrogate);
+                                                       if (surrogate != 0) {
+                                                               if (is_identifier_part_character ((char) c))
+                                                                       id_builder[pos++] = (char) c;
+                                                               c = surrogate;
+                                                       }
+
+                                                       continue;
+                                               }
+                                       } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
+                                               id_builder [pos++] = (char) c;
+                                               continue;
                                        }
-                                       goto loop;
-                               } else {
-                                       putback (c);
+
+                                       putback_char = c;
                                        break;
                                }
+                       } catch (IndexOutOfRangeException) {
+                               Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
+                               col += pos - 1;
+                               return Token.ERROR;
                        }
 
+                       col += pos - 1;
+
                        //
                        // Optimization: avoids doing the keyword lookup
                        // on uppercase letters
@@ -2421,8 +2523,7 @@ namespace Mono.CSharp
                        if (id_builder [0] >= '_' && !quoted) {
                                int keyword = GetKeyword (id_builder, pos);
                                if (keyword != -1) {
-                                       // TODO: No need to store location for keyword, required location cleanup
-                                       val = loc;
+                                       val = LocatedToken.Create (null, ref_line, column);
                                        return keyword;
                                }
                        }
@@ -2431,38 +2532,33 @@ namespace Mono.CSharp
                        // Keep identifiers in an array of hashtables to avoid needless
                        // allocations
                        //
-                       CharArrayHashtable identifiers_group = identifiers [pos];
+                       var identifiers_group = identifiers [pos];
+                       string s;
                        if (identifiers_group != null) {
-                               val = identifiers_group [id_builder];
-                               if (val != null) {
-                                       val = new LocatedToken (loc, (string) val);
+                               if (identifiers_group.TryGetValue (id_builder, out s)) {
+                                       val = LocatedToken.Create (s, ref_line, column);
                                        if (quoted)
-                                               AddEscapedIdentifier ((LocatedToken) val);
+                                               AddEscapedIdentifier (((LocatedToken) val).Location);
                                        return Token.IDENTIFIER;
                                }
                        } else {
-                               identifiers_group = new CharArrayHashtable (pos);
+                               // TODO: this should be number of files dependant
+                               // corlib compilation peaks at 1000 and System.Core at 150
+                               int capacity = pos > 20 ? 10 : 100;
+                               identifiers_group = new Dictionary<char[],string> (capacity, new IdentifiersComparer (pos));
                                identifiers [pos] = identifiers_group;
                        }
 
                        char [] chars = new char [pos];
                        Array.Copy (id_builder, chars, pos);
 
-                       val = new String (id_builder, 0, pos);
-                       identifiers_group.Add (chars, val);
+                       s = new string (id_builder, 0, pos);
+                       identifiers_group.Add (chars, s);
 
-                       if (RootContext.Version == LanguageVersion.ISO_1) {
-                               for (int i = 1; i < chars.Length; i += 3) {
-                                       if (chars [i] == '_' && (chars [i - 1] == '_' || chars [i + 1] == '_')) {
-                                               Report.Error (1638, loc,
-                                                       "`{0}': Any identifier with double underscores cannot be used when ISO language version mode is specified", val.ToString ());
-                                       }
-                               }
-                       }
-
-                       val = new LocatedToken (loc, (string) val);
+                       val = LocatedToken.Create (s, ref_line, column);
                        if (quoted)
-                               AddEscapedIdentifier ((LocatedToken) val);
+                               AddEscapedIdentifier (((LocatedToken) val).Location);
+
                        return Token.IDENTIFIER;
                }
                
@@ -2513,20 +2609,22 @@ namespace Mono.CSharp
                                        return consume_identifier (c);
 
                                case '{':
-                                       val = Location;
+                                       val = LocatedToken.Create (ref_line, col);
                                        return Token.OPEN_BRACE;
                                case '}':
-                                       val = Location;
+                                       val = LocatedToken.Create (ref_line, col);
                                        return Token.CLOSE_BRACE;
                                case '[':
                                        // To block doccomment inside attribute declaration.
                                        if (doc_state == XmlCommentState.Allowed)
                                                doc_state = XmlCommentState.NotAllowed;
+                                       val = LocatedToken.CreateOptional (ref_line, col);
                                        return Token.OPEN_BRACKET;
                                case ']':
+                                       val = LocatedToken.CreateOptional (ref_line, col);
                                        return Token.CLOSE_BRACKET;
                                case '(':
-                                       val = Location;
+                                       val = LocatedToken.Create (ref_line, col);
                                        //
                                        // An expression versions of parens can appear in block context only
                                        //
@@ -2571,22 +2669,29 @@ namespace Mono.CSharp
 
                                        return Token.OPEN_PARENS;
                                case ')':
+                                       val = LocatedToken.CreateOptional (ref_line, col);
                                        return Token.CLOSE_PARENS;
                                case ',':
+                                       val = LocatedToken.CreateOptional (ref_line, col);
                                        return Token.COMMA;
                                case ';':
+                                       val = LocatedToken.CreateOptional (ref_line, col);
                                        return Token.SEMICOLON;
                                case '~':
+                                       val = LocatedToken.Create (ref_line, col);
                                        return Token.TILDE;
                                case '?':
+                                       val = LocatedToken.Create (ref_line, col);
                                        return TokenizePossibleNullableType ();
                                case '<':
+                                       val = LocatedToken.Create (ref_line, col);
                                        if (parsing_generic_less_than++ > 0)
                                                return Token.OP_GENERICS_LT;
 
                                        return TokenizeLessThan ();
 
                                case '>':
+                                       val = LocatedToken.Create (ref_line, col);
                                        d = peek_char ();
 
                                        if (d == '='){
@@ -2613,6 +2718,7 @@ namespace Mono.CSharp
                                        return Token.OP_GT;
 
                                case '+':
+                                       val = LocatedToken.Create (ref_line, col);
                                        d = peek_char ();
                                        if (d == '+') {
                                                d = Token.OP_INC;
@@ -2625,6 +2731,7 @@ namespace Mono.CSharp
                                        return d;
 
                                case '-':
+                                       val = LocatedToken.Create (ref_line, col);
                                        d = peek_char ();
                                        if (d == '-') {
                                                d = Token.OP_DEC;
@@ -2639,6 +2746,7 @@ namespace Mono.CSharp
                                        return d;
 
                                case '!':
+                                       val = LocatedToken.Create (ref_line, col);
                                        if (peek_char () == '='){
                                                get_char ();
                                                return Token.OP_NE;
@@ -2646,6 +2754,7 @@ namespace Mono.CSharp
                                        return Token.BANG;
 
                                case '=':
+                                       val = LocatedToken.Create (ref_line, col);
                                        d = peek_char ();
                                        if (d == '='){
                                                get_char ();
@@ -2659,6 +2768,7 @@ namespace Mono.CSharp
                                        return Token.ASSIGN;
 
                                case '&':
+                                       val = LocatedToken.Create (ref_line, col);
                                        d = peek_char ();
                                        if (d == '&'){
                                                get_char ();
@@ -2671,6 +2781,7 @@ namespace Mono.CSharp
                                        return Token.BITWISE_AND;
 
                                case '|':
+                                       val = LocatedToken.Create (ref_line, col);
                                        d = peek_char ();
                                        if (d == '|'){
                                                get_char ();
@@ -2683,16 +2794,17 @@ namespace Mono.CSharp
                                        return Token.BITWISE_OR;
 
                                case '*':
+                                       val = LocatedToken.Create (ref_line, col);
                                        if (peek_char () == '='){
                                                get_char ();
                                                return Token.OP_MULT_ASSIGN;
                                        }
-                                       val = Location;
                                        return Token.STAR;
 
                                case '/':
                                        d = peek_char ();
                                        if (d == '='){
+                                               val = LocatedToken.Create (ref_line, col);
                                                get_char ();
                                                return Token.OP_DIV_ASSIGN;
                                        }
@@ -2769,6 +2881,7 @@ namespace Mono.CSharp
                                        return Token.DIV;
 
                                case '%':
+                                       val = LocatedToken.Create (ref_line, col);
                                        if (peek_char () == '='){
                                                get_char ();
                                                return Token.OP_MOD_ASSIGN;
@@ -2776,6 +2889,7 @@ namespace Mono.CSharp
                                        return Token.PERCENT;
 
                                case '^':
+                                       val = LocatedToken.Create (ref_line, col);
                                        if (peek_char () == '='){
                                                get_char ();
                                                return Token.OP_XOR_ASSIGN;
@@ -2783,6 +2897,7 @@ namespace Mono.CSharp
                                        return Token.CARRET;
 
                                case ':':
+                                       val = LocatedToken.Create (ref_line, col);
                                        if (peek_char () == ':') {
                                                get_char ();
                                                return Token.DOUBLE_COLON;
@@ -2916,7 +3031,7 @@ namespace Mono.CSharp
                        if (d != 0)
                                throw new NotImplementedException ();
 
-                       val = (char) c;
+                       val = new CharLiteral ((char) c, Location);
                        c = get_char ();
 
                        if (c != '\'') {
@@ -2930,7 +3045,7 @@ namespace Mono.CSharp
                                return Token.ERROR;
                        }
 
-                       return Token.LITERAL_CHARACTER;
+                       return Token.LITERAL;
                }
 
                int TokenizeLessThan ()
@@ -3077,6 +3192,10 @@ namespace Mono.CSharp
                        return null;
                }
 
+               Report Report {
+                       get { return context.Report; }
+               }
+
                void reset_doc_comment ()
                {
                        xml_comment_buffer.Length = 0;
@@ -3086,7 +3205,7 @@ namespace Mono.CSharp
                public void cleanup ()
                {
                        if (ifstack != null && ifstack.Count >= 1) {
-                               int state = (int) ifstack.Pop ();
+                               int state = ifstack.Pop ();
                                if ((state & REGION) != 0)
                                        Report.Error (1038, Location, "#endregion directive expected");
                                else