Fix #77035.
[mono.git] / mcs / mcs / cs-tokenizer.cs
old mode 100755 (executable)
new mode 100644 (file)
index 921ae6c..83b92d2
@@ -7,6 +7,7 @@
 // Licensed under the terms of the GNU GPL
 //
 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
+// (C) 2004 Novell, Inc
 //
 
 /*
@@ -31,15 +32,30 @@ namespace Mono.CSharp
        public class Tokenizer : yyParser.yyInput
        {
                SeekableStreamReader reader;
-               public SourceFile ref_name;
-               public SourceFile file_name;
-               public int ref_line = 1;
-               public int line = 1;
-               public int col = 1;
-               public int current_token;
+               SourceFile ref_name;
+               SourceFile file_name;
+               int ref_line = 1;
+               int line = 1;
+               int col = 0;
+               int previous_col;
+               int current_token;
                bool handle_get_set = false;
                bool handle_remove_add = false;
                bool handle_assembly = false;
+               Location current_location;
+               Location current_comment_location = Location.Null;
+               ArrayList escapedIdentifiers = new ArrayList ();
+
+               //
+               // XML documentation buffer. The save point is used to divide
+               // comments on types and comments on members.
+               //
+               StringBuilder xml_comment_buffer;
+
+               //
+               // See comment on XmlCommentState enumeration.
+               //
+               XmlCommentState xmlDocState = XmlCommentState.Allowed;
 
                //
                // Whether tokens have been seen on this line
@@ -52,6 +68,7 @@ namespace Mono.CSharp
                // after a token has been seen.
                //
                bool any_token_seen = false;
+
                static Hashtable tokenValues;
 
                private static Hashtable TokenValueName
@@ -132,11 +149,32 @@ namespace Mono.CSharp
                                handle_remove_add = value;
                        }
                }
+
+               public XmlCommentState doc_state {
+                       get { return xmlDocState; }
+                       set {
+                               if (value == XmlCommentState.Allowed) {
+                                       check_incorrect_doc_comment ();
+                                       reset_doc_comment ();
+                               }
+                               xmlDocState = value;
+                       }
+               }
+
+               public bool IsEscapedIdentifier (Location loc)
+               {
+                       foreach (LocatedToken lt in escapedIdentifiers)
+                               if (lt.Location.Equals (loc))
+                                       return true;
+                       return false;
+               }
+
                
                //
                // Class variables
                // 
                static CharArrayHashtable[] keywords;
+               static Hashtable keywordStrings = new Hashtable ();
                static NumberStyles styles;
                static NumberFormatInfo csharp_format_info;
                
@@ -169,7 +207,7 @@ namespace Mono.CSharp
 
                static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
 
-               const int max_number_size = 128;
+               const int max_number_size = 512;
                static char [] number_builder = new char [max_number_size];
                static int number_pos;
                
@@ -197,6 +235,7 @@ namespace Mono.CSharp
                }
 
                static void AddKeyword (string kw, int token) {
+                       keywordStrings.Add (kw, kw);
                        if (keywords [kw.Length] == null) {
                                keywords [kw.Length] = new CharArrayHashtable (kw.Length);
                        }
@@ -333,9 +372,7 @@ namespace Mono.CSharp
                }
 
                public Location Location {
-                       get {
-                               return new Location (ref_line);
-                       }
+                       get { return current_location; }
                }
 
                void define (string def)
@@ -362,15 +399,13 @@ namespace Mono.CSharp
                                        define (def);
                        }
 
+                       xml_comment_buffer = new StringBuilder ();
+
                        //
                        // FIXME: This could be `Location.Push' but we have to
                        // find out why the MS compiler allows this
                        //
-                       Mono.CSharp.Location.Push (file);
-               }
-
-               public static void Cleanup () {
-                       identifiers = null;
+                       Mono.CSharp.Location.Push (file, 0);
                }
 
                static bool is_identifier_start_character (char c)
@@ -382,12 +417,17 @@ namespace Mono.CSharp
                {
                        return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
                }
-               
+
+               public static bool IsKeyword (string s)
+               {
+                       return keywordStrings [s] != null;
+               }
+
                public static bool IsValidIdentifier (string s)
                {
                        if (s == null || s.Length == 0)
                                return false;
-                       
+
                        if (!is_identifier_start_character (s [0]))
                                return false;
                        
@@ -407,10 +447,15 @@ namespace Mono.CSharp
 
                        switch (c){
                        case '{':
+                               val = Location;
                                return Token.OPEN_BRACE;
                        case '}':
+                               val = Location;
                                return Token.CLOSE_BRACE;
                        case '[':
+                               // To block doccomment inside attribute declaration.
+                               if (doc_state == XmlCommentState.Allowed)
+                                       doc_state = XmlCommentState.NotAllowed;
                                return Token.OPEN_BRACKET;
                        case ']':
                                return Token.CLOSE_BRACKET;
@@ -424,8 +469,16 @@ namespace Mono.CSharp
 
                                // Save current position and parse next token.
                                int old = reader.Position;
+                               int old_ref_line = ref_line;
+                               int old_col = col;
+
+                               // disable preprocessing directives when peeking
+                               process_directives = false;
                                int new_token = token ();
+                               process_directives = true;
                                reader.Position = old;
+                               ref_line = old_ref_line;
+                               col = old_col;
                                putback_char = -1;
 
                                if (new_token == Token.OPEN_PARENS)
@@ -440,11 +493,11 @@ namespace Mono.CSharp
 
                        case ',':
                                return Token.COMMA;
-                       case ':':
-                               return Token.COLON;
                        case ';':
+                               val = Location;
                                return Token.SEMICOLON;
                        case '~':
+                               val = Location;
                                return Token.TILDE;
                        case '?':
                                return Token.INTERR;
@@ -453,24 +506,32 @@ namespace Mono.CSharp
                        d = peekChar ();
                        if (c == '+'){
                                
-                               if (d == '+')
+                               if (d == '+') {
+                                       val = Location;
                                        t = Token.OP_INC;
+                               }
                                else if (d == '=')
                                        t = Token.OP_ADD_ASSIGN;
-                               else
+                               else {
+                                       val = Location;
                                        return Token.PLUS;
+                               }
                                doread = true;
                                return t;
                        }
                        if (c == '-'){
-                               if (d == '-')
+                               if (d == '-') {
+                                       val = Location;
                                        t = Token.OP_DEC;
+                               }
                                else if (d == '=')
                                        t = Token.OP_SUB_ASSIGN;
                                else if (d == '>')
                                        t = Token.OP_PTR;
-                               else
+                               else {
+                                       val = Location;
                                        return Token.MINUS;
+                               }
                                doread = true;
                                return t;
                        }
@@ -480,6 +541,7 @@ namespace Mono.CSharp
                                        doread = true;
                                        return Token.OP_NE;
                                }
+                               val = Location;
                                return Token.BANG;
                        }
 
@@ -499,6 +561,7 @@ namespace Mono.CSharp
                                        doread = true;
                                        return Token.OP_AND_ASSIGN;
                                }
+                               val = Location;
                                return Token.BITWISE_AND;
                        }
 
@@ -518,6 +581,7 @@ namespace Mono.CSharp
                                        doread = true;
                                        return Token.OP_MULT_ASSIGN;
                                }
+                               val = Location;
                                return Token.STAR;
                        }
 
@@ -578,6 +642,15 @@ namespace Mono.CSharp
                                }
                                return Token.OP_GT;
                        }
+                       if (c == ':'){
+                               if (d == ':'){
+                                       doread = true;
+                                       return Token.DOUBLE_COLON;
+                               }
+                               val = Location;
+                               return Token.COLON;
+                       }
+
                        return Token.ERROR;
                }
 
@@ -627,14 +700,7 @@ namespace Mono.CSharp
                {
                        return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
                }
-               
-               void hex_digits (int c)
-               {
-                       if (c != -1)
-                               number_builder [number_pos++] = (char) c;
-                       
-               }
-               
+                               
                int real_type_suffix (int c)
                {
                        int t;
@@ -677,10 +743,19 @@ namespace Mono.CSharp
                                                        // if we have not seen anything in between
                                                        // report this error
                                                        //
-                                                       Report.Warning (78, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
+                                                       Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
                                                }
-                                               goto case 'L';
-                                               
+                                               //
+                                               // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
+                                               // work around that.
+                                               //
+                                               //goto case 'L';
+                                               if (is_long)
+                                                       scanning = false;
+                                               is_long = true;
+                                               getChar ();
+                                               break;
+
                                        case 'L': 
                                                if (is_long)
                                                        scanning = false;
@@ -760,11 +835,17 @@ namespace Mono.CSharp
                                val = 0ul;
                                return Token.LITERAL_INTEGER;
                        }
+                       catch (FormatException) {
+                               Report.Error (1013, Location, "Invalid number");
+                               val = 0ul;
+                               return Token.LITERAL_INTEGER;
+                       }
                }
                
                int adjust_real (int t)
                {
                        string s = new String (number_builder, 0, number_pos);
+                       const string error_details = "Floating-point constant is outside the range of type `{0}'";
 
                        switch (t){
                        case Token.LITERAL_DECIMAL:
@@ -772,8 +853,7 @@ namespace Mono.CSharp
                                        val = System.Decimal.Parse (s, styles, csharp_format_info);
                                } catch (OverflowException) {
                                        val = 0m;     
-                                       error_details = "Floating-point constant is outside the range of the type 'decimal'";
-                                       Report.Error (594, Location, error_details);
+                                       Report.Error (594, Location, error_details, "decimal");
                                }
                                break;
                        case Token.LITERAL_FLOAT:
@@ -781,8 +861,7 @@ namespace Mono.CSharp
                                        val = (float) System.Double.Parse (s, styles, csharp_format_info);
                                } catch (OverflowException) {
                                        val = 0.0f;     
-                                       error_details = "Floating-point constant is outside the range of the type 'float'";
-                                       Report.Error (594, Location, error_details);
+                                       Report.Error (594, Location, error_details, "float");
                                }
                                break;
                                
@@ -793,8 +872,7 @@ namespace Mono.CSharp
                                        val = System.Double.Parse (s, styles, csharp_format_info);
                                } catch (OverflowException) {
                                        val = 0.0;     
-                                       error_details = "Floating-point constant is outside the range of the type 'double'";
-                                       Report.Error (594, Location, error_details);
+                                       Report.Error (594, Location, error_details, "double");
                                }
                                break;
                        }
@@ -827,6 +905,11 @@ namespace Mono.CSharp
                                val = 0ul;
                                return Token.LITERAL_INTEGER;
                        }
+                       catch (FormatException) {
+                               Report.Error (1013, Location, "Invalid number");
+                               val = 0ul;
+                               return Token.LITERAL_INTEGER;
+                       }
                        
                        return integer_type_suffix (ul, peekChar ());
                }
@@ -999,7 +1082,7 @@ namespace Mono.CSharp
                                        goto default;
                                return v;
                        default:
-                               Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);
+                               Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
                                return d;
                        }
                        getChar ();
@@ -1008,13 +1091,22 @@ namespace Mono.CSharp
 
                int getChar ()
                {
-                       if (putback_char != -1){
-                               int x = putback_char;
+                       int x;
+                       if (putback_char != -1) {
+                               x = putback_char;
                                putback_char = -1;
-
-                               return x;
                        }
-                       return reader.Read ();
+                       else
+                               x = reader.Read ();
+                       if (x == '\n') {
+                               line++;
+                               ref_line++;
+                               previous_col = col;
+                               col = 0;
+                       }
+                       else
+                               col++;
+                       return x;
                }
 
                int peekChar ()
@@ -1041,6 +1133,14 @@ namespace Mono.CSharp
                                Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
                                throw new Exception ("This should not happen putback on putback");
                        }
+                       if (c == '\n' || col == 0) {
+                               // It won't happen though.
+                               line--;
+                               ref_line--;
+                               col = previous_col;
+                       }
+                       else
+                               col--;
                        putback_char = c;
                }
 
@@ -1139,22 +1239,19 @@ namespace Mono.CSharp
                        cmd = static_cmd_arg.ToString ();
 
                        if (c == '\n'){
-                               line++;
-                               ref_line++;
                                return;
-                       } else if (c == '\r')
-                               col = 0;
+                       }
 
                        // skip over white space
                        while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
                                ;
 
                        if (c == '\n'){
-                               line++;
-                               ref_line++;
                                return;
                        } else if (c == '\r'){
-                               col = 0;
+                               return;
+                       } else if (c == -1){
+                               arg = "";
                                return;
                        }
                        
@@ -1165,11 +1262,6 @@ namespace Mono.CSharp
                                static_cmd_arg.Append ((char) c);
                        }
 
-                       if (c == '\n'){
-                               line++;
-                               ref_line++;
-                       } else if (c == '\r')
-                               col = 0;
                        arg = static_cmd_arg.ToString ().Trim ();
                }
 
@@ -1184,7 +1276,7 @@ namespace Mono.CSharp
                        if (arg == "default"){
                                ref_line = line;
                                ref_name = file_name;
-                               Location.Push (ref_name);
+                               Location.Push (ref_name, line);
                                return true;
                        } else if (arg == "hidden"){
                                //
@@ -1206,7 +1298,7 @@ namespace Mono.CSharp
                                        ref_name = Location.LookupFile (name);
                                        file_name.HasLineDirective = true;
                                        ref_name.HasLineDirective = true;
-                                       Location.Push (ref_name);
+                                       Location.Push (ref_name, ref_line);
                                } else {
                                        ref_line = System.Int32.Parse (arg);
                                }
@@ -1255,6 +1347,69 @@ namespace Mono.CSharp
                        }
                }
 
+               /// <summary>
+               /// Handles #pragma directive
+               /// </summary>
+               void PreProcessPragma (string arg)
+               {
+                       const string warning = "warning";
+                       const string w_disable = "warning disable";
+                       const string w_restore = "warning restore";
+
+                       if (arg == w_disable) {
+                               Report.RegisterWarningRegion (Location).WarningDisable (line);
+                               return;
+                       }
+
+                       if (arg == w_restore) {
+                               Report.RegisterWarningRegion (Location).WarningEnable (line);
+                               return;
+                       }
+
+                       if (arg.StartsWith (w_disable)) {
+                               int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
+                               foreach (int code in codes) {
+                                       if (code != 0)
+                                               Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
+                               }
+                               return;
+                       }
+
+                       if (arg.StartsWith (w_restore)) {
+                               int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
+                               Hashtable w_table = Report.warning_ignore_table;
+                               foreach (int code in codes) {
+                                       if (w_table != null && w_table.Contains (code))
+                                               Report.Warning (1635, 1, Location, String.Format ("Cannot restore warning `CS{0:0000}' because it was disabled globally", code));
+                                       Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
+                               }
+                               return;
+                       }
+
+                       if (arg.StartsWith (warning)) {
+                               Report.Warning (1634, 1, Location, "Expected disable or restore");
+                               return;
+                       }
+
+                       Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
+               }
+
+               int[] ParseNumbers (string text)
+               {
+                       string[] string_array = text.Split (',');
+                       int[] values = new int [string_array.Length];
+                       int index = 0;
+                       foreach (string string_code in string_array) {
+                               try {
+                                       values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
+                               }
+                               catch (FormatException) {
+                                       Report.Warning (1692, 1, Location, "Invalid number");
+                               }
+                       }
+                       return values;
+               }
+
                bool eval_val (string s)
                {
                        if (s == "true")
@@ -1370,7 +1525,7 @@ namespace Mono.CSharp
                                if (s [0] == '&'){
                                        if (len > 2 && s [1] == '&'){
                                                s = s.Substring (2);
-                                               return (va & pp_eq (ref s));
+                                               return (va & pp_and (ref s));
                                        } else {
                                                Error_InvalidDirective ();
                                                return false;
@@ -1419,7 +1574,7 @@ namespace Mono.CSharp
                
                void Error_InvalidDirective ()
                {
-                       Report.Error (1517, Location, "Invalid pre-processor directive");
+                       Report.Error (1517, Location, "Invalid preprocessor directive");
                }
 
                void Error_UnexpectedDirective (string extra)
@@ -1431,11 +1586,15 @@ namespace Mono.CSharp
 
                void Error_TokensSeen ()
                {
-                       Report.Error (
-                               1032, Location,
-                               "Cannot define or undefine pre-processor symbols after a token in the file");
+                       Report.Error (1032, Location,
+                               "Cannot define or undefine preprocessor symbols after first token in file");
                }
                
+               //
+               // Set to false to stop handling preprocesser directives
+               // 
+               bool process_directives = true;
+
                //
                // if true, then the code continues processing the code
                // if false, the code stays in a loop until another directive is
@@ -1458,15 +1617,19 @@ namespace Mono.CSharp
                        //
                        switch (cmd){
                        case "pragma":
-                               if (RootContext.Version != LanguageVersion.ISO_1)
+                               if (RootContext.Version == LanguageVersion.ISO_1) {
+                                       Report.FeatureIsNotStandardized (Location, "#pragma");
                                        return caller_is_taking;
-                               break;
-                               
+                               }
+
+                               PreProcessPragma (arg);
+                               return caller_is_taking;
+
                        case "line":
                                if (!PreProcessLine (arg))
                                        Report.Error (
                                                1576, Location,
-                                               "Argument to #line directive is missing or invalid");
+                                               "The line number specified for #line directive is missing or invalid");
                                return caller_is_taking;
 
                        case "region":
@@ -1517,10 +1680,14 @@ namespace Mono.CSharp
                                        int pop = (int) ifstack.Pop ();
                                        
                                        if (region_directive && ((pop & REGION) == 0))
-                                               Report.Error (1027, Location, "#endif directive expected");
+                                               Report.Error (1027, Location, "Expected `#endif' directive");
                                        else if (!region_directive && ((pop & REGION) != 0))
                                                Report.Error (1038, Location, "#endregion directive expected");
                                        
+                                       if (!region_directive && arg.Length != 0) {
+                                               Report.Error (1025, Location, "Single-line comment or end-of-line expected");
+                                       }
+                                       
                                        if (ifstack.Count == 0)
                                                return true;
                                        else {
@@ -1563,9 +1730,7 @@ namespace Mono.CSharp
 
                        case "else":
                                if (ifstack == null || ifstack.Count == 0){
-                                       Report.Error (
-                                               1028, Location,
-                                               "Unexpected processor directive (no #if for this #else)");
+                                       Error_UnexpectedDirective ("no #if for this #else");
                                        return true;
                                } else {
                                        int state = (int) ifstack.Peek ();
@@ -1627,16 +1792,16 @@ namespace Mono.CSharp
                                return true;
 
                        case "warning":
-                               Report.Warning (1030, Location, "#warning: '{0}'", arg);
+                               Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
                                return true;
                        }
 
-                       Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");
+                       Report.Error (1024, Location, "Wrong preprocessor directive");
                        return true;
 
                }
 
-               private int consume_string (bool quoted) 
+               private int consume_string (bool quoted)
                {
                        int c;
                        string_builder.Length = 0;
@@ -1656,11 +1821,7 @@ namespace Mono.CSharp
                                if (c == '\n'){
                                        if (!quoted)
                                                Report.Error (1010, Location, "Newline in constant");
-                                       line++;
-                                       ref_line++;
-                                       col = 0;
-                               } else
-                                       col++;
+                               }
 
                                if (!quoted){
                                        c = escape (c);
@@ -1678,25 +1839,39 @@ namespace Mono.CSharp
                {
                        int res = consume_identifier (s, false);
 
+                       if (doc_state == XmlCommentState.Allowed)
+                               doc_state = XmlCommentState.NotAllowed;
+                       switch (res) {
+                       case Token.USING:
+                       case Token.NAMESPACE:
+                               check_incorrect_doc_comment ();
+                               break;
+                       }
+
                        if (res == Token.PARTIAL) {
                                // Save current position and parse next token.
                                int old = reader.Position;
                                int old_putback = putback_char;
+                               int old_ref_line = ref_line;
+                               int old_col = col;
 
                                putback_char = -1;
 
                                int next_token = token ();
                                bool ok = (next_token == Token.CLASS) ||
                                        (next_token == Token.STRUCT) ||
-                                       (next_token == Token.INTERFACE);
+                                       (next_token == Token.INTERFACE) ||
+                                       (next_token == Token.ENUM); // "partial" is a keyword in 'partial enum', even though it's not valid
 
                                reader.Position = old;
+                               ref_line = old_ref_line;
+                               col = old_col;
                                putback_char = old_putback;
 
                                if (ok)
                                        return res;
                                else {
-                                       val = "partial";
+                                       val = new LocatedToken (Location, "partial");
                                        return Token.IDENTIFIER;
                                }
                        }
@@ -1707,11 +1882,13 @@ namespace Mono.CSharp
                private int consume_identifier (int s, bool quoted) 
                {
                        int pos = 1;
-                       int c;
+                       int c = -1;
                        
                        id_builder [0] = (char) s;
 
-                       while ((c = reader.Read ()) != -1) {
+                       current_location = new Location (ref_line, Col);
+
+                       while ((c = getChar ()) != -1) {
                                if (is_identifier_part_character ((char) c)){
                                        if (pos == max_id_size){
                                                Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
@@ -1719,10 +1896,10 @@ namespace Mono.CSharp
                                        }
                                        
                                        id_builder [pos++] = (char) c;
-                                       putback_char = -1;
-                                       col++;
+//                                     putback_char = -1;
                                } else {
-                                       putback_char = c;
+//                                     putback_char = c;
+                                       putback (c);
                                        break;
                                }
                        }
@@ -1733,8 +1910,10 @@ namespace Mono.CSharp
                        //
                        if (!quoted && (s >= 'a' || s == '_')){
                                int keyword = GetKeyword (id_builder, pos);
-                               if (keyword != -1)
+                               if (keyword != -1) {
+                                       val = Location;
                                        return keyword;
+                               }
                        }
 
                        //
@@ -1745,6 +1924,9 @@ namespace Mono.CSharp
                        if (identifiers [pos] != null) {
                                val = identifiers [pos][id_builder];
                                if (val != null) {
+                                       val = new LocatedToken (Location, (string) val);
+                                       if (quoted)
+                                               escapedIdentifiers.Add (val);
                                        return Token.IDENTIFIER;
                                }
                        }
@@ -1752,12 +1934,24 @@ namespace Mono.CSharp
                                identifiers [pos] = new CharArrayHashtable (pos);
 
                        val = new String (id_builder, 0, pos);
+                       if (RootContext.Version == LanguageVersion.ISO_1) {
+                               for (int i = 1; i < id_builder.Length; i += 3) {
+                                       if (id_builder [i] == '_' && (id_builder [i - 1] == '_' || id_builder [i + 1] == '_')) {
+                                               Report.Error (1638, Location, 
+                                                       "`{0}': Any identifier with double underscores cannot be used when ISO language version mode is specified", val.ToString ());
+                                               break;
+                                       }
+                               }
+                       }
 
                        char [] chars = new char [pos];
                        Array.Copy (id_builder, chars, pos);
 
                        identifiers [pos] [chars] = val;
 
+                       val = new LocatedToken (Location, (string) val);
+                       if (quoted)
+                               escapedIdentifiers.Add (val);
                        return Token.IDENTIFIER;
                }
                
@@ -1767,14 +1961,16 @@ namespace Mono.CSharp
                        bool doread = false;
                        int c;
 
+                       // Whether we have seen comments on the current line
+                       bool comments_seen = false;
+                       
                        val = null;
                        // optimization: eliminate col and implement #directive semantic correctly.
-                       for (;(c = getChar ()) != -1; col++) {
+                       for (;(c = getChar ()) != -1;) {
                                if (c == ' ')
                                        continue;
                                
                                if (c == '\t') {
-                                       col = (((col + 8) / 8) * 8) - 1;
                                        continue;
                                }
                                
@@ -1785,11 +1981,9 @@ namespace Mono.CSharp
                                        if (peekChar () == '\n')
                                                getChar ();
 
-                                       line++;
-                                       ref_line++;
-                                       col = 0;
                                        any_token_seen |= tokens_seen;
                                        tokens_seen = false;
+                                       comments_seen = false;
                                        continue;
                                }
 
@@ -1799,33 +1993,73 @@ namespace Mono.CSharp
                                
                                        if (d == '/'){
                                                getChar ();
+                                               if (RootContext.Documentation != null && peekChar () == '/') {
+                                                       getChar ();
+                                                       // Don't allow ////.
+                                                       if ((d = peekChar ()) != '/') {
+                                                               update_comment_location ();
+                                                               if (doc_state == XmlCommentState.Allowed)
+                                                                       handle_one_line_xml_comment ();
+                                                               else if (doc_state == XmlCommentState.NotAllowed)
+                                                                       warn_incorrect_doc_comment ();
+                                                       }
+                                               }
                                                while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
-                                                       col++;
                                                if (d == '\n'){
-                                                       line++;
-                                                       ref_line++;
-                                                       col = 0;
                                                }
                                                any_token_seen |= tokens_seen;
                                                tokens_seen = false;
+                                               comments_seen = false;
                                                continue;
                                        } else if (d == '*'){
                                                getChar ();
+                                               bool docAppend = false;
+                                               if (RootContext.Documentation != null && peekChar () == '*') {
+                                                       getChar ();
+                                                       update_comment_location ();
+                                                       // But when it is /**/, just do nothing.
+                                                       if (peekChar () == '/') {
+                                                               getChar ();
+                                                               continue;
+                                                       }
+                                                       if (doc_state == XmlCommentState.Allowed)
+                                                               docAppend = true;
+                                                       else if (doc_state == XmlCommentState.NotAllowed)
+                                                               warn_incorrect_doc_comment ();
+                                               }
+
+                                               int current_comment_start = 0;
+                                               if (docAppend) {
+                                                       current_comment_start = xml_comment_buffer.Length;
+                                                       xml_comment_buffer.Append (Environment.NewLine);
+                                               }
+
+                                               Location start_location = Location;
 
                                                while ((d = getChar ()) != -1){
                                                        if (d == '*' && peekChar () == '/'){
                                                                getChar ();
-                                                               col++;
+                                                               comments_seen = true;
                                                                break;
                                                        }
+                                                       if (docAppend)
+                                                               xml_comment_buffer.Append ((char) d);
+                                                       
                                                        if (d == '\n'){
-                                                               line++;
-                                                               ref_line++;
-                                                               col = 0;
                                                                any_token_seen |= tokens_seen;
                                                                tokens_seen = false;
+                                                               // 
+                                                               // Reset 'comments_seen' just to be consistent.
+                                                               // It doesn't matter either way, here.
+                                                               //
+                                                               comments_seen = false;
                                                        }
                                                }
+                                               if (!comments_seen)
+                                                       Report.Error (1035, start_location, "End-of-file found, '*/' expected");
+
+                                               if (docAppend)
+                                                       update_formatted_doc_comment (current_comment_start);
                                                continue;
                                        }
                                        goto is_punct_label;
@@ -1838,22 +2072,20 @@ namespace Mono.CSharp
                                }
 
                        is_punct_label:
+                               current_location = new Location (ref_line, Col);
                                if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
                                        tokens_seen = true;
                                        if (doread){
                                                getChar ();
-                                               col++;
                                        }
                                        return t;
                                }
 
                                // white space
                                if (c == '\n'){
-                                       line++;
-                                       ref_line++;
-                                       col = 0;
                                        any_token_seen |= tokens_seen;
                                        tokens_seen = false;
+                                       comments_seen = false;
                                        continue;
                                }
 
@@ -1873,25 +2105,32 @@ namespace Mono.CSharp
                                /* For now, ignore pre-processor commands */
                                // FIXME: In C# the '#' is not limited to appear
                                // on the first column.
-                               if (c == '#' && !tokens_seen){
+                               if (c == '#') {
+                                       // return NONE if we're not processing directives (during token peeks)
+                                       if (!process_directives)
+                                               return Token.NONE;
+
                                        bool cont = true;
+                                       if (tokens_seen || comments_seen) {
+                                               error_details = "Preprocessor directives must appear as the first" +
+                                              " non-whitespace character on a line.";
+
+                                               Report.Error (1040, Location, error_details);
+
+                                               return Token.ERROR;
+                                       }
                                        
                                start_again:
                                        
                                        cont = handle_preprocessing_directive (cont);
 
                                        if (cont){
-                                               col = 0;
                                                continue;
                                        }
-                                       col = 1;
 
                                        bool skipping = false;
-                                       for (;(c = getChar ()) != -1; col++){
+                                       for (;(c = getChar ()) != -1;){
                                                if (c == '\n'){
-                                                       col = 0;
-                                                       line++;
-                                                       ref_line++;
                                                        skipping = false;
                                                } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
                                                        continue;
@@ -1903,7 +2142,7 @@ namespace Mono.CSharp
                                        any_token_seen |= tokens_seen;
                                        tokens_seen = false;
                                        if (c == -1)
-                                               Report.Error (1027, Location, "#endif/#endregion expected");
+                                               Report.Error (1027, Location, "Expected `#endif' directive");
                                        continue;
                                }
                                
@@ -1918,6 +2157,10 @@ namespace Mono.CSharp
                                                Report.Error (1011, Location, error_details);
                                                return Token.ERROR;
                                        }
+                                       if (c == '\r' || c == '\n') {
+                                               Report.Error (1010, Location, "Newline in constant");
+                                               return Token.ERROR;
+                                       }
                                        c = escape (c);
                                        if (c == -1)
                                                return Token.ERROR;
@@ -1931,14 +2174,11 @@ namespace Mono.CSharp
 
                                                // Try to recover, read until newline or next "'"
                                                while ((c = getChar ()) != -1){
-                                                       if (c == '\n' || c == '\''){
-                                                               line++;
-                                                               ref_line++;
-                                                               col = 0;
+                                                       if (c == '\n'){
+                                                               break;
+                                                       }
+                                                       else if (c == '\'')
                                                                break;
-                                                       } else
-                                                               col++;
-                                                       
                                                }
                                                return Token.ERROR;
                                        }
@@ -1953,19 +2193,10 @@ namespace Mono.CSharp
                                        } else if (is_identifier_start_character ((char) c)){
                                                return consume_identifier (c, true);
                                        } else {
-                                               Report.Error (1033, Location, "'@' must be followed by string constant or identifier");
+                                               Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
                                        }
                                }
 
-                               if (c == '#') {
-                                       error_details = "Preprocessor directives must appear as the first non-whitespace " +
-                                               "character on a line.";
-
-                                       Report.Error (1040, Location, error_details);
-
-                                       return Token.ERROR;
-                               }
-
                                error_details = ((char)c).ToString ();
                                
                                return Token.ERROR;
@@ -1974,17 +2205,133 @@ namespace Mono.CSharp
                        return Token.EOF;
                }
 
+               //
+               // Handles one line xml comment
+               //
+               private void handle_one_line_xml_comment ()
+               {
+                       int c;
+                       while ((c = peekChar ()) == ' ')
+                               getChar (); // skip heading whitespaces.
+                       while ((c = peekChar ()) != -1 && c != '\n' && c != '\r') {
+                               xml_comment_buffer.Append ((char) getChar ());
+                       }
+                       if (c == '\r' || c == '\n')
+                               xml_comment_buffer.Append (Environment.NewLine);
+               }
+
+               //
+               // Remove heading "*" in Javadoc-like xml documentation.
+               //
+               private void update_formatted_doc_comment (int current_comment_start)
+               {
+                       int length = xml_comment_buffer.Length - current_comment_start;
+                       string [] lines = xml_comment_buffer.ToString (
+                               current_comment_start,
+                               length).Replace ("\r", "").Split ('\n');
+                       
+                       // The first line starts with /**, thus it is not target
+                       // for the format check.
+                       for (int i = 1; i < lines.Length; i++) {
+                               string s = lines [i];
+                               int idx = s.IndexOf ('*');
+                               string head = null;
+                               if (idx < 0) {
+                                       if (i < lines.Length - 1)
+                                               return;
+                                       head = s;
+                               } else
+                                       head = s.Substring (0, idx);
+                               foreach (char c in head)
+                                       if (c != ' ')
+                                               return;
+                               lines [i] = s.Substring (idx + 1);
+                       }
+                       xml_comment_buffer.Remove (current_comment_start, length);
+                       xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
+               }
+
+               //
+               // Updates current comment location.
+               //
+               private void update_comment_location ()
+               {
+                       if (current_comment_location.IsNull) {
+                               // "-2" is for heading "//" or "/*"
+                               current_comment_location =
+                                       new Location (ref_line, col - 2);
+                       }
+               }
+
+               //
+               // Checks if there was incorrect doc comments and raise
+               // warnings.
+               //
+               public void check_incorrect_doc_comment ()
+               {
+                       if (xml_comment_buffer.Length > 0)
+                               warn_incorrect_doc_comment ();
+               }
+
+               //
+               // Raises a warning when tokenizer found incorrect doccomment
+               // markup.
+               //
+               private void warn_incorrect_doc_comment ()
+               {
+                       if (doc_state != XmlCommentState.Error) {
+                               doc_state = XmlCommentState.Error;
+                               // in csc, it is 'XML comment is not placed on 
+                               // a valid language element'. But that does not
+                               // make sense.
+                               Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
+                       }
+               }
+
+               //
+               // Consumes the saved xml comment lines (if any)
+               // as for current target member or type.
+               //
+               public string consume_doc_comment ()
+               {
+                       if (xml_comment_buffer.Length > 0) {
+                               string ret = xml_comment_buffer.ToString ();
+                               reset_doc_comment ();
+                               return ret;
+                       }
+                       return null;
+               }
+
+               void reset_doc_comment ()
+               {
+                       xml_comment_buffer.Length = 0;
+                       current_comment_location = Location.Null;
+               }
+
                public void cleanup ()
                {
                        if (ifstack != null && ifstack.Count >= 1) {
                                int state = (int) ifstack.Pop ();
                                if ((state & REGION) != 0)
-                                       Report.Error (1038, "#endregion directive expected");
+                                       Report.Error (1038, Location, "#endregion directive expected");
                                else 
-                                       Report.Error (1027, "#endif directive expected");
+                                       Report.Error (1027, Location, "Expected `#endif' directive");
                        }
                                
                }
        }
-}
 
+       //
+       // Indicates whether it accepts XML documentation or not.
+       //
+       public enum XmlCommentState {
+               // comment is allowed in this state.
+               Allowed,
+               // comment is not allowed in this state.
+               NotAllowed,
+               // once comments appeared when it is NotAllowed, then the
+               // state is changed to it, until the state is changed to
+               // .Allowed.
+               Error
+       }
+}