2005-03-16 Alexander Olk <xenomorph2@onlinehome.de>
[mono.git] / mcs / gmcs / cs-tokenizer.cs
old mode 100755 (executable)
new mode 100644 (file)
index 3785eb0..e4daa8f
@@ -7,6 +7,7 @@
 // Licensed under the terms of the GNU GPL
 //
 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
+// (C) 2004 Novell, Inc
 //
 
 /*
@@ -43,6 +44,17 @@ namespace Mono.CSharp
                bool handle_constraints = false;
                bool handle_typeof = false;
 
+               //
+               // XML documentation buffer. The save point is used to divide
+               // comments on types and comments on members.
+               //
+               StringBuilder xml_comment_buffer;
+
+               //
+               // See comment on XmlCommentState enumeration.
+               //
+               XmlCommentState xmlDocState = XmlCommentState.Allowed;
+
                //
                // Whether tokens have been seen on this line
                //
@@ -54,6 +66,7 @@ namespace Mono.CSharp
                // after a token has been seen.
                //
                bool any_token_seen = false;
+
                static Hashtable tokenValues;
                
                private static Hashtable TokenValueName
@@ -154,7 +167,18 @@ namespace Mono.CSharp
                                handle_typeof = value;
                        }
                }
-               
+
+               public XmlCommentState doc_state {
+                       get { return xmlDocState; }
+                       set {
+                               if (value == XmlCommentState.Allowed) {
+                                       check_incorrect_doc_comment ();
+                                       consume_doc_comment ();
+                               }
+                               xmlDocState = value;
+                       }
+               }
+
                //
                // Class variables
                // 
@@ -387,6 +411,8 @@ namespace Mono.CSharp
                                        define (def);
                        }
 
+                       xml_comment_buffer = new StringBuilder ();
+
                        //
                        // FIXME: This could be `Location.Push' but we have to
                        // find out why the MS compiler allows this
@@ -473,6 +499,8 @@ namespace Mono.CSharp
                                return true;
                        else if ((the_token == Token.COMMA) || (the_token == Token.DOT))
                                goto start;
+                       else if (the_token == Token.INTERR)
+                               goto again;
                        else if (the_token == Token.OP_GENERICS_LT) {
                                if (!parse_less_than ())
                                        return false;
@@ -506,6 +534,9 @@ namespace Mono.CSharp
                        case '}':
                                return Token.CLOSE_BRACE;
                        case '[':
+                               // To block doccomment inside attribute declaration.
+                               if (doc_state == XmlCommentState.Allowed)
+                                       doc_state = XmlCommentState.NotAllowed;
                                return Token.OPEN_BRACKET;
                        case ']':
                                return Token.CLOSE_BRACKET;
@@ -716,11 +747,38 @@ namespace Mono.CSharp
                        deambiguate_close_parens++;
                }
 
+               public void PutbackNullable ()
+               {
+                       if (nullable_pos < 0)
+                               throw new Exception ();
+
+                       current_token = -1;
+                       val = null;
+                       reader.Position = nullable_pos;
+
+                       putback_char = '?';
+               }
+
+               public void PutbackCloseParens ()
+               {
+                       putback_char = ')';
+               }
+
                void Error_NumericConstantTooLong ()
                {
                        Report.Error (1021, Location, "Numeric constant too long");                     
                }
-               
+
+               int nullable_pos = -1;
+
+               public void CheckNullable (bool is_nullable)
+               {
+                       if (is_nullable)
+                               nullable_pos = reader.Position;
+                       else
+                               nullable_pos = -1;
+               }
+
                bool decimal_digits (int c)
                {
                        int d;
@@ -806,8 +864,17 @@ namespace Mono.CSharp
                                                        //
                                                        Report.Warning (78, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
                                                }
-                                               goto case 'L';
-                                               
+                                               //
+                                               // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so
+                                               // work around that.
+                                               //
+                                               //goto case 'L';
+                                               if (is_long)
+                                                       scanning = false;
+                                               is_long = true;
+                                               getChar ();
+                                               break;
+
                                        case 'L': 
                                                if (is_long)
                                                        scanning = false;
@@ -1879,6 +1946,15 @@ namespace Mono.CSharp
                {
                        int res = consume_identifier (s, false);
 
+                       if (doc_state == XmlCommentState.Allowed)
+                               doc_state = XmlCommentState.NotAllowed;
+                       switch (res) {
+                       case Token.USING:
+                       case Token.NAMESPACE:
+                               check_incorrect_doc_comment ();
+                               break;
+                       }
+
                        if (res == Token.PARTIAL) {
                                // Save current position and parse next token.
                                int old = reader.Position;
@@ -1964,10 +2040,11 @@ namespace Mono.CSharp
 
                int consume_whitespace ()
                {
-                       int t;
-                       bool doread = false;
                        int c;
 
+                       // Whether we have seen comments on the current line
+                       bool comments_seen = false;
+                       
                        val = null;
                        // optimization: eliminate col and implement #directive semantic correctly.
                        for (;(c = getChar ()) != -1; col++) {
@@ -1991,6 +2068,7 @@ namespace Mono.CSharp
                                        col = 0;
                                        any_token_seen |= tokens_seen;
                                        tokens_seen = false;
+                                       comments_seen = false;
                                        continue;
                                }
 
@@ -2000,6 +2078,16 @@ namespace Mono.CSharp
                                
                                        if (d == '/'){
                                                getChar ();
+                                               if (RootContext.Documentation != null && peekChar () == '/') {
+                                                       getChar ();
+                                                       // Don't allow ////.
+                                                       if ((d = peekChar ()) != '/') {
+                                                               if (doc_state == XmlCommentState.Allowed)
+                                                                       handle_one_line_xml_comment ();
+                                                               else if (doc_state == XmlCommentState.NotAllowed)
+                                                                       warn_incorrect_doc_comment ();
+                                                       }
+                                               }
                                                while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
                                                        col++;
                                                if (d == '\n'){
@@ -2009,24 +2097,55 @@ namespace Mono.CSharp
                                                }
                                                any_token_seen |= tokens_seen;
                                                tokens_seen = false;
+                                               comments_seen = false;
                                                continue;
                                        } else if (d == '*'){
                                                getChar ();
+                                               bool docAppend = false;
+                                               if (RootContext.Documentation != null && peekChar () == '*') {
+                                                       getChar ();
+                                                       // But when it is /**/, just do nothing.
+                                                       if (peekChar () == '/') {
+                                                               getChar ();
+                                                               continue;
+                                                       }
+                                                       if (doc_state == XmlCommentState.Allowed)
+                                                               docAppend = true;
+                                                       else if (doc_state == XmlCommentState.NotAllowed)
+                                                               warn_incorrect_doc_comment ();
+                                               }
+
+                                               int current_comment_start = 0;
+                                               if (docAppend) {
+                                                       current_comment_start = xml_comment_buffer.Length;
+                                                       xml_comment_buffer.Append (Environment.NewLine);
+                                               }
 
                                                while ((d = getChar ()) != -1){
                                                        if (d == '*' && peekChar () == '/'){
                                                                getChar ();
                                                                col++;
+                                                               comments_seen = true;
                                                                break;
                                                        }
+                                                       if (docAppend)
+                                                               xml_comment_buffer.Append ((char) d);
+                                                       
                                                        if (d == '\n'){
                                                                line++;
                                                                ref_line++;
                                                                col = 0;
                                                                any_token_seen |= tokens_seen;
                                                                tokens_seen = false;
+                                                               // 
+                                                               // Reset 'comments_seen' just to be consistent.
+                                                               // It doesn't matter either way, here.
+                                                               //
+                                                               comments_seen = false;
                                                        }
                                                }
+                                               if (docAppend)
+                                                       update_formatted_doc_comment (current_comment_start);
                                                continue;
                                        }
                                        goto is_punct_label;
@@ -2040,15 +2159,25 @@ namespace Mono.CSharp
                                        col = 0;
                                        any_token_seen |= tokens_seen;
                                        tokens_seen = false;
+                                       comments_seen = false;
                                        continue;
                                }
 
                                /* For now, ignore pre-processor commands */
                                // FIXME: In C# the '#' is not limited to appear
                                // on the first column.
-                               if (c == '#' && !tokens_seen){
+                               if (c == '#'{
                                        bool cont = true;
                                        
+                                       if (tokens_seen || comments_seen) {
+                                               error_details = "Preprocessor directives must appear as the first non-whitespace " +
+                                                       "character on a line.";
+
+                                               Report.Error (1040, Location, error_details);
+
+                                               return Token.ERROR;
+                                       }
+                                       
                                start_again:
                                        
                                        cont = handle_preprocessing_directive (cont);
@@ -2190,6 +2319,89 @@ namespace Mono.CSharp
                        return Token.ERROR;
                }
 
+               //
+               // Handles one line xml comment
+               //
+               private void handle_one_line_xml_comment ()
+               {
+                       int c;
+                       while ((c = peekChar ()) == ' ')
+                               getChar (); // skip heading whitespaces.
+                       while ((c = peekChar ()) != -1 && c != '\n' && c != '\r') {
+                               col++;
+                               xml_comment_buffer.Append ((char) getChar ());
+                       }
+                       if (c == '\r' || c == '\n')
+                               xml_comment_buffer.Append (Environment.NewLine);
+               }
+
+               //
+               // Remove heading "*" in Javadoc-like xml documentation.
+               //
+               private void update_formatted_doc_comment (int current_comment_start)
+               {
+                       int length = xml_comment_buffer.Length - current_comment_start;
+                       string [] lines = xml_comment_buffer.ToString (
+                               current_comment_start,
+                               length).Replace ("\r", "").Split ('\n');
+                       
+                       // The first line starts with /**, thus it is not target
+                       // for the format check.
+                       for (int i = 1; i < lines.Length; i++) {
+                               string s = lines [i];
+                               int idx = s.IndexOf ('*');
+                               string head = null;
+                               if (idx < 0) {
+                                       if (i < lines.Length - 1)
+                                               return;
+                                       head = s;
+                               } else
+                                       head = s.Substring (0, idx);
+                               foreach (char c in head)
+                                       if (c != ' ')
+                                               return;
+                               lines [i] = s.Substring (idx + 1);
+                       }
+                       xml_comment_buffer.Remove (current_comment_start, length);
+                       xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
+               }
+
+               //
+               // Checks if there was incorrect doc comments and raise
+               // warnings.
+               //
+               public void check_incorrect_doc_comment ()
+               {
+                       if (xml_comment_buffer.Length > 0)
+                               warn_incorrect_doc_comment ();
+               }
+
+               //
+               // Raises a warning when tokenizer found incorrect doccomment
+               // markup.
+               //
+               private void warn_incorrect_doc_comment ()
+               {
+                       doc_state = XmlCommentState.Error;
+                       // in csc, it is 'XML comment is not placed on a valid 
+                       // language element'. But that does not make sense.
+                       Report.Warning (1587, 2, Location, "XML comment is placed on an invalid language element which can not accept it.");
+               }
+
+               //
+               // Consumes the saved xml comment lines (if any)
+               // as for current target member or type.
+               //
+               public string consume_doc_comment ()
+               {
+                       if (xml_comment_buffer.Length > 0) {
+                               string ret = xml_comment_buffer.ToString ();
+                               xml_comment_buffer.Length = 0;
+                               return ret;
+                       }
+                       return null;
+               }
+
                public void cleanup ()
                {
                        if (ifstack != null && ifstack.Count >= 1) {
@@ -2202,5 +2414,19 @@ namespace Mono.CSharp
                                
                }
        }
+
+       //
+       // Indicates whether it accepts XML documentation or not.
+       //
+       public enum XmlCommentState {
+               // comment is allowed in this state.
+               Allowed,
+               // comment is not allowed in this state.
+               NotAllowed,
+               // once comments appeared when it is NotAllowed, then the
+               // state is changed to it, until the state is changed to
+               // .Allowed.
+               Error
+       }
 }