Merge pull request #686 from LHCGreg/12971-IsRedirectedAfterSet
[mono.git] / mcs / mcs / cs-tokenizer.cs
index b59aab461834e328f5b85890045af3f97b3fbd8f..37447f6df9b474f548b0f7c708b3f66ef719d6fe 100644 (file)
@@ -250,6 +250,9 @@ namespace Mono.CSharp
                public const int EvalCompilationUnitParserCharacter = 0x100001;
                public const int EvalUsingDeclarationsParserCharacter = 0x100002;
                public const int DocumentationXref = 0x100003;
+
+               const int UnicodeLS = 0x2028;
+               const int UnicodePS = 0x2029;
                
                //
                // XML documentation buffer. The save point is used to divide
@@ -1806,18 +1809,25 @@ namespace Mono.CSharp
                                x = reader.Read ();
                        }
                        
-                       if (x == '\r') {
-                               if (peek_char () == '\n') {
-                                       putback_char = -1;
-                               }
+                       if (x <= 13) {
+                               if (x == '\r') {
+                                       if (peek_char () == '\n') {
+                                               putback_char = -1;
+                                       }
 
-                               x = '\n';
-                               advance_line ();
-                       } else if (x == '\n') {
+                                       x = '\n';
+                                       advance_line ();
+                               } else if (x == '\n') {
+                                       advance_line ();
+                               } else {
+                                       col++;
+                               }
+                       } else if (x >= UnicodeLS && x <= UnicodePS) {
                                advance_line ();
                        } else {
                                col++;
                        }
+
                        return x;
                }
 
@@ -1849,7 +1859,7 @@ namespace Mono.CSharp
                                throw new InternalErrorException (string.Format ("Secondary putback [{0}] putting back [{1}] is not allowed", (char)putback_char, (char) c), Location);
                        }
 
-                       if (c == '\n' || col == 0) {
+                       if (c == '\n' || col == 0 || (c >= UnicodeLS && c <= UnicodePS)) {
                                // It won't happen though.
                                line--;
                                ref_line--;
@@ -1931,7 +1941,7 @@ namespace Mono.CSharp
                        int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
                        int pos = 0;
 
-                       while (c != -1 && c != '\n') {
+                       while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
                                if (c == '\\' && has_identifier_argument >= 0) {
                                        if (has_identifier_argument != 0) {
                                                has_identifier_argument = 1;
@@ -1958,10 +1968,7 @@ namespace Mono.CSharp
                                        // Eat single-line comments
                                        //
                                        get_char ();
-                                       do {
-                                               c = get_char ();
-                                       } while (c != -1 && c != '\n');
-
+                                       ReadToEndOfLine ();
                                        break;
                                }
 
@@ -2023,10 +2030,7 @@ namespace Mono.CSharp
                                //
                                // Eat any remaining characters to continue parsing on next line
                                //
-                               while (c != -1 && c != '\n') {
-                                       c = get_char ();
-                               }
-
+                               ReadToEndOfLine ();
                                return false;
                        }
 
@@ -2035,10 +2039,7 @@ namespace Mono.CSharp
                                //
                                // Eat any remaining characters to continue parsing on next line
                                //
-                               while (c != -1 && c != '\n') {
-                                       c = get_char ();
-                               }
-
+                               ReadToEndOfLine ();
                                return new_line != 0;
                        }
 
@@ -2052,13 +2053,11 @@ namespace Mono.CSharp
                                c = 0;
                        }
 
-                       if (c != '\n' && c != '/' && c != '"') {
+                       if (c != '\n' && c != '/' && c != '"' && c != UnicodeLS && c != UnicodePS) {
                                //
                                // Eat any remaining characters to continue parsing on next line
                                //
-                               while (c != -1 && c != '\n') {
-                                       c = get_char ();
-                               }
+                               ReadToEndOfLine ();
 
                                Report.Error (1578, loc, "Filename, single-line comment or end-of-line expected");
                                return true;
@@ -2074,16 +2073,15 @@ namespace Mono.CSharp
                                }
                        }
 
-                       if (c == '\n') {
+                       if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
+
                        } else if (c == '/') {
                                ReadSingleLineComment ();
                        } else {
                                //
                                // Eat any remaining characters to continue parsing on next line
                                //
-                               while (c != -1 && c != '\n') {
-                                       c = get_char ();
-                               }
+                               ReadToEndOfLine ();
 
                                Error_EndLineExpected ();
                                return true;
@@ -2318,7 +2316,7 @@ namespace Mono.CSharp
                string TokenizeFileName (ref int c)
                {
                        var string_builder = new StringBuilder ();
-                       while (c != -1 && c != '\n') {
+                       while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
                                c = get_char ();
                                if (c == '"') {
                                        c = get_char ();
@@ -2366,25 +2364,28 @@ namespace Mono.CSharp
                                        Report.Warning (1692, 1, Location, "Invalid number");
 
                                        // Read everything till the end of the line or file
-                                       do {
-                                               c = get_char ();
-                                       } while (c != -1 && c != '\n');
+                                       ReadToEndOfLine ();
                                }
                        }
 
                        return number;
                }
 
+               void ReadToEndOfLine ()
+               {
+                       int c;
+                       do {
+                               c = get_char ();
+                       } while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS);
+               }
+
                void ReadSingleLineComment ()
                {
                        if (peek_char () != '/')
                                Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
 
                        // Read everything till the end of the line or file
-                       int c;
-                       do {
-                               c = get_char ();
-                       } while (c != -1 && c != '\n');
+                       ReadToEndOfLine ();
                }
 
                /// <summary>
@@ -2410,7 +2411,7 @@ namespace Mono.CSharp
 
                                                var loc = Location;
 
-                                               if (c == '\n' || c == '/') {
+                                               if (c == '\n' || c == '/' || c == UnicodeLS || c == UnicodePS) {
                                                        if (c == '/')
                                                                ReadSingleLineComment ();
 
@@ -2436,7 +2437,7 @@ namespace Mono.CSharp
                                                                                Report.RegisterWarningRegion (loc).WarningEnable (loc, code, context);
                                                                        }
                                                                }
-                                                       } while (code >= 0 && c != '\n' && c != -1);
+                                                       } while (code >= 0 && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS);
                                                }
 
                                                return;
@@ -2446,8 +2447,7 @@ namespace Mono.CSharp
                                Report.Warning (1634, 1, Location, "Expected disable or restore");
 
                                // Eat any remaining characters on the line
-                               while (c != '\n' && c != -1)
-                                       c = get_char ();
+                               ReadToEndOfLine ();
 
                                return;
                        }
@@ -2927,7 +2927,7 @@ namespace Mono.CSharp
                                        return Token.LITERAL;
                                }
 
-                               if (c == '\n') {
+                               if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
                                        if (!quoted) {
                                                Report.Error (1010, Location, "Newline in constant");
 
@@ -3150,6 +3150,8 @@ namespace Mono.CSharp
                                        case '\v':
                                        case '\r':
                                        case '\n':
+                                       case UnicodeLS:
+                                       case UnicodePS:
                                        case '/':
                                                next = peek_token ();
                                                if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
@@ -3180,6 +3182,7 @@ namespace Mono.CSharp
                                                case Token.FOREACH:
                                                case Token.TYPEOF:
                                                case Token.WHILE:
+                                               case Token.SWITCH:
                                                case Token.USING:
                                                case Token.DEFAULT:
                                                case Token.DELEGATE:
@@ -3367,7 +3370,7 @@ namespace Mono.CSharp
                                                        }
                                                }
 
-                                               while ((d = get_char ()) != -1 && d != '\n');
+                                               ReadToEndOfLine ();
 
                                                any_token_seen |= tokens_seen;
                                                tokens_seen = false;
@@ -3405,7 +3408,7 @@ namespace Mono.CSharp
                                                        if (docAppend)
                                                                xml_comment_buffer.Append ((char) d);
                                                        
-                                                       if (d == '\n'){
+                                                       if (d == '\n' || d == UnicodeLS || d == UnicodePS){
                                                                any_token_seen |= tokens_seen;
                                                                tokens_seen = false;
                                                                // 
@@ -3455,6 +3458,8 @@ namespace Mono.CSharp
                                        return is_number (c, false);
 
                                case '\n': // white space
+                               case UnicodeLS:
+                               case UnicodePS:
                                        any_token_seen |= tokens_seen;
                                        tokens_seen = false;
                                        comments_seen = false;
@@ -3491,7 +3496,7 @@ namespace Mono.CSharp
                                                        continue;
                                                }
 
-                                               if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' )
+                                               if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' || c == UnicodeLS || c == UnicodePS)
                                                        continue;
 
                                                if (c == '#') {
@@ -3575,7 +3580,7 @@ namespace Mono.CSharp
                                return Token.LITERAL;
                        }
 
-                       if (c == '\n') {
+                       if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
                                Report.Error (1010, start_location, "Newline in constant");
                                return Token.ERROR;
                        }
@@ -3596,7 +3601,7 @@ namespace Mono.CSharp
 
                                // Try to recover, read until newline or next "'"
                                while ((c = get_char ()) != -1) {
-                                       if (c == '\n' || c == '\'')
+                                       if (c == '\n' || c == '\'' || c == UnicodeLS || c == UnicodePS)
                                                break;
                                }
                        }