2007-02-13 Martin Baulig <martin@ximian.com>
[mono.git] / mcs / mcs / cs-tokenizer.cs
index c94047322cf4444794735fb0e6dc442a7ca07b5e..e288f8d9dfc330d54839c4a957b3b9e22ce765ec 100644 (file)
@@ -3,18 +3,14 @@
 //                  This also implements the preprocessor
 //
 // Author: Miguel de Icaza (miguel@gnu.org)
+//         Marek Safar (marek.safar@seznam.cz)
 //
 // Licensed under the terms of the GNU GPL
 //
 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
 // (C) 2004 Novell, Inc
 //
-
-/*
- * TODO:
- *   Make sure we accept the proper Unicode ranges, per the spec.
- *   Report error 1032
-*/
+//
 
 using System;
 using System.Text;
@@ -42,6 +38,9 @@ namespace Mono.CSharp
                bool handle_get_set = false;
                bool handle_remove_add = false;
                bool handle_assembly = false;
+               bool handle_constraints = false;
+               bool handle_typeof = false;
+               bool linq;
                Location current_location;
                Location current_comment_location = Location.Null;
                ArrayList escapedIdentifiers = new ArrayList ();
@@ -70,6 +69,7 @@ namespace Mono.CSharp
                bool any_token_seen = false;
 
                static Hashtable tokenValues;
+               static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 
                private static Hashtable TokenValueName
                {
@@ -150,6 +150,26 @@ namespace Mono.CSharp
                        }
                }
 
+               public bool ConstraintsParsing {
+                       get {
+                               return handle_constraints;
+                       }
+
+                       set {
+                               handle_constraints = value;
+                       }
+               }
+
+               public bool TypeOfParsing {
+                       get {
+                               return handle_typeof;
+                       }
+
+                       set {
+                               handle_typeof = value;
+                       }
+               }
+
                public XmlCommentState doc_state {
                        get { return xmlDocState; }
                        set {
@@ -169,7 +189,6 @@ namespace Mono.CSharp
                        return false;
                }
 
-               
                //
                // Class variables
                // 
@@ -190,7 +209,6 @@ namespace Mono.CSharp
                Hashtable defines;
 
                const int TAKING        = 1;
-               const int TAKEN_BEFORE  = 2;
                const int ELSE_SEEN     = 4;
                const int PARENT_TAKING = 8;
                const int REGION        = 16;           
@@ -234,6 +252,62 @@ namespace Mono.CSharp
                        }
                }
 
+               //
+               // This is used when the tokenizer needs to save
+               // the current position as it needs to do some parsing
+               // on its own to deamiguate a token in behalf of the
+               // parser.
+               //
+               Stack position_stack = new Stack (2);
+               class Position {
+                       public int position;
+                       public int ref_line;
+                       public int col;
+                       public int putback_char;
+                       public int previous_col;
+                       public Stack ifstack;
+                       public int parsing_generic_less_than;
+                       public int current_token;
+
+                       public Position (Tokenizer t)
+                       {
+                               position = t.reader.Position;
+                               ref_line = t.ref_line;
+                               col = t.col;
+                               putback_char = t.putback_char;
+                               previous_col = t.previous_col;
+                               if (t.ifstack != null && t.ifstack.Count != 0)
+                                       ifstack = (Stack)t.ifstack.Clone ();
+                               parsing_generic_less_than = t.parsing_generic_less_than;
+                               current_token = t.current_token;
+                       }
+               }
+               
+               public void PushPosition ()
+               {
+                       position_stack.Push (new Position (this));
+               }
+
+               public void PopPosition ()
+               {
+                       Position p = (Position) position_stack.Pop ();
+
+                       reader.Position = p.position;
+                       ref_line = p.ref_line;
+                       col = p.col;
+                       putback_char = p.putback_char;
+                       previous_col = p.previous_col;
+                       ifstack = p.ifstack;
+                       parsing_generic_less_than = p.parsing_generic_less_than;
+                       current_token = p.current_token;
+               }
+
+               // Do not reset the position, ignore it.
+               public void DiscardPosition ()
+               {
+                       position_stack.Pop ();
+               }
+               
                static void AddKeyword (string kw, int token) {
                        keywordStrings.Add (kw, kw);
                        if (keywords [kw.Length] == null) {
@@ -330,6 +404,9 @@ namespace Mono.CSharp
                        AddKeyword ("volatile", Token.VOLATILE);
                        AddKeyword ("while", Token.WHILE);
                        AddKeyword ("partial", Token.PARTIAL);
+#if GMCS_SOURCE
+                       AddKeyword ("where", Token.WHERE);
+#endif
                }
 
                //
@@ -366,7 +443,10 @@ namespace Mono.CSharp
                                return -1;
                        if (handle_assembly == false && res == Token.ASSEMBLY)
                                return -1;
-
+#if GMCS_SOURCE
+                       if (handle_constraints == false && res == Token.WHERE)
+                               return -1;
+#endif
                        return res;
                        
                }
@@ -389,6 +469,7 @@ namespace Mono.CSharp
                {
                        this.ref_name = file;
                        this.file_name = file;
+                       linq = RootContext.Version == LanguageVersion.LINQ;
                        reader = input;
                        
                        putback_char = -1;
@@ -410,12 +491,13 @@ namespace Mono.CSharp
 
                static bool is_identifier_start_character (char c)
                {
-                       return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c);
+                       return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter (c);
                }
 
                static bool is_identifier_part_character (char c)
                {
-                       return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
+                       return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') ||
+                               Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
                }
 
                public static bool IsKeyword (string s)
@@ -438,6 +520,259 @@ namespace Mono.CSharp
                        return true;
                }
 
+               bool parse_generic_dimension (out int dimension)
+               {
+                       dimension = 1;
+
+               again:
+                       int the_token = token ();
+                       if (the_token == Token.OP_GENERICS_GT)
+                               return true;
+                       else if (the_token == Token.COMMA) {
+                               dimension++;
+                               goto again;
+                       }
+
+                       return false;
+               }
+
+               bool parse_less_than ()
+               {
+               start:
+                       int the_token = token ();
+                       if (the_token == Token.OPEN_BRACKET) {
+                               do {
+                                       the_token = token ();
+                               } while (the_token != Token.CLOSE_BRACKET);
+                               the_token = token ();
+                       }
+                       switch (the_token) {
+                       case Token.IDENTIFIER:
+                       case Token.OBJECT:
+                       case Token.STRING:
+                       case Token.BOOL:
+                       case Token.DECIMAL:
+                       case Token.FLOAT:
+                       case Token.DOUBLE:
+                       case Token.SBYTE:
+                       case Token.BYTE:
+                       case Token.SHORT:
+                       case Token.USHORT:
+                       case Token.INT:
+                       case Token.UINT:
+                       case Token.LONG:
+                       case Token.ULONG:
+                       case Token.CHAR:
+                       case Token.VOID:
+                               break;
+
+                       default:
+                               return false;
+                       }
+               again:
+                       the_token = token ();
+
+                       if (the_token == Token.OP_GENERICS_GT)
+                               return true;
+                       else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
+                               goto start;
+                       else if (the_token == Token.INTERR || the_token == Token.STAR)
+                               goto again;
+                       else if (the_token == Token.OP_GENERICS_LT) {
+                               if (!parse_less_than ())
+                                       return false;
+                               goto again;
+                       } else if (the_token == Token.OPEN_BRACKET) {
+                       rank_specifiers:
+                               the_token = token ();
+                               if (the_token == Token.CLOSE_BRACKET)
+                                       goto again;
+                               else if (the_token == Token.COMMA)
+                                       goto rank_specifiers;
+                               return false;
+                       }
+
+                       return false;
+               }
+
+#if GMCS_SOURCE
+               public void PutbackNullable ()
+               {
+                       if (nullable_pos < 0)
+                               throw new Exception ();
+
+                       current_token = -1;
+                       val = null;
+                       reader.Position = nullable_pos;
+
+                       putback_char = '?';
+               }
+
+               public void PutbackCloseParens ()
+               {
+                       putback_char = ')';
+               }
+
+
+               int nullable_pos = -1;
+
+               public void CheckNullable (bool is_nullable)
+               {
+                       if (is_nullable)
+                               nullable_pos = reader.Position;
+                       else
+                               nullable_pos = -1;
+               }
+#endif
+               
+               public int peek_token ()
+               {
+                       int the_token;
+
+                       PushPosition ();
+                       the_token = token ();
+                       PopPosition ();
+                       
+                       return the_token;
+               }
+               
+               bool parse_namespace_or_typename (int next)
+               {
+                       if (next == -1)
+                               next = peek_token ();
+                       while (next == Token.IDENTIFIER){
+                               token ();
+                         again:
+                               next = peek_token ();
+                               if (next == Token.DOT || next == Token.DOUBLE_COLON){
+                                       token ();
+                                       next = peek_token ();
+                                       continue;
+                               }
+                               if (next == Token.OP_GENERICS_LT){
+                                       token ();
+                                       if (!parse_less_than ())
+                                               return false;
+                                       goto again;
+                               }
+                               return true;
+                       }
+
+                       return false;
+               }
+
+               bool is_simple_type (int token)
+               {
+                       return  (token == Token.BOOL ||
+                                token == Token.DECIMAL ||
+                                token == Token.SBYTE ||
+                                token == Token.BYTE ||
+                                token == Token.SHORT ||
+                                token == Token.USHORT ||
+                                token == Token.INT ||
+                                token == Token.UINT ||
+                                token == Token.LONG ||
+                                token == Token.ULONG ||
+                                token == Token.CHAR ||
+                                token == Token.FLOAT ||
+                                token == Token.DOUBLE);
+               }
+
+               bool is_builtin_reference_type (int token)
+               {
+                       return (token == Token.OBJECT || token == Token.STRING);
+               }
+
+               bool parse_opt_rank (int next)
+               {
+                       while (true){
+                               if (next != Token.OPEN_BRACKET)
+                                       return true;
+
+                               token ();
+                               while (true){
+                                       next = token ();
+                                       if (next == Token.CLOSE_BRACKET){
+                                               next = peek_token ();
+                                               break;
+                                       }
+                                       if (next == Token.COMMA)
+                                               continue;
+                                       
+                                       return false;
+                               }
+                       }
+               }
+                       
+               bool parse_type ()
+               {
+                       int next = peek_token ();
+                       
+                       if (is_simple_type (next)){
+                               token ();
+                               next = peek_token ();
+                               if (next == Token.INTERR)
+                                       token ();
+                               return parse_opt_rank (peek_token ());
+                       }
+                       if (parse_namespace_or_typename (next)){
+                               next = peek_token ();
+                               if (next == Token.INTERR)
+                                       token ();
+                               return parse_opt_rank (peek_token ());
+                       } else if (is_builtin_reference_type (next)){
+                               token ();
+                               return parse_opt_rank (peek_token ());
+                       }
+                       
+                       return false;
+               }
+               
+               //
+               // Invoked after '(' has been seen and tries to parse:
+               // type identifier [, type identifier]*
+               //
+               // if this is the case, instead of returning an
+               // OPEN_PARENS token we return a special token that
+               // triggers lambda parsing.
+               //
+               // This is needed because we can not introduce the
+               // explicitly_typed_lambda_parameter_list after a '(' in the
+               // grammar without introducing reduce/reduce conflicts.
+               //
+               // We need to parse a type and if it is followed by an
+               // identifier, we know it has to be parsed as a lambda
+               // expression.  
+               //
+               // the type expression can be prefixed with `ref' or `out'
+               //
+               public bool parse_lambda_parameters ()
+               {
+                       while (true){
+                               int next = peek_token ();
+
+                               if (next == Token.REF || next == Token.OUT)
+                                       token ();
+                                                
+                               if (parse_type ()){
+                                       next = peek_token ();
+                                       if (next == Token.IDENTIFIER){
+                                               token ();
+                                               next = peek_token ();
+                                               if (next == Token.COMMA){
+                                                       token ();
+                                                       continue;
+                                               }
+                                               if (next == Token.CLOSE_PARENS)
+                                                       return true;
+                                       }
+                               }
+                               return false;
+                       }
+               }
+
+               int parsing_generic_less_than = 0;
+               
                int is_punct (char c, ref bool doread)
                {
                        int d;
@@ -460,26 +795,28 @@ namespace Mono.CSharp
                        case ']':
                                return Token.CLOSE_BRACKET;
                        case '(':
-                               return Token.OPEN_PARENS;
+                               if (linq){
+                                       PushPosition ();
+                                       bool have_lambda_parameter = parse_lambda_parameters ();
+                                       PopPosition ();
+                                       
+                                       if (have_lambda_parameter)
+                                               return Token.OPEN_PARENS_LAMBDA;
+                                       else
+                                               return Token.OPEN_PARENS;
+                               } else
+                                       return Token.OPEN_PARENS;
                        case ')': {
                                if (deambiguate_close_parens == 0)
                                        return Token.CLOSE_PARENS;
 
                                --deambiguate_close_parens;
 
-                               // Save current position and parse next token.
-                               int old = reader.Position;
-                               int old_ref_line = ref_line;
-                               int old_col = col;
-
-                               // disable preprocessing directives when peeking
-                               process_directives = false;
-                               int new_token = token ();
-                               process_directives = true;
-                               reader.Position = old;
-                               ref_line = old_ref_line;
-                               col = old_col;
-                               putback_char = -1;
+                               PushPosition ();
+
+                               int new_token = xtoken ();
+
+                               PopPosition ();
 
                                if (new_token == Token.OPEN_PARENS)
                                        return Token.CLOSE_PARENS_OPEN_PARENS;
@@ -502,7 +839,70 @@ namespace Mono.CSharp
                        case '?':
                                return Token.INTERR;
                        }
+#if GMCS_SOURCE
+                       if (c == '<') {
+                               if (parsing_generic_less_than++ > 0)
+                                       return Token.OP_GENERICS_LT;
+
+                               if (handle_typeof) {
+                                       int dimension;
+                                       PushPosition ();
+                                       if (parse_generic_dimension (out dimension)) {
+                                               val = dimension;
+                                               DiscardPosition ();
+                                               return Token.GENERIC_DIMENSION;
+                                       }
+                                       PopPosition ();
+                               }
+
+                               // Save current position and parse next token.
+                               PushPosition ();
+                               bool is_generic_lt = parse_less_than ();
+                               PopPosition ();
+
+                               if (is_generic_lt) {
+                                       return Token.OP_GENERICS_LT;
+                               } else
+                                       parsing_generic_less_than = 0;
+
+                               d = peekChar ();
+                               if (d == '<'){
+                                       getChar ();
+                                       d = peekChar ();
+
+                                       if (d == '='){
+                                               doread = true;
+                                               return Token.OP_SHIFT_LEFT_ASSIGN;
+                                       }
+                                       return Token.OP_SHIFT_LEFT;
+                               } else if (d == '='){
+                                       doread = true;
+                                       return Token.OP_LE;
+                               }
+                               return Token.OP_LT;
+                       } else if (c == '>') {
+                               if (parsing_generic_less_than > 0) {
+                                       parsing_generic_less_than--;
+                                       return Token.OP_GENERICS_GT;
+                               }
+
+                               d = peekChar ();
+                               if (d == '>'){
+                                       getChar ();
+                                       d = peekChar ();
 
+                                       if (d == '='){
+                                               doread = true;
+                                               return Token.OP_SHIFT_RIGHT_ASSIGN;
+                                       }
+                                       return Token.OP_SHIFT_RIGHT;
+                               } else if (d == '='){
+                                       doread = true;
+                                       return Token.OP_GE;
+                               }
+                               return Token.OP_GT;
+                       }
+#endif
                        d = peekChar ();
                        if (c == '+'){
                                
@@ -550,6 +950,11 @@ namespace Mono.CSharp
                                        doread = true;
                                        return Token.OP_EQ;
                                }
+                               if (d == '>'){
+                                       doread = true;
+                                       val = Location;
+                                       return Token.ARROW;
+                               }
                                return Token.ASSIGN;
                        }
 
@@ -609,6 +1014,7 @@ namespace Mono.CSharp
                                return Token.CARRET;
                        }
 
+#if !GMCS_SOURCE
                        if (c == '<'){
                                if (d == '<'){
                                        getChar ();
@@ -642,6 +1048,7 @@ namespace Mono.CSharp
                                }
                                return Token.OP_GT;
                        }
+#endif
                        if (c == ':'){
                                if (d == ':'){
                                        doread = true;
@@ -656,17 +1063,17 @@ namespace Mono.CSharp
 
                int deambiguate_close_parens = 0;
 
-               public void Deambiguate_CloseParens ()
+               public void Deambiguate_CloseParens (object expression)
                {
                        putback (')');
+
+                       // When any binary operation is used we are sure it is not a cast
+                       if (expression is Binary)
+                               return;
+
                        deambiguate_close_parens++;
                }
 
-               void Error_NumericConstantTooLong ()
-               {
-                       Report.Error (1021, Location, "Numeric constant too long");                     
-               }
-               
                bool decimal_digits (int c)
                {
                        int d;
@@ -1095,8 +1502,7 @@ namespace Mono.CSharp
                        if (putback_char != -1) {
                                x = putback_char;
                                putback_char = -1;
-                       }
-                       else
+                       } else
                                x = reader.Read ();
                        if (x == '\n') {
                                line++;
@@ -1174,6 +1580,7 @@ namespace Mono.CSharp
                        case Token.LITERAL_STRING:
                        case Token.BASE:
                        case Token.CHECKED:
+                       case Token.DELEGATE:
                        case Token.FALSE:
                        case Token.FIXED:
                        case Token.NEW:
@@ -1185,6 +1592,9 @@ namespace Mono.CSharp
                        case Token.TYPEOF:
                        case Token.UNCHECKED:
                        case Token.UNSAFE:
+#if GMCS_SOURCE
+                       case Token.DEFAULT:
+#endif
 
                                //
                                // These can be part of a member access
@@ -1206,10 +1616,26 @@ namespace Mono.CSharp
                }
 
                public int token ()
-                {
+               {
                        current_token = xtoken ();
-                        return current_token;
-                }
+
+#if GMCS_SOURCE
+                       if (current_token != Token.DEFAULT)
+                               return current_token;
+
+                       PushPosition();
+                       int c = xtoken();
+                       if (c == -1)
+                               current_token = Token.ERROR;
+                       else if (c == Token.OPEN_PARENS)
+                               current_token = Token.DEFAULT_OPEN_PARENS;
+                       else if (c == Token.COLON)
+                               current_token = Token.DEFAULT_COLON;
+                       else
+                               PopPosition();
+#endif
+                       return current_token;
+               }
 
                static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
                
@@ -1224,8 +1650,7 @@ namespace Mono.CSharp
                        // skip over white space
                        while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
                                ;
-
-
+                               
                        while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
                                if (is_identifier_part_character ((char) c)){
                                        static_cmd_arg.Append ((char) c);
@@ -1238,7 +1663,7 @@ namespace Mono.CSharp
 
                        cmd = static_cmd_arg.ToString ();
 
-                       if (c == '\n'){
+                       if (c == '\n' || c == '\r'){
                                return;
                        }
 
@@ -1262,7 +1687,7 @@ namespace Mono.CSharp
                                static_cmd_arg.Append ((char) c);
                        }
 
-                       arg = static_cmd_arg.ToString ().Trim ();
+                       arg = static_cmd_arg.ToString ();
                }
 
                //
@@ -1284,7 +1709,7 @@ namespace Mono.CSharp
                                //
                                return true;
                        }
-
+                       
                        try {
                                int pos;
 
@@ -1312,16 +1737,15 @@ namespace Mono.CSharp
                //
                // Handles #define and #undef
                //
-               void PreProcessDefinition (bool is_define, string arg)
+               void PreProcessDefinition (bool is_define, string arg, bool caller_is_taking)
                {
                        if (arg.Length == 0 || arg == "true" || arg == "false"){
                                Report.Error (1001, Location, "Missing identifer to pre-processor directive");
                                return;
                        }
 
-                       char[] whitespace = { ' ', '\t' };
-                       if (arg.IndexOfAny (whitespace) != -1){
-                               Report.Error (1025, Location, "Single-line comment or end-of-line expected");
+                       if (arg.IndexOfAny (simple_whitespaces) != -1){
+                               Error_EndLineExpected ();
                                return;
                        }
 
@@ -1335,6 +1759,9 @@ namespace Mono.CSharp
                                }
                        }
 
+                       if (!caller_is_taking)
+                               return;
+
                        if (is_define){
                                if (defines == null)
                                        defines = new Hashtable ();
@@ -1435,7 +1862,7 @@ namespace Mono.CSharp
                                
                                if (c == '('){
                                        s = s.Substring (1);
-                                       bool val = pp_expr (ref s);
+                                       bool val = pp_expr (ref s, false);
                                        if (s.Length > 0 && s [0] == ')'){
                                                s = s.Substring (1);
                                                return val;
@@ -1538,7 +1965,7 @@ namespace Mono.CSharp
                //
                // Evaluates an expression for `#if' or `#elif'
                //
-               bool pp_expr (ref string s)
+               bool pp_expr (ref string s, bool isTerm)
                {
                        bool va = pp_and (ref s);
                        s = s.Trim ();
@@ -1549,12 +1976,16 @@ namespace Mono.CSharp
                                if (c == '|'){
                                        if (len > 2 && s [1] == '|'){
                                                s = s.Substring (2);
-                                               return va | pp_expr (ref s);
+                                               return va | pp_expr (ref s, isTerm);
                                        } else {
                                                Error_InvalidDirective ();
                                                return false;
                                        }
-                               } 
+                               }
+                               if (isTerm) {
+                                       Error_EndLineExpected ();
+                                       return false;
+                               }
                        }
                        
                        return va;
@@ -1562,15 +1993,19 @@ namespace Mono.CSharp
 
                bool eval (string s)
                {
-                       bool v = pp_expr (ref s);
+                       bool v = pp_expr (ref s, true);
                        s = s.Trim ();
                        if (s.Length != 0){
-                               Error_InvalidDirective ();
                                return false;
                        }
 
                        return v;
                }
+
+               void Error_NumericConstantTooLong ()
+               {
+                       Report.Error (1021, Location, "Numeric constant too long");                     
+               }
                
                void Error_InvalidDirective ()
                {
@@ -1589,17 +2024,24 @@ namespace Mono.CSharp
                        Report.Error (1032, Location,
                                "Cannot define or undefine preprocessor symbols after first token in file");
                }
-               
-               //
-               // Set to false to stop handling preprocesser directives
-               // 
-               bool process_directives = true;
 
+               void Eror_WrongPreprocessorLocation ()
+               {
+                       Report.Error (1040, Location,
+                               "Preprocessor directives must appear as the first non-whitespace character on a line");
+               }
+
+               void Error_EndLineExpected ()
+               {
+                       Report.Error (1025, Location, "Single-line comment or end-of-line expected");
+               }
+               
                //
                // if true, then the code continues processing the code
                // if false, the code stays in a loop until another directive is
                // reached.
-               //
+               // When caller_is_taking is false we ignore all directives except the ones
+               // which can help us to identify where the #if block ends
                bool handle_preprocessing_directive (bool caller_is_taking)
                {
                        string cmd, arg;
@@ -1610,94 +2052,69 @@ namespace Mono.CSharp
                        // Eat any trailing whitespaces and single-line comments
                        if (arg.IndexOf ("//") != -1)
                                arg = arg.Substring (0, arg.IndexOf ("//"));
-                       arg = arg.TrimEnd (' ', '\t');
+                       arg = arg.Trim (simple_whitespaces);
 
                        //
                        // The first group of pre-processing instructions is always processed
                        //
                        switch (cmd){
-                       case "pragma":
-                               if (RootContext.Version == LanguageVersion.ISO_1) {
-                                       Report.FeatureIsNotStandardized (Location, "#pragma");
-                                       return caller_is_taking;
-                               }
-
-                               PreProcessPragma (arg);
-                               return caller_is_taking;
-
-                       case "line":
-                               if (!PreProcessLine (arg))
-                                       Report.Error (
-                                               1576, Location,
-                                               "The line number specified for #line directive is missing or invalid");
-                               return caller_is_taking;
-
                        case "region":
                                region_directive = true;
                                arg = "true";
                                goto case "if";
 
                        case "endregion":
-                               region_directive = true;
-                               goto case "endif";
-                               
-                       case "if":
-                               if (arg.Length == 0){
-                                       Error_InvalidDirective ();
+                               if (ifstack == null || ifstack.Count == 0){
+                                       Error_UnexpectedDirective ("no #region for this #endregion");
                                        return true;
                                }
-                               bool taking = false;
+                               int pop = (int) ifstack.Pop ();
+                                       
+                               if ((pop & REGION) == 0)
+                                       Report.Error (1027, Location, "Expected `#endif' directive");
+                                       
+                               return caller_is_taking;
+                               
+                       case "if":
                                if (ifstack == null)
                                        ifstack = new Stack (2);
 
+                               int flags = region_directive ? REGION : 0;
                                if (ifstack.Count == 0){
-                                       taking = true;
+                                       flags |= PARENT_TAKING;
                                } else {
                                        int state = (int) ifstack.Peek ();
-                                       if ((state & TAKING) != 0)
-                                               taking = true;
+                                       if ((state & TAKING) != 0) {
+                                               flags |= PARENT_TAKING;
+                                       }
                                }
 
-                               if (eval (arg) && taking){
-                                       int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;
-                                       if (region_directive)
-                                               push |= REGION;
-                                       ifstack.Push (push);
+                               if (caller_is_taking && eval (arg)) {
+                                       ifstack.Push (flags | TAKING);
                                        return true;
-                               } else {
-                                       int push = (taking ? PARENT_TAKING : 0);
-                                       if (region_directive)
-                                               push |= REGION;
-                                       ifstack.Push (push);
-                                       return false;
                                }
+                               ifstack.Push (flags);
+                               return false;
                                
                        case "endif":
                                if (ifstack == null || ifstack.Count == 0){
                                        Error_UnexpectedDirective ("no #if for this #endif");
                                        return true;
                                } else {
-                                       int pop = (int) ifstack.Pop ();
+                                       pop = (int) ifstack.Pop ();
                                        
-                                       if (region_directive && ((pop & REGION) == 0))
-                                               Report.Error (1027, Location, "Expected `#endif' directive");
-                                       else if (!region_directive && ((pop & REGION) != 0))
+                                       if ((pop & REGION) != 0)
                                                Report.Error (1038, Location, "#endregion directive expected");
                                        
-                                       if (!region_directive && arg.Length != 0) {
-                                               Report.Error (1025, Location, "Single-line comment or end-of-line expected");
+                                       if (arg.Length != 0) {
+                                               Error_EndLineExpected ();
                                        }
                                        
                                        if (ifstack.Count == 0)
                                                return true;
-                                       else {
-                                               int state = (int) ifstack.Peek ();
 
-                                               if ((state & TAKING) != 0)
-                                                       return true;
-                                               else
-                                                       return false;
-                                       }
+                                       int state = (int) ifstack.Peek ();
+                                       return (state & TAKING) != 0;
                                }
 
                        case "elif":
@@ -1705,7 +2122,7 @@ namespace Mono.CSharp
                                        Error_UnexpectedDirective ("no #if for this #elif");
                                        return true;
                                } else {
-                                       int state = (int) ifstack.Peek ();
+                                       int state = (int) ifstack.Pop ();
 
                                        if ((state & REGION) != 0) {
                                                Report.Error (1038, Location, "#endregion directive expected");
@@ -1717,15 +2134,18 @@ namespace Mono.CSharp
                                                return true;
                                        }
 
-                                       if ((state & (TAKEN_BEFORE | TAKING)) != 0)
+                                       if ((state & TAKING) != 0) {
+                                               ifstack.Push (0);
                                                return false;
+                                       }
 
                                        if (eval (arg) && ((state & PARENT_TAKING) != 0)){
-                                               state = (int) ifstack.Pop ();
-                                               ifstack.Push (state | TAKING | TAKEN_BEFORE);
+                                               ifstack.Push (state | TAKING);
                                                return true;
-                                       } else 
-                                               return false;
+                                       }
+
+                                       ifstack.Push (state);
+                                       return false;
                                }
 
                        case "else":
@@ -1747,21 +2167,40 @@ namespace Mono.CSharp
 
                                        ifstack.Pop ();
 
-                                       bool ret;
-                                       if ((state & TAKEN_BEFORE) == 0){
-                                               ret = ((state & PARENT_TAKING) != 0);
-                                       } else
-                                               ret = false;
-                                       
-                                       if (ret)
-                                               state |= TAKING;
-                                       else
-                                               state &= ~TAKING;
+                                       if (arg.Length != 0) {
+                                               Error_EndLineExpected ();
+                                               return true;
+                                       }
+
+                                       bool ret = false;
+                                       if ((state & PARENT_TAKING) != 0) {
+                                               ret = (state & TAKING) == 0;
                                        
+                                               if (ret)
+                                                       state |= TAKING;
+                                               else
+                                                       state &= ~TAKING;
+                                       }
+       
                                        ifstack.Push (state | ELSE_SEEN);
                                        
                                        return ret;
                                }
+                               case "define":
+                                       if (any_token_seen){
+                                               Error_TokensSeen ();
+                                               return caller_is_taking;
+                                       }
+                                       PreProcessDefinition (true, arg, caller_is_taking);
+                                       return caller_is_taking;
+
+                               case "undef":
+                                       if (any_token_seen){
+                                               Error_TokensSeen ();
+                                               return caller_is_taking;
+                                       }
+                                       PreProcessDefinition (false, arg, caller_is_taking);
+                                       return caller_is_taking;
                        }
 
                        //
@@ -1771,22 +2210,6 @@ namespace Mono.CSharp
                                return false;
                                        
                        switch (cmd){
-                       case "define":
-                               if (any_token_seen){
-                                       Error_TokensSeen ();
-                                       return true;
-                               }
-                               PreProcessDefinition (true, arg);
-                               return true;
-
-                       case "undef":
-                               if (any_token_seen){
-                                       Error_TokensSeen ();
-                                       return true;
-                               }
-                               PreProcessDefinition (false, arg);
-                               return true;
-
                        case "error":
                                Report.Error (1029, Location, "#error: '" + arg + "'");
                                return true;
@@ -1794,6 +2217,22 @@ namespace Mono.CSharp
                        case "warning":
                                Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
                                return true;
+
+                       case "pragma":
+                               if (RootContext.Version == LanguageVersion.ISO_1) {
+                                       Report.FeatureIsNotISO1 (Location, "#pragma");
+                                       return true;
+                               }
+
+                               PreProcessPragma (arg);
+                               return true;
+
+                       case "line":
+                               if (!PreProcessLine (arg))
+                                       Report.Error (
+                                               1576, Location,
+                                               "The line number specified for #line directive is missing or invalid");
+                               return caller_is_taking;
                        }
 
                        Report.Error (1024, Location, "Wrong preprocessor directive");
@@ -1850,30 +2289,23 @@ namespace Mono.CSharp
 
                        if (res == Token.PARTIAL) {
                                // Save current position and parse next token.
-                               int old = reader.Position;
-                               int old_putback = putback_char;
-                               int old_ref_line = ref_line;
-                               int old_col = col;
-
-                               putback_char = -1;
+                               PushPosition ();
 
                                int next_token = token ();
                                bool ok = (next_token == Token.CLASS) ||
                                        (next_token == Token.STRUCT) ||
-                                       (next_token == Token.INTERFACE) ||
-                                       (next_token == Token.ENUM); // "partial" is a keyword in 'partial enum', even though it's not valid
+                                       (next_token == Token.INTERFACE);
 
-                               reader.Position = old;
-                               ref_line = old_ref_line;
-                               col = old_col;
-                               putback_char = old_putback;
+                               PopPosition ();
 
                                if (ok)
                                        return res;
-                               else {
-                                       val = new LocatedToken (Location, "partial");
-                                       return Token.IDENTIFIER;
-                               }
+
+                               if (next_token < Token.LAST_KEYWORD)
+                                       Report.Error (267, Location, "The `partial' modifier can be used only immediately before keyword `class', `struct', or `interface'");
+
+                               val = new LocatedToken (Location, "partial");
+                               return Token.IDENTIFIER;
                        }
 
                        return res;
@@ -1889,6 +2321,7 @@ namespace Mono.CSharp
                        current_location = new Location (ref_line, Col);
 
                        while ((c = getChar ()) != -1) {
+                       loop:
                                if (is_identifier_part_character ((char) c)){
                                        if (pos == max_id_size){
                                                Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
@@ -1897,6 +2330,9 @@ namespace Mono.CSharp
                                        
                                        id_builder [pos++] = (char) c;
 //                                     putback_char = -1;
+                               } else if (c == '\\') {
+                                       c = escape (c);
+                                       goto loop;
                                } else {
 //                                     putback_char = c;
                                        putback (c);
@@ -1963,18 +2399,14 @@ namespace Mono.CSharp
 
                        // Whether we have seen comments on the current line
                        bool comments_seen = false;
-                       
                        val = null;
-                       // optimization: eliminate col and implement #directive semantic correctly.
                        for (;(c = getChar ()) != -1;) {
-                               if (c == ' ')
-                                       continue;
-                               
-                               if (c == '\t') {
+                               if (c == '\t'){
+                                       col = ((col + 8) / 8) * 8;
                                        continue;
                                }
                                
-                               if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0)
+                               if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == 0xa0)
                                        continue;
 
                                if (c == '\r') {
@@ -2005,8 +2437,8 @@ namespace Mono.CSharp
                                                        }
                                                }
                                                while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
-                                               if (d == '\n'){
-                                               }
+                                                       if (d == '\n'){
+                                                       }
                                                any_token_seen |= tokens_seen;
                                                tokens_seen = false;
                                                comments_seen = false;
@@ -2066,7 +2498,7 @@ namespace Mono.CSharp
                                }
 
                                
-                               if (is_identifier_start_character ((char)c)){
+                               if (c == '\\' || is_identifier_start_character ((char)c)){
                                        tokens_seen = true;
                                        return consume_identifier (c);
                                }
@@ -2102,48 +2534,44 @@ namespace Mono.CSharp
                                        return Token.DOT;
                                }
                                
-                               /* For now, ignore pre-processor commands */
-                               // FIXME: In C# the '#' is not limited to appear
-                               // on the first column.
                                if (c == '#') {
-                                       // return NONE if we're not processing directives (during token peeks)
-                                       if (!process_directives)
-                                               return Token.NONE;
-
-                                       bool cont = true;
                                        if (tokens_seen || comments_seen) {
-                                               error_details = "Preprocessor directives must appear as the first" +
-                                              " non-whitespace character on a line.";
+                                               Eror_WrongPreprocessorLocation ();
+                                               return Token.ERROR;
+                                       }
+                                       
+                                       if (handle_preprocessing_directive (true))
+                                               continue;
 
-                                               Report.Error (1040, Location, error_details);
+                                       bool directive_expected = false;
+                                       while ((c = getChar ()) != -1) {
+                                               if (col == 1) {
+                                                       directive_expected = true;
+                                               } else if (!directive_expected) {
+                                                       // TODO: Implement comment support for disabled code and uncomment this code
+//                                                     if (c == '#') {
+//                                                             Eror_WrongPreprocessorLocation ();
+//                                                             return Token.ERROR;
+//                                                     }
+                                                       continue;
+                                               }
 
-                                               return Token.ERROR;
-                                       }
-                                       
-                               start_again:
-                                       
-                                       cont = handle_preprocessing_directive (cont);
+                                               if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
+                                                       continue;
 
-                                       if (cont){
-                                               continue;
+                                               if (c == '#') {
+                                                       if (handle_preprocessing_directive (false))
+                                                               break;
+                                               }
+                                               directive_expected = false;
                                        }
 
-                                       bool skipping = false;
-                                       for (;(c = getChar ()) != -1;){
-                                               if (c == '\n'){
-                                                       skipping = false;
-                                               } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
-                                                       continue;
-                                               else if (c != '#')
-                                                       skipping = true;
-                                               if (c == '#' && !skipping)
-                                                       goto start_again;
+                                       if (c != -1) {
+                                               tokens_seen = false;
+                                               continue;
                                        }
-                                       any_token_seen |= tokens_seen;
-                                       tokens_seen = false;
-                                       if (c == -1)
-                                               Report.Error (1027, Location, "Expected `#endif' directive");
-                                       continue;
+
+                                       return Token.EOF;
                                }
                                
                                if (c == '"') 
@@ -2311,13 +2739,13 @@ namespace Mono.CSharp
                public void cleanup ()
                {
                        if (ifstack != null && ifstack.Count >= 1) {
+                               current_location = new Location (ref_line, Col);
                                int state = (int) ifstack.Pop ();
                                if ((state & REGION) != 0)
                                        Report.Error (1038, Location, "#endregion directive expected");
                                else 
                                        Report.Error (1027, Location, "Expected `#endif' directive");
                        }
-                               
                }
        }
 
@@ -2335,3 +2763,4 @@ namespace Mono.CSharp
                Error
        }
 }
+