2008-06-11 Martin Baulig <martin@ximian.com>
[mono.git] / mcs / mcs / cs-tokenizer.cs
index 4efc236f228990c17aafe269103672038071443a..8e107f2682e622ed90c76506c896f3f1b8160d58 100644 (file)
@@ -5,17 +5,12 @@
 // Author: Miguel de Icaza (miguel@gnu.org)
 //         Marek Safar (marek.safar@seznam.cz)
 //
-// Licensed under the terms of the GNU GPL
+// Dual licensed under the terms of the MIT X11 or GNU GPL
+//
+// Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
+// Copyright 2004-2008 Novell, Inc
 //
-// (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
-// (C) 2004 Novell, Inc
 //
-
-/*
- * TODO:
- *   Make sure we accept the proper Unicode ranges, per the spec.
- *   Report error 1032
-*/
 
 using System;
 using System.Text;
@@ -34,7 +29,8 @@ namespace Mono.CSharp
        {
                SeekableStreamReader reader;
                SourceFile ref_name;
-               SourceFile file_name;
+               CompilationUnit file_name;
+               bool hidden = false;
                int ref_line = 1;
                int line = 1;
                int col = 0;
@@ -43,10 +39,15 @@ namespace Mono.CSharp
                bool handle_get_set = false;
                bool handle_remove_add = false;
                bool handle_assembly = false;
+               bool handle_where = false;
+               bool handle_typeof = false;
+               bool lambda_arguments_parsing;
                Location current_location;
                Location current_comment_location = Location.Null;
-               ArrayList escapedIdentifiers = new ArrayList ();
-
+               ArrayList escaped_identifiers = new ArrayList ();
+               public int parsing_block;
+               internal int query_parsing;
+               
                //
                // XML documentation buffer. The save point is used to divide
                // comments on types and comments on members.
@@ -56,7 +57,7 @@ namespace Mono.CSharp
                //
                // See comment on XmlCommentState enumeration.
                //
-               XmlCommentState xmlDocState = XmlCommentState.Allowed;
+               XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 
                //
                // Whether tokens have been seen on this line
@@ -70,15 +71,16 @@ namespace Mono.CSharp
                //
                bool any_token_seen = false;
 
-               static Hashtable tokenValues;
+               static Hashtable token_values;
+               static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 
                private static Hashtable TokenValueName
                {
                        get {
-                               if (tokenValues == null)
-                                       tokenValues = GetTokenValueNameHash ();
+                               if (token_values == null)
+                                       token_values = GetTokenValueNameHash ();
 
-                               return tokenValues;
+                               return token_values;
                        }
                }
 
@@ -122,60 +124,54 @@ namespace Mono.CSharp
                }
 
                public bool PropertyParsing {
-                       get {
-                               return handle_get_set;
-                       }
-
-                       set {
-                               handle_get_set = value;
-                       }
+                       get { return handle_get_set; }
+                       set { handle_get_set = value; }
                 }
 
                public bool AssemblyTargetParsing {
-                       get {
-                               return handle_assembly;
-                       }
-
-                       set {
-                               handle_assembly = value;
-                       }
+                       get { return handle_assembly; }
+                       set { handle_assembly = value; }
                }
 
                public bool EventParsing {
-                       get {
-                               return handle_remove_add;
-                       }
+                       get { return handle_remove_add; }
+                       set { handle_remove_add = value; }
+               }
 
-                       set {
-                               handle_remove_add = value;
-                       }
+               public bool ConstraintsParsing {
+                       get { return handle_where; }
+                       set { handle_where = value; }
                }
 
+               public bool TypeOfParsing {
+                       get { return handle_typeof; }
+                       set { handle_typeof = value; }
+               }
+               
                public XmlCommentState doc_state {
-                       get { return xmlDocState; }
+                       get { return xml_doc_state; }
                        set {
                                if (value == XmlCommentState.Allowed) {
                                        check_incorrect_doc_comment ();
                                        reset_doc_comment ();
                                }
-                               xmlDocState = value;
+                               xml_doc_state = value;
                        }
                }
 
                public bool IsEscapedIdentifier (Location loc)
                {
-                       foreach (LocatedToken lt in escapedIdentifiers)
+                       foreach (LocatedToken lt in escaped_identifiers)
                                if (lt.Location.Equals (loc))
                                        return true;
                        return false;
                }
 
-               
                //
                // Class variables
                // 
                static CharArrayHashtable[] keywords;
-               static Hashtable keywordStrings = new Hashtable ();
+               static Hashtable keyword_strings;
                static NumberStyles styles;
                static NumberFormatInfo csharp_format_info;
                
@@ -191,7 +187,6 @@ namespace Mono.CSharp
                Hashtable defines;
 
                const int TAKING        = 1;
-               const int TAKEN_BEFORE  = 2;
                const int ELSE_SEEN     = 4;
                const int PARENT_TAKING = 8;
                const int REGION        = 16;           
@@ -235,8 +230,71 @@ namespace Mono.CSharp
                        }
                }
 
-               static void AddKeyword (string kw, int token) {
-                       keywordStrings.Add (kw, kw);
+               //
+               // This is used when the tokenizer needs to save
+               // the current position as it needs to do some parsing
+               // on its own to deamiguate a token in behalf of the
+               // parser.
+               //
+               Stack position_stack = new Stack (2);
+               class Position {
+                       public int position;
+                       public int line;
+                       public int ref_line;
+                       public int col;
+                       public bool hidden;
+                       public int putback_char;
+                       public int previous_col;
+                       public Stack ifstack;
+                       public int parsing_generic_less_than;
+                       public int current_token;
+
+                       public Position (Tokenizer t)
+                       {
+                               position = t.reader.Position;
+                               line = t.line;
+                               ref_line = t.ref_line;
+                               col = t.col;
+                               hidden = t.hidden;
+                               putback_char = t.putback_char;
+                               previous_col = t.previous_col;
+                               if (t.ifstack != null && t.ifstack.Count != 0)
+                                       ifstack = (Stack)t.ifstack.Clone ();
+                               parsing_generic_less_than = t.parsing_generic_less_than;
+                               current_token = t.current_token;
+                       }
+               }
+               
+               public void PushPosition ()
+               {
+                       position_stack.Push (new Position (this));
+               }
+
+               public void PopPosition ()
+               {
+                       Position p = (Position) position_stack.Pop ();
+
+                       reader.Position = p.position;
+                       ref_line = p.ref_line;
+                       line = p.line;
+                       col = p.col;
+                       hidden = p.hidden;
+                       putback_char = p.putback_char;
+                       previous_col = p.previous_col;
+                       ifstack = p.ifstack;
+                       parsing_generic_less_than = p.parsing_generic_less_than;
+                       current_token = p.current_token;
+               }
+
+               // Do not reset the position, ignore it.
+               public void DiscardPosition ()
+               {
+                       position_stack.Pop ();
+               }
+               
+               static void AddKeyword (string kw, int token)
+               {
+                       keyword_strings.Add (kw, kw);
                        if (keywords [kw.Length] == null) {
                                keywords [kw.Length] = new CharArrayHashtable (kw.Length);
                        }
@@ -245,6 +303,7 @@ namespace Mono.CSharp
 
                static void InitTokens ()
                {
+                       keyword_strings = new Hashtable ();
                        keywords = new CharArrayHashtable [64];
 
                        AddKeyword ("__arglist", Token.ARGLIST);
@@ -331,17 +390,37 @@ namespace Mono.CSharp
                        AddKeyword ("volatile", Token.VOLATILE);
                        AddKeyword ("while", Token.WHILE);
                        AddKeyword ("partial", Token.PARTIAL);
+                       AddKeyword ("where", Token.WHERE);
+
+                       // LINQ keywords
+                       AddKeyword ("from", Token.FROM);
+                       AddKeyword ("join", Token.JOIN);
+                       AddKeyword ("on", Token.ON);
+                       AddKeyword ("equals", Token.EQUALS);
+                       AddKeyword ("select", Token.SELECT);
+                       AddKeyword ("group", Token.GROUP);
+                       AddKeyword ("by", Token.BY);
+                       AddKeyword ("let", Token.LET);
+                       AddKeyword ("orderby", Token.ORDERBY);
+                       AddKeyword ("ascending", Token.ASCENDING);
+                       AddKeyword ("descending", Token.DESCENDING);
+                       AddKeyword ("into", Token.INTO);
                }
 
                //
                // Class initializer
                // 
                static Tokenizer ()
+               {
+                       Reset ();
+               }
+
+               public static void Reset ()
                {
                        InitTokens ();
                        csharp_format_info = NumberFormatInfo.InvariantInfo;
                        styles = NumberStyles.Float;
-                       
+
                        string_builder = new System.Text.StringBuilder ();
                }
 
@@ -361,15 +440,66 @@ namespace Mono.CSharp
                        
                        int res = (int) o;
 
-                       if (handle_get_set == false && (res == Token.GET || res == Token.SET))
+                       if (!handle_get_set && (res == Token.GET || res == Token.SET))
                                return -1;
-                       if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
+                       if (!handle_remove_add && (res == Token.REMOVE || res == Token.ADD))
                                return -1;
-                       if (handle_assembly == false && res == Token.ASSEMBLY)
+                       if (!handle_assembly && res == Token.ASSEMBLY)
                                return -1;
+                       
+                       //
+                       // A query expression is any expression that starts with `from identifier'
+                       // followed by any token except ; , =
+                       // 
+                       if (query_parsing == 0) {
+                               if (res == Token.FROM && !lambda_arguments_parsing) {
+                                       PushPosition ();
+                                       // HACK: to disable generics micro-parser, because PushPosition does not
+                                       // store identifiers array
+                                       parsing_generic_less_than = 1;
+                                       switch (xtoken ()) {
+                                               case Token.IDENTIFIER:
+                                               case Token.INT:
+                                               case Token.BOOL:
+                                               case Token.BYTE:
+                                               case Token.CHAR:
+                                               case Token.DECIMAL:
+                                               case Token.FLOAT:
+                                               case Token.LONG:
+                                               case Token.OBJECT:
+                                               case Token.STRING:
+                                               case Token.UINT:
+                                               case Token.ULONG:
+                                                       int next_token = xtoken ();
+                                                       if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
+                                                               goto default;
+
+                                                       ++query_parsing;
+                                                       if (RootContext.Version <= LanguageVersion.ISO_2)
+                                                               Report.FeatureIsNotAvailable (Location, "query expressions");
+                                                       break;
+                                               case Token.VOID:
+                                                       Expression.Error_VoidInvalidInTheContext (Location);
+                                                       break;
+                                               default:
+                                                       PopPosition ();
+                                                       // HACK: A token is not a keyword so we need to restore identifiers buffer
+                                                       // which has been overwritten before we grabbed the identifier
+                                                       id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
+                                                       return -1;
+                                       }
+                                       PopPosition ();
+                                       return res;
+                               }
 
-                       return res;
+                               if (res > Token.QUERY_FIRST_TOKEN && res < Token.QUERY_LAST_TOKEN)
+                                       return -1;
+                       }
+
+                       if (res == Token.WHERE && !handle_where && query_parsing == 0)
+                               return -1;
                        
+                       return res;
                }
 
                public Location Location {
@@ -386,7 +516,7 @@ namespace Mono.CSharp
                        defines [def] = true;
                }
                
-               public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)
+               public Tokenizer (SeekableStreamReader input, CompilationUnit file, ArrayList defs)
                {
                        this.ref_name = file;
                        this.file_name = file;
@@ -406,22 +536,47 @@ namespace Mono.CSharp
                        // FIXME: This could be `Location.Push' but we have to
                        // find out why the MS compiler allows this
                        //
-                       Mono.CSharp.Location.Push (file);
+                       Mono.CSharp.Location.Push (file, file);
                }
 
                static bool is_identifier_start_character (char c)
                {
-                       return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c);
+                       return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter (c);
                }
 
                static bool is_identifier_part_character (char c)
                {
-                       return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
+                       return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') ||
+                               Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
                }
 
                public static bool IsKeyword (string s)
                {
-                       return keywordStrings [s] != null;
+                       return keyword_strings [s] != null;
+               }
+
+               //
+               // Tests whether '(' is beggining of lambda parameters
+               //
+               bool IsLambdaOpenParens ()
+               {
+                       int ntoken;
+                       while ((ntoken = xtoken ()) != Token.EOF) {
+                               switch (ntoken) {
+                                       case Token.CLOSE_PARENS:
+                                               return xtoken () == Token.ARROW;
+
+                                       case Token.STAR:
+                                       case Token.SEMICOLON:
+                                       case Token.OPEN_BRACE:
+                                       case Token.OPEN_PARENS:
+                                       case Token.LITERAL_STRING:
+                                               return false;
+                               }
+                       }
+
+                       Error_TokenExpected (",' or `)");
+                       return false;
                }
 
                public static bool IsValidIdentifier (string s)
@@ -439,6 +594,122 @@ namespace Mono.CSharp
                        return true;
                }
 
+               bool parse_less_than ()
+               {
+               start:
+                       int the_token = token ();
+                       if (the_token == Token.OPEN_BRACKET) {
+                               do {
+                                       the_token = token ();
+                               } while (the_token != Token.CLOSE_BRACKET);
+                               the_token = token ();
+                       }
+                       switch (the_token) {
+                       case Token.IDENTIFIER:
+                       case Token.OBJECT:
+                       case Token.STRING:
+                       case Token.BOOL:
+                       case Token.DECIMAL:
+                       case Token.FLOAT:
+                       case Token.DOUBLE:
+                       case Token.SBYTE:
+                       case Token.BYTE:
+                       case Token.SHORT:
+                       case Token.USHORT:
+                       case Token.INT:
+                       case Token.UINT:
+                       case Token.LONG:
+                       case Token.ULONG:
+                       case Token.CHAR:
+                       case Token.VOID:
+                               break;
+
+                       default:
+                               return false;
+                       }
+               again:
+                       the_token = token ();
+
+                       if (the_token == Token.OP_GENERICS_GT)
+                               return true;
+                       else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
+                               goto start;
+                       else if (the_token == Token.INTERR || the_token == Token.STAR)
+                               goto again;
+                       else if (the_token == Token.OP_GENERICS_LT) {
+                               if (!parse_less_than ())
+                                       return false;
+                               goto again;
+                       } else if (the_token == Token.OPEN_BRACKET) {
+                       rank_specifiers:
+                               the_token = token ();
+                               if (the_token == Token.CLOSE_BRACKET)
+                                       goto again;
+                               else if (the_token == Token.COMMA)
+                                       goto rank_specifiers;
+                               return false;
+                       }
+
+                       return false;
+               }
+
+               public void PutbackNullable ()
+               {
+                       if (nullable_pos < 0)
+                               throw new Exception ();
+
+                       current_token = -1;
+                       val = null;
+                       reader.Position = nullable_pos;
+
+                       putback_char = '?';
+               }
+
+               public void PutbackCloseParens ()
+               {
+                       putback_char = ')';
+               }
+
+
+               int nullable_pos = -1;
+
+               public void CheckNullable (bool is_nullable)
+               {
+                       if (is_nullable)
+                               nullable_pos = reader.Position;
+                       else
+                               nullable_pos = -1;
+               }
+               
+               bool parse_generic_dimension (out int dimension)
+               {
+                       dimension = 1;
+
+               again:
+                       int the_token = token ();
+                       if (the_token == Token.OP_GENERICS_GT)
+                               return true;
+                       else if (the_token == Token.COMMA) {
+                               dimension++;
+                               goto again;
+                       }
+
+                       return false;
+               }
+               
+               public int peek_token ()
+               {
+                       int the_token;
+
+                       PushPosition ();
+                       the_token = token ();
+                       PopPosition ();
+                       
+                       return the_token;
+               }
+                                       
+               int parsing_generic_less_than = 0;
+               
                int is_punct (char c, ref bool doread)
                {
                        int d;
@@ -461,6 +732,19 @@ namespace Mono.CSharp
                        case ']':
                                return Token.CLOSE_BRACKET;
                        case '(':
+                               if (!lambda_arguments_parsing) {
+                                       lambda_arguments_parsing = true;
+                                       PushPosition ();
+                                       bool lambda_start = IsLambdaOpenParens ();
+                                       PopPosition ();
+                                       lambda_arguments_parsing = false;
+                                       if (lambda_start) {
+                                               if (RootContext.Version <= LanguageVersion.ISO_2)
+                                                       Report.FeatureIsNotAvailable (Location, "lambda expressions");
+                                               
+                                               return Token.OPEN_PARENS_LAMBDA;
+                                       }
+                               }
                                return Token.OPEN_PARENS;
                        case ')': {
                                if (deambiguate_close_parens == 0)
@@ -468,19 +752,11 @@ namespace Mono.CSharp
 
                                --deambiguate_close_parens;
 
-                               // Save current position and parse next token.
-                               int old = reader.Position;
-                               int old_ref_line = ref_line;
-                               int old_col = col;
-
-                               // disable preprocessing directives when peeking
-                               process_directives = false;
-                               int new_token = token ();
-                               process_directives = true;
-                               reader.Position = old;
-                               ref_line = old_ref_line;
-                               col = old_col;
-                               putback_char = -1;
+                               PushPosition ();
+
+                               int new_token = xtoken ();
+
+                               PopPosition ();
 
                                if (new_token == Token.OPEN_PARENS)
                                        return Token.CLOSE_PARENS_OPEN_PARENS;
@@ -501,10 +777,78 @@ namespace Mono.CSharp
                                val = Location;
                                return Token.TILDE;
                        case '?':
+                               d = peek_char ();
+                               if (d == '?') {
+                                       get_char ();
+                                       return Token.OP_COALESCING;
+                               }
                                return Token.INTERR;
                        }
+                       
+                       if (c == '<') {
+                               if (parsing_generic_less_than++ > 0)
+                                       return Token.OP_GENERICS_LT;
+
+                               if (handle_typeof) {
+                                       int dimension;
+                                       PushPosition ();
+                                       if (parse_generic_dimension (out dimension)) {
+                                               val = dimension;
+                                               DiscardPosition ();
+                                               return Token.GENERIC_DIMENSION;
+                                       }
+                                       PopPosition ();
+                               }
 
-                       d = peekChar ();
+                               // Save current position and parse next token.
+                               PushPosition ();
+                               bool is_generic_lt = parse_less_than ();
+                               PopPosition ();
+
+                               if (is_generic_lt) {
+                                       return Token.OP_GENERICS_LT;
+                               } else
+                                       parsing_generic_less_than = 0;
+
+                               d = peek_char ();
+                               if (d == '<'){
+                                       get_char ();
+                                       d = peek_char ();
+
+                                       if (d == '='){
+                                               doread = true;
+                                               return Token.OP_SHIFT_LEFT_ASSIGN;
+                                       }
+                                       return Token.OP_SHIFT_LEFT;
+                               } else if (d == '='){
+                                       doread = true;
+                                       return Token.OP_LE;
+                               }
+                               return Token.OP_LT;
+                       } else if (c == '>') {
+                               if (parsing_generic_less_than > 0) {
+                                       parsing_generic_less_than--;
+                                       return Token.OP_GENERICS_GT;
+                               }
+
+                               d = peek_char ();
+                               if (d == '>'){
+                                       get_char ();
+                                       d = peek_char ();
+
+                                       if (d == '='){
+                                               doread = true;
+                                               return Token.OP_SHIFT_RIGHT_ASSIGN;
+                                       }
+                                       return Token.OP_SHIFT_RIGHT;
+                               } else if (d == '='){
+                                       doread = true;
+                                       return Token.OP_GE;
+                               }
+                               return Token.OP_GT;
+                       }
+                       
+                       d = peek_char ();
                        if (c == '+'){
                                
                                if (d == '+') {
@@ -551,6 +895,12 @@ namespace Mono.CSharp
                                        doread = true;
                                        return Token.OP_EQ;
                                }
+                               if (d == '>'){
+                                       doread = true;
+                                       val = Location;
+                                       return Token.ARROW;
+                               }
+
                                return Token.ASSIGN;
                        }
 
@@ -610,39 +960,6 @@ namespace Mono.CSharp
                                return Token.CARRET;
                        }
 
-                       if (c == '<'){
-                               if (d == '<'){
-                                       getChar ();
-                                       d = peekChar ();
-
-                                       if (d == '='){
-                                               doread = true;
-                                               return Token.OP_SHIFT_LEFT_ASSIGN;
-                                       }
-                                       return Token.OP_SHIFT_LEFT;
-                               } else if (d == '='){
-                                       doread = true;
-                                       return Token.OP_LE;
-                               }
-                               return Token.OP_LT;
-                       }
-
-                       if (c == '>'){
-                               if (d == '>'){
-                                       getChar ();
-                                       d = peekChar ();
-
-                                       if (d == '='){
-                                               doread = true;
-                                               return Token.OP_SHIFT_RIGHT_ASSIGN;
-                                       }
-                                       return Token.OP_SHIFT_RIGHT;
-                               } else if (d == '='){
-                                       doread = true;
-                                       return Token.OP_GE;
-                               }
-                               return Token.OP_GT;
-                       }
                        if (c == ':'){
                                if (d == ':'){
                                        doread = true;
@@ -661,18 +978,15 @@ namespace Mono.CSharp
                {
                        putback (')');
 
-                       // When any binary operation is used we are sure it is not a cast
-                       if (expression is Binary)
+                       // When any binary operation, a conditional is used we are sure it is not a cast
+                       // maybe more.
+                       
+                       if (expression is Binary || expression is Conditional)
                                return;
 
                        deambiguate_close_parens++;
                }
 
-               void Error_NumericConstantTooLong ()
-               {
-                       Report.Error (1021, Location, "Numeric constant too long");                     
-               }
-               
                bool decimal_digits (int c)
                {
                        int d;
@@ -685,15 +999,15 @@ namespace Mono.CSharp
                        }
                        
                        //
-                       // We use peekChar2, because decimal_digits needs to do a 
+                       // We use peek_char2, because decimal_digits needs to do a 
                        // 2-character look-ahead (5.ToString for example).
                        //
-                       while ((d = peekChar2 ()) != -1){
+                       while ((d = peek_char2 ()) != -1){
                                if (d >= '0' && d <= '9'){
                                        if (number_pos == max_number_size)
                                                Error_NumericConstantTooLong ();
                                        number_builder [number_pos++] = (char) d;
-                                       getChar ();
+                                       get_char ();
                                        seen_digits = true;
                                } else
                                        break;
@@ -706,7 +1020,7 @@ namespace Mono.CSharp
                {
                        return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
                }
-                               
+
                static int real_type_suffix (int c)
                {
                        int t;
@@ -740,11 +1054,11 @@ namespace Mono.CSharp
                                                if (is_unsigned)
                                                        scanning = false;
                                                is_unsigned = true;
-                                               getChar ();
+                                               get_char ();
                                                break;
 
                                        case 'l':
-                                               if (!is_unsigned && (RootContext.WarningLevel >= 4)){
+                                               if (!is_unsigned){
                                                        //
                                                        // if we have not seen anything in between
                                                        // report this error
@@ -759,21 +1073,21 @@ namespace Mono.CSharp
                                                if (is_long)
                                                        scanning = false;
                                                is_long = true;
-                                               getChar ();
+                                               get_char ();
                                                break;
 
                                        case 'L': 
                                                if (is_long)
                                                        scanning = false;
                                                is_long = true;
-                                               getChar ();
+                                               get_char ();
                                                break;
                                                
                                        default:
                                                scanning = false;
                                                break;
                                        }
-                                       c = peekChar ();
+                                       c = peek_char ();
                                } while (scanning);
                        }
 
@@ -890,11 +1204,11 @@ namespace Mono.CSharp
                        int d;
                        ulong ul;
                        
-                       getChar ();
-                       while ((d = peekChar ()) != -1){
+                       get_char ();
+                       while ((d = peek_char ()) != -1){
                                if (is_hex (d)){
                                        number_builder [number_pos++] = (char) d;
-                                       getChar ();
+                                       get_char ();
                                } else
                                        break;
                        }
@@ -917,7 +1231,7 @@ namespace Mono.CSharp
                                return Token.LITERAL_INTEGER;
                        }
                        
-                       return integer_type_suffix (ul, peekChar ());
+                       return integer_type_suffix (ul, peek_char ());
                }
 
                //
@@ -932,13 +1246,13 @@ namespace Mono.CSharp
 
                        if (c >= '0' && c <= '9'){
                                if (c == '0'){
-                                       int peek = peekChar ();
+                                       int peek = peek_char ();
 
                                        if (peek == 'x' || peek == 'X')
                                                return handle_hex ();
                                }
                                decimal_digits (c);
-                               c = getChar ();
+                               c = get_char ();
                        }
 
                        //
@@ -948,7 +1262,7 @@ namespace Mono.CSharp
                        if (c == '.'){
                                if (decimal_digits ('.')){
                                        is_real = true;
-                                       c = getChar ();
+                                       c = get_char ();
                                } else {
                                        putback ('.');
                                        number_pos--;
@@ -961,7 +1275,7 @@ namespace Mono.CSharp
                                if (number_pos == max_number_size)
                                        Error_NumericConstantTooLong ();
                                number_builder [number_pos++] = 'e';
-                               c = getChar ();
+                               c = get_char ();
                                
                                if (c == '+'){
                                        if (number_pos == max_number_size)
@@ -980,7 +1294,7 @@ namespace Mono.CSharp
                                }
                                        
                                decimal_digits (c);
-                               c = getChar ();
+                               c = get_char ();
                        }
 
                        type = real_type_suffix (c);
@@ -1011,10 +1325,10 @@ namespace Mono.CSharp
                        int c;
                        int top = count != -1 ? count : 4;
                        
-                       getChar ();
+                       get_char ();
                        error = false;
                        for (i = 0; i < top; i++){
-                               c = getChar ();
+                               c = get_char ();
                                
                                if (c >= '0' && c <= '9')
                                        c = (int) c - (int) '0';
@@ -1029,7 +1343,7 @@ namespace Mono.CSharp
                                
                                total = (total * 16) + c;
                                if (count == -1){
-                                       int p = peekChar ();
+                                       int p = peek_char ();
                                        if (p == -1)
                                                break;
                                        if (!is_hex ((char)p))
@@ -1045,7 +1359,7 @@ namespace Mono.CSharp
                        int d;
                        int v;
 
-                       d = peekChar ();
+                       d = peek_char ();
                        if (c != '\\')
                                return c;
                        
@@ -1091,31 +1405,35 @@ namespace Mono.CSharp
                                Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
                                return d;
                        }
-                       getChar ();
+                       get_char ();
                        return v;
                }
 
-               int getChar ()
+               int get_char ()
                {
                        int x;
                        if (putback_char != -1) {
                                x = putback_char;
                                putback_char = -1;
-                       }
-                       else
+                       } else
                                x = reader.Read ();
                        if (x == '\n') {
-                               line++;
-                               ref_line++;
-                               previous_col = col;
-                               col = 0;
+                               advance_line ();
                        }
                        else
                                col++;
                        return x;
                }
 
-               int peekChar ()
+               void advance_line ()
+               {
+                       line++;
+                       ref_line++;
+                       previous_col = col;
+                       col = 0;
+               }
+
+               int peek_char ()
                {
                        if (putback_char != -1)
                                return putback_char;
@@ -1123,7 +1441,7 @@ namespace Mono.CSharp
                        return putback_char;
                }
 
-               int peekChar2 ()
+               int peek_char2 ()
                {
                        if (putback_char != -1)
                                return putback_char;
@@ -1152,7 +1470,7 @@ namespace Mono.CSharp
 
                public bool advance ()
                {
-                       return peekChar () != -1;
+                       return peek_char () != -1;
                }
 
                public Object Value {
@@ -1192,6 +1510,7 @@ namespace Mono.CSharp
                        case Token.TYPEOF:
                        case Token.UNCHECKED:
                        case Token.UNSAFE:
+                       case Token.DEFAULT:
 
                                //
                                // These can be part of a member access
@@ -1205,6 +1524,8 @@ namespace Mono.CSharp
                        case Token.DOUBLE:
                        case Token.FLOAT:
                        case Token.CHAR:
+                       case Token.BYTE:
+                       case Token.DECIMAL:                     
                                return true;
 
                        default:
@@ -1213,10 +1534,25 @@ namespace Mono.CSharp
                }
 
                public int token ()
-                {
+               {
                        current_token = xtoken ();
-                        return current_token;
-                }
+
+                       if (current_token != Token.DEFAULT)
+                               return current_token;
+
+                       PushPosition();
+                       int c = xtoken();
+                       if (c == -1)
+                               current_token = Token.ERROR;
+                       else if (c == Token.OPEN_PARENS)
+                               current_token = Token.DEFAULT_OPEN_PARENS;
+                       else if (c == Token.COLON)
+                               current_token = Token.DEFAULT_COLON;
+                       else
+                               PopPosition();
+                       
+                       return current_token;
+               }
 
                static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
                
@@ -1229,14 +1565,13 @@ namespace Mono.CSharp
                        static_cmd_arg.Length = 0;
 
                        // skip over white space
-                       while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
+                       while ((c = get_char ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
                                ;
-
-
+                               
                        while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
                                if (is_identifier_part_character ((char) c)){
                                        static_cmd_arg.Append ((char) c);
-                                       c = getChar ();
+                                       c = get_char ();
                                } else {
                                        putback (c);
                                        break;
@@ -1245,12 +1580,12 @@ namespace Mono.CSharp
 
                        cmd = static_cmd_arg.ToString ();
 
-                       if (c == '\n'){
+                       if (c == '\n' || c == '\r'){
                                return;
                        }
 
                        // skip over white space
-                       while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
+                       while ((c = get_char ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
                                ;
 
                        if (c == '\n'){
@@ -1265,11 +1600,11 @@ namespace Mono.CSharp
                        static_cmd_arg.Length = 0;
                        static_cmd_arg.Append ((char) c);
                        
-                       while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
+                       while ((c = get_char ()) != -1 && (c != '\n') && (c != '\r')){
                                static_cmd_arg.Append ((char) c);
                        }
 
-                       arg = static_cmd_arg.ToString ().Trim ();
+                       arg = static_cmd_arg.ToString ();
                }
 
                //
@@ -1283,15 +1618,14 @@ namespace Mono.CSharp
                        if (arg == "default"){
                                ref_line = line;
                                ref_name = file_name;
-                               Location.Push (ref_name);
+                               hidden = false;
+                               Location.Push (file_name, ref_name);
                                return true;
                        } else if (arg == "hidden"){
-                               //
-                               // We ignore #line hidden
-                               //
+                               hidden = true;
                                return true;
                        }
-
+                       
                        try {
                                int pos;
 
@@ -1303,11 +1637,12 @@ namespace Mono.CSharp
                                        
                                        string name = arg.Substring (pos). Trim (quotes);
                                        ref_name = Location.LookupFile (name);
-                                       file_name.HasLineDirective = true;
-                                       ref_name.HasLineDirective = true;
-                                       Location.Push (ref_name);
+                                       file_name.AddFile (ref_name);
+                                       hidden = false;
+                                       Location.Push (file_name, ref_name);
                                } else {
                                        ref_line = System.Int32.Parse (arg);
+                                       hidden = false;
                                }
                        } catch {
                                return false;
@@ -1319,16 +1654,15 @@ namespace Mono.CSharp
                //
                // Handles #define and #undef
                //
-               void PreProcessDefinition (bool is_define, string arg)
+               void PreProcessDefinition (bool is_define, string arg, bool caller_is_taking)
                {
                        if (arg.Length == 0 || arg == "true" || arg == "false"){
                                Report.Error (1001, Location, "Missing identifer to pre-processor directive");
                                return;
                        }
 
-                       char[] whitespace = { ' ', '\t' };
-                       if (arg.IndexOfAny (whitespace) != -1){
-                               Report.Error (1025, Location, "Single-line comment or end-of-line expected");
+                       if (arg.IndexOfAny (simple_whitespaces) != -1){
+                               Error_EndLineExpected ();
                                return;
                        }
 
@@ -1342,6 +1676,9 @@ namespace Mono.CSharp
                                }
                        }
 
+                       if (!caller_is_taking)
+                               return;
+
                        if (is_define){
                                if (defines == null)
                                        defines = new Hashtable ();
@@ -1354,6 +1691,126 @@ namespace Mono.CSharp
                        }
                }
 
+               byte read_hex (string arg, int pos, out bool error)
+               {
+                       error = false;
+
+                       int total;
+                       char c = arg [pos];
+
+                       if ((c >= '0') && (c <= '9'))
+                               total = (int) c - (int) '0';
+                       else if ((c >= 'A') && (c <= 'F'))
+                               total = (int) c - (int) 'A' + 10;
+                       else if ((c >= 'a') && (c <= 'f'))
+                               total = (int) c - (int) 'a' + 10;
+                       else {
+                               error = true;
+                               return 0;
+                       }
+
+                       total *= 16;
+                       c = arg [pos+1];
+
+                       if ((c >= '0') && (c <= '9'))
+                               total += (int) c - (int) '0';
+                       else if ((c >= 'A') && (c <= 'F'))
+                               total += (int) c - (int) 'A' + 10;
+                       else if ((c >= 'a') && (c <= 'f'))
+                               total += (int) c - (int) 'a' + 10;
+                       else {
+                               error = true;
+                               return 0;
+                       }
+
+                       return (byte) total;
+               }
+
+               /// <summary>
+               /// Handles #pragma checksum
+               /// </summary>
+               bool PreProcessPragmaChecksum (string arg)
+               {
+                       if ((arg [0] != ' ') && (arg [0] != '\t'))
+                               return false;
+
+                       arg = arg.Trim (simple_whitespaces);
+                       if ((arg.Length < 2) || (arg [0] != '"'))
+                               return false;
+
+                       StringBuilder file_sb = new StringBuilder ();
+
+                       int pos = 1;
+                       char ch;
+                       while ((ch = arg [pos++]) != '"') {
+                               if (pos >= arg.Length)
+                                       return false;
+
+                               if (ch == '\\') {
+                                       if (pos+1 >= arg.Length)
+                                               return false;
+                                       ch = arg [pos++];
+                               }
+
+                               file_sb.Append (ch);
+                       }
+
+                       if ((pos+2 >= arg.Length) || ((arg [pos] != ' ') && (arg [pos] != '\t')))
+                               return false;
+
+                       arg = arg.Substring (pos).Trim (simple_whitespaces);
+                       if ((arg.Length < 42) || (arg [0] != '"') || (arg [1] != '{') ||
+                           (arg [10] != '-') || (arg [15] != '-') || (arg [20] != '-') ||
+                           (arg [25] != '-') || (arg [38] != '}') || (arg [39] != '"'))
+                               return false;
+
+                       bool error;
+                       byte[] guid_bytes = new byte [16];
+
+                       for (int i = 0; i < 4; i++) {
+                               guid_bytes [i] = read_hex (arg, 2+2*i, out error);
+                               if (error)
+                                       return false;
+                       }
+                       for (int i = 0; i < 2; i++) {
+                               guid_bytes [i+4] = read_hex (arg, 11+2*i, out error);
+                               if (error)
+                                       return false;
+                               guid_bytes [i+6] = read_hex (arg, 16+2*i, out error);
+                               if (error)
+                                       return false;
+                               guid_bytes [i+8] = read_hex (arg, 21+2*i, out error);
+                               if (error)
+                                       return false;
+                       }
+
+                       for (int i = 0; i < 6; i++) {
+                               guid_bytes [i+10] = read_hex (arg, 26+2*i, out error);
+                               if (error)
+                                       return false;
+                       }
+
+                       arg = arg.Substring (40).Trim (simple_whitespaces);
+                       if ((arg.Length < 34) || (arg [0] != '"') || (arg [33] != '"'))
+                               return false;
+
+                       byte[] checksum_bytes = new byte [16];
+                       for (int i = 0; i < 16; i++) {
+                               checksum_bytes [i] = read_hex (arg, 1+2*i, out error);
+                               if (error)
+                                       return false;
+                       }
+
+                       arg = arg.Substring (34).Trim (simple_whitespaces);
+                       if (arg.Length > 0)
+                               return false;
+
+                       SourceFile file = Location.LookupFile (file_sb.ToString ());
+                       file.SetChecksum (guid_bytes, checksum_bytes);
+                       ref_name.AutoGenerated = true;
+                       return true;
+               }
+
                /// <summary>
                /// Handles #pragma directive
                /// </summary>
@@ -1362,14 +1819,15 @@ namespace Mono.CSharp
                        const string warning = "warning";
                        const string w_disable = "warning disable";
                        const string w_restore = "warning restore";
+                       const string checksum = "checksum";
 
                        if (arg == w_disable) {
-                               Report.RegisterWarningRegion (Location).WarningDisable (line);
+                               Report.RegisterWarningRegion (Location).WarningDisable (Location.Row);
                                return;
                        }
 
                        if (arg == w_restore) {
-                               Report.RegisterWarningRegion (Location).WarningEnable (line);
+                               Report.RegisterWarningRegion (Location).WarningEnable (Location.Row);
                                return;
                        }
 
@@ -1398,6 +1856,12 @@ namespace Mono.CSharp
                                return;
                        }
 
+                       if (arg.StartsWith (checksum)) {
+                               if (!PreProcessPragmaChecksum (arg.Substring (checksum.Length)))
+                                       Warning_InvalidPragmaChecksum ();
+                               return;
+                       }
+
                        Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
                }
 
@@ -1442,7 +1906,7 @@ namespace Mono.CSharp
                                
                                if (c == '('){
                                        s = s.Substring (1);
-                                       bool val = pp_expr (ref s);
+                                       bool val = pp_expr (ref s, false);
                                        if (s.Length > 0 && s [0] == ')'){
                                                s = s.Substring (1);
                                                return val;
@@ -1545,7 +2009,7 @@ namespace Mono.CSharp
                //
                // Evaluates an expression for `#if' or `#elif'
                //
-               bool pp_expr (ref string s)
+               bool pp_expr (ref string s, bool isTerm)
                {
                        bool va = pp_and (ref s);
                        s = s.Trim ();
@@ -1556,12 +2020,16 @@ namespace Mono.CSharp
                                if (c == '|'){
                                        if (len > 2 && s [1] == '|'){
                                                s = s.Substring (2);
-                                               return va | pp_expr (ref s);
+                                               return va | pp_expr (ref s, isTerm);
                                        } else {
                                                Error_InvalidDirective ();
                                                return false;
                                        }
-                               } 
+                               }
+                               if (isTerm) {
+                                       Error_EndLineExpected ();
+                                       return false;
+                               }
                        }
                        
                        return va;
@@ -1569,7 +2037,7 @@ namespace Mono.CSharp
 
                bool eval (string s)
                {
-                       bool v = pp_expr (ref s);
+                       bool v = pp_expr (ref s, true);
                        s = s.Trim ();
                        if (s.Length != 0){
                                return false;
@@ -1577,6 +2045,11 @@ namespace Mono.CSharp
 
                        return v;
                }
+
+               void Error_NumericConstantTooLong ()
+               {
+                       Report.Error (1021, Location, "Numeric constant too long");                     
+               }
                
                void Error_InvalidDirective ()
                {
@@ -1590,22 +2063,41 @@ namespace Mono.CSharp
                                "Unexpected processor directive (" + extra + ")");
                }
 
+               void Error_TokenExpected (string token)
+               {
+                       Report.Error (1026, Location, "Expected `{0}'", token);
+               }
+
                void Error_TokensSeen ()
                {
                        Report.Error (1032, Location,
                                "Cannot define or undefine preprocessor symbols after first token in file");
                }
-               
-               //
-               // Set to false to stop handling preprocesser directives
-               // 
-               bool process_directives = true;
 
+               void Eror_WrongPreprocessorLocation ()
+               {
+                       Report.Error (1040, Location,
+                               "Preprocessor directives must appear as the first non-whitespace character on a line");
+               }
+
+               void Error_EndLineExpected ()
+               {
+                       Report.Error (1025, Location, "Single-line comment or end-of-line expected");
+               }
+               
+               void Warning_InvalidPragmaChecksum ()
+               {
+                       Report.Warning (1695, 1, Location,
+                                       "Invalid #pragma checksum syntax; should be " +
+                                       "#pragma checksum \"filename\" " +
+                                       "\"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
+               }
                //
                // if true, then the code continues processing the code
                // if false, the code stays in a loop until another directive is
                // reached.
-               //
+               // When caller_is_taking is false we ignore all directives except the ones
+               // which can help us to identify where the #if block ends
                bool handle_preprocessing_directive (bool caller_is_taking)
                {
                        string cmd, arg;
@@ -1616,94 +2108,69 @@ namespace Mono.CSharp
                        // Eat any trailing whitespaces and single-line comments
                        if (arg.IndexOf ("//") != -1)
                                arg = arg.Substring (0, arg.IndexOf ("//"));
-                       arg = arg.TrimEnd (' ', '\t');
+                       arg = arg.Trim (simple_whitespaces);
 
                        //
                        // The first group of pre-processing instructions is always processed
                        //
                        switch (cmd){
-                       case "pragma":
-                               if (RootContext.Version == LanguageVersion.ISO_1) {
-                                       Report.FeatureIsNotStandardized (Location, "#pragma");
-                                       return caller_is_taking;
-                               }
-
-                               PreProcessPragma (arg);
-                               return caller_is_taking;
-
-                       case "line":
-                               if (!PreProcessLine (arg))
-                                       Report.Error (
-                                               1576, Location,
-                                               "The line number specified for #line directive is missing or invalid");
-                               return caller_is_taking;
-
                        case "region":
                                region_directive = true;
                                arg = "true";
                                goto case "if";
 
                        case "endregion":
-                               region_directive = true;
-                               goto case "endif";
-                               
-                       case "if":
-                               if (arg.Length == 0){
-                                       Error_InvalidDirective ();
+                               if (ifstack == null || ifstack.Count == 0){
+                                       Error_UnexpectedDirective ("no #region for this #endregion");
                                        return true;
                                }
-                               bool taking = false;
+                               int pop = (int) ifstack.Pop ();
+                                       
+                               if ((pop & REGION) == 0)
+                                       Report.Error (1027, Location, "Expected `#endif' directive");
+                                       
+                               return caller_is_taking;
+                               
+                       case "if":
                                if (ifstack == null)
                                        ifstack = new Stack (2);
 
+                               int flags = region_directive ? REGION : 0;
                                if (ifstack.Count == 0){
-                                       taking = true;
+                                       flags |= PARENT_TAKING;
                                } else {
                                        int state = (int) ifstack.Peek ();
-                                       if ((state & TAKING) != 0)
-                                               taking = true;
+                                       if ((state & TAKING) != 0) {
+                                               flags |= PARENT_TAKING;
+                                       }
                                }
 
-                               if (eval (arg) && taking){
-                                       int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;
-                                       if (region_directive)
-                                               push |= REGION;
-                                       ifstack.Push (push);
+                               if (caller_is_taking && eval (arg)) {
+                                       ifstack.Push (flags | TAKING);
                                        return true;
-                               } else {
-                                       int push = (taking ? PARENT_TAKING : 0);
-                                       if (region_directive)
-                                               push |= REGION;
-                                       ifstack.Push (push);
-                                       return false;
                                }
+                               ifstack.Push (flags);
+                               return false;
                                
                        case "endif":
                                if (ifstack == null || ifstack.Count == 0){
                                        Error_UnexpectedDirective ("no #if for this #endif");
                                        return true;
                                } else {
-                                       int pop = (int) ifstack.Pop ();
+                                       pop = (int) ifstack.Pop ();
                                        
-                                       if (region_directive && ((pop & REGION) == 0))
-                                               Report.Error (1027, Location, "Expected `#endif' directive");
-                                       else if (!region_directive && ((pop & REGION) != 0))
+                                       if ((pop & REGION) != 0)
                                                Report.Error (1038, Location, "#endregion directive expected");
                                        
-                                       if (!region_directive && arg.Length != 0) {
-                                               Report.Error (1025, Location, "Single-line comment or end-of-line expected");
+                                       if (arg.Length != 0) {
+                                               Error_EndLineExpected ();
                                        }
                                        
                                        if (ifstack.Count == 0)
                                                return true;
-                                       else {
-                                               int state = (int) ifstack.Peek ();
 
-                                               if ((state & TAKING) != 0)
-                                                       return true;
-                                               else
-                                                       return false;
-                                       }
+                                       int state = (int) ifstack.Peek ();
+                                       return (state & TAKING) != 0;
                                }
 
                        case "elif":
@@ -1711,7 +2178,7 @@ namespace Mono.CSharp
                                        Error_UnexpectedDirective ("no #if for this #elif");
                                        return true;
                                } else {
-                                       int state = (int) ifstack.Peek ();
+                                       int state = (int) ifstack.Pop ();
 
                                        if ((state & REGION) != 0) {
                                                Report.Error (1038, Location, "#endregion directive expected");
@@ -1723,15 +2190,18 @@ namespace Mono.CSharp
                                                return true;
                                        }
 
-                                       if ((state & (TAKEN_BEFORE | TAKING)) != 0)
+                                       if ((state & TAKING) != 0) {
+                                               ifstack.Push (0);
                                                return false;
+                                       }
 
                                        if (eval (arg) && ((state & PARENT_TAKING) != 0)){
-                                               state = (int) ifstack.Pop ();
-                                               ifstack.Push (state | TAKING | TAKEN_BEFORE);
+                                               ifstack.Push (state | TAKING);
                                                return true;
-                                       } else 
-                                               return false;
+                                       }
+
+                                       ifstack.Push (state);
+                                       return false;
                                }
 
                        case "else":
@@ -1753,21 +2223,40 @@ namespace Mono.CSharp
 
                                        ifstack.Pop ();
 
-                                       bool ret;
-                                       if ((state & TAKEN_BEFORE) == 0){
-                                               ret = ((state & PARENT_TAKING) != 0);
-                                       } else
-                                               ret = false;
-                                       
-                                       if (ret)
-                                               state |= TAKING;
-                                       else
-                                               state &= ~TAKING;
+                                       if (arg.Length != 0) {
+                                               Error_EndLineExpected ();
+                                               return true;
+                                       }
+
+                                       bool ret = false;
+                                       if ((state & PARENT_TAKING) != 0) {
+                                               ret = (state & TAKING) == 0;
                                        
+                                               if (ret)
+                                                       state |= TAKING;
+                                               else
+                                                       state &= ~TAKING;
+                                       }
+       
                                        ifstack.Push (state | ELSE_SEEN);
                                        
                                        return ret;
                                }
+                               case "define":
+                                       if (any_token_seen){
+                                               Error_TokensSeen ();
+                                               return caller_is_taking;
+                                       }
+                                       PreProcessDefinition (true, arg, caller_is_taking);
+                                       return caller_is_taking;
+
+                               case "undef":
+                                       if (any_token_seen){
+                                               Error_TokensSeen ();
+                                               return caller_is_taking;
+                                       }
+                                       PreProcessDefinition (false, arg, caller_is_taking);
+                                       return caller_is_taking;
                        }
 
                        //
@@ -1777,22 +2266,6 @@ namespace Mono.CSharp
                                return false;
                                        
                        switch (cmd){
-                       case "define":
-                               if (any_token_seen){
-                                       Error_TokensSeen ();
-                                       return true;
-                               }
-                               PreProcessDefinition (true, arg);
-                               return true;
-
-                       case "undef":
-                               if (any_token_seen){
-                                       Error_TokensSeen ();
-                                       return true;
-                               }
-                               PreProcessDefinition (false, arg);
-                               return true;
-
                        case "error":
                                Report.Error (1029, Location, "#error: '" + arg + "'");
                                return true;
@@ -1800,6 +2273,22 @@ namespace Mono.CSharp
                        case "warning":
                                Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
                                return true;
+
+                       case "pragma":
+                               if (RootContext.Version == LanguageVersion.ISO_1) {
+                                       Report.FeatureIsNotAvailable (Location, "#pragma");
+                                       return true;
+                               }
+
+                               PreProcessPragma (arg);
+                               return true;
+
+                       case "line":
+                               if (!PreProcessLine (arg))
+                                       Report.Error (
+                                               1576, Location,
+                                               "The line number specified for #line directive is missing or invalid");
+                               return caller_is_taking;
                        }
 
                        Report.Error (1024, Location, "Wrong preprocessor directive");
@@ -1811,12 +2300,18 @@ namespace Mono.CSharp
                {
                        int c;
                        string_builder.Length = 0;
-                                                               
-                       while ((c = getChar ()) != -1){
+
+                       //
+                       // No need to parse full string when parsing lambda arguments
+                       //
+                       if (lambda_arguments_parsing)
+                               return Token.LITERAL_STRING;                    
+                       
+                       while ((c = get_char ()) != -1){
                                if (c == '"'){
-                                       if (quoted && peekChar () == '"'){
+                                       if (quoted && peek_char () == '"'){
                                                string_builder.Append ((char) c);
-                                               getChar ();
+                                               get_char ();
                                                continue;
                                        } else {
                                                val = string_builder.ToString ();
@@ -1855,31 +2350,37 @@ namespace Mono.CSharp
                        }
 
                        if (res == Token.PARTIAL) {
-                               // Save current position and parse next token.
-                               int old = reader.Position;
-                               int old_putback = putback_char;
-                               int old_ref_line = ref_line;
-                               int old_col = col;
+                               if (parsing_block > 0) {
+                                       val = new LocatedToken (Location, "partial");
+                                       return Token.IDENTIFIER;
+                               }
 
-                               putback_char = -1;
+                               // Save current position and parse next token.
+                               PushPosition ();
 
                                int next_token = token ();
                                bool ok = (next_token == Token.CLASS) ||
                                        (next_token == Token.STRUCT) ||
                                        (next_token == Token.INTERFACE) ||
-                                       (next_token == Token.ENUM); // "partial" is a keyword in 'partial enum', even though it's not valid
+                                       (next_token == Token.VOID);
 
-                               reader.Position = old;
-                               ref_line = old_ref_line;
-                               col = old_col;
-                               putback_char = old_putback;
+                               PopPosition ();
+
+                               if (ok) {
+                                       if (next_token == Token.VOID) {
+                                               if (RootContext.Version <= LanguageVersion.ISO_2)
+                                                       Report.FeatureIsNotAvailable (Location, "partial methods");
+                                       } else if (RootContext.Version == LanguageVersion.ISO_1)
+                                               Report.FeatureIsNotAvailable (Location, "partial types");
 
-                               if (ok)
                                        return res;
-                               else {
-                                       val = new LocatedToken (Location, "partial");
-                                       return Token.IDENTIFIER;
                                }
+
+                               if (next_token < Token.LAST_KEYWORD)
+                                       Report.Error (267, Location, "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
+
+                               val = new LocatedToken (Location, "partial");
+                               return Token.IDENTIFIER;
                        }
 
                        return res;
@@ -1892,9 +2393,9 @@ namespace Mono.CSharp
                        
                        id_builder [0] = (char) s;
 
-                       current_location = new Location (ref_line, Col);
+                       current_location = new Location (ref_line, hidden ? -1 : Col);
 
-                       while ((c = getChar ()) != -1) {
+                       while ((c = get_char ()) != -1) {
                        loop:
                                if (is_identifier_part_character ((char) c)){
                                        if (pos == max_id_size){
@@ -1936,7 +2437,7 @@ namespace Mono.CSharp
                                if (val != null) {
                                        val = new LocatedToken (Location, (string) val);
                                        if (quoted)
-                                               escapedIdentifiers.Add (val);
+                                               escaped_identifiers.Add (val);
                                        return Token.IDENTIFIER;
                                }
                        }
@@ -1961,7 +2462,7 @@ namespace Mono.CSharp
 
                        val = new LocatedToken (Location, (string) val);
                        if (quoted)
-                               escapedIdentifiers.Add (val);
+                               escaped_identifiers.Add (val);
                        return Token.IDENTIFIER;
                }
                
@@ -1973,23 +2474,21 @@ namespace Mono.CSharp
 
                        // Whether we have seen comments on the current line
                        bool comments_seen = false;
-                       
                        val = null;
-                       // optimization: eliminate col and implement #directive semantic correctly.
-                       for (;(c = getChar ()) != -1;) {
-                               if (c == ' ')
-                                       continue;
-                               
-                               if (c == '\t') {
+                       for (;(c = get_char ()) != -1;) {
+                               if (c == '\t'){
+                                       col = ((col + 8) / 8) * 8;
                                        continue;
                                }
                                
-                               if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0)
+                               if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0 || c == 0)
                                        continue;
 
                                if (c == '\r') {
-                                       if (peekChar () == '\n')
-                                               getChar ();
+                                       if (peek_char () != '\n')
+                                               advance_line ();
+                                       else
+                                               get_char ();
 
                                        any_token_seen |= tokens_seen;
                                        tokens_seen = false;
@@ -1999,14 +2498,14 @@ namespace Mono.CSharp
 
                                // Handle double-slash comments.
                                if (c == '/'){
-                                       int d = peekChar ();
+                                       int d = peek_char ();
                                
                                        if (d == '/'){
-                                               getChar ();
-                                               if (RootContext.Documentation != null && peekChar () == '/') {
-                                                       getChar ();
+                                               get_char ();
+                                               if (RootContext.Documentation != null && peek_char () == '/') {
+                                                       get_char ();
                                                        // Don't allow ////.
-                                                       if ((d = peekChar ()) != '/') {
+                                                       if ((d = peek_char ()) != '/') {
                                                                update_comment_location ();
                                                                if (doc_state == XmlCommentState.Allowed)
                                                                        handle_one_line_xml_comment ();
@@ -2014,22 +2513,22 @@ namespace Mono.CSharp
                                                                        warn_incorrect_doc_comment ();
                                                        }
                                                }
-                                               while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
-                                               if (d == '\n'){
-                                               }
+                                               while ((d = get_char ()) != -1 && (d != '\n') && d != '\r')
+                                                       if (d == '\n'){
+                                                       }
                                                any_token_seen |= tokens_seen;
                                                tokens_seen = false;
                                                comments_seen = false;
                                                continue;
                                        } else if (d == '*'){
-                                               getChar ();
+                                               get_char ();
                                                bool docAppend = false;
-                                               if (RootContext.Documentation != null && peekChar () == '*') {
-                                                       getChar ();
+                                               if (RootContext.Documentation != null && peek_char () == '*') {
+                                                       get_char ();
                                                        update_comment_location ();
                                                        // But when it is /**/, just do nothing.
-                                                       if (peekChar () == '/') {
-                                                               getChar ();
+                                                       if (peek_char () == '/') {
+                                                               get_char ();
                                                                continue;
                                                        }
                                                        if (doc_state == XmlCommentState.Allowed)
@@ -2046,9 +2545,9 @@ namespace Mono.CSharp
 
                                                Location start_location = Location;
 
-                                               while ((d = getChar ()) != -1){
-                                                       if (d == '*' && peekChar () == '/'){
-                                                               getChar ();
+                                               while ((d = get_char ()) != -1){
+                                                       if (d == '*' && peek_char () == '/'){
+                                                               get_char ();
                                                                comments_seen = true;
                                                                break;
                                                        }
@@ -2082,11 +2581,11 @@ namespace Mono.CSharp
                                }
 
                        is_punct_label:
-                               current_location = new Location (ref_line, Col);
+                               current_location = new Location (ref_line, hidden ? -1 : Col);
                                if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
                                        tokens_seen = true;
                                        if (doread){
-                                               getChar ();
+                                               get_char ();
                                        }
                                        return t;
                                }
@@ -2106,61 +2605,57 @@ namespace Mono.CSharp
 
                                if (c == '.'){
                                        tokens_seen = true;
-                                       int peek = peekChar ();
+                                       int peek = peek_char ();
                                        if (peek >= '0' && peek <= '9')
                                                return is_number (c);
                                        return Token.DOT;
                                }
                                
-                               /* For now, ignore pre-processor commands */
-                               // FIXME: In C# the '#' is not limited to appear
-                               // on the first column.
                                if (c == '#') {
-                                       // return NONE if we're not processing directives (during token peeks)
-                                       if (!process_directives)
-                                               return Token.NONE;
-
-                                       bool cont = true;
                                        if (tokens_seen || comments_seen) {
-                                               error_details = "Preprocessor directives must appear as the first" +
-                                              " non-whitespace character on a line.";
+                                               Eror_WrongPreprocessorLocation ();
+                                               return Token.ERROR;
+                                       }
+                                       
+                                       if (handle_preprocessing_directive (true))
+                                               continue;
 
-                                               Report.Error (1040, Location, error_details);
+                                       bool directive_expected = false;
+                                       while ((c = get_char ()) != -1) {
+                                               if (col == 1) {
+                                                       directive_expected = true;
+                                               } else if (!directive_expected) {
+                                                       // TODO: Implement comment support for disabled code and uncomment this code
+//                                                     if (c == '#') {
+//                                                             Eror_WrongPreprocessorLocation ();
+//                                                             return Token.ERROR;
+//                                                     }
+                                                       continue;
+                                               }
 
-                                               return Token.ERROR;
-                                       }
-                                       
-                               start_again:
-                                       
-                                       cont = handle_preprocessing_directive (cont);
+                                               if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
+                                                       continue;
 
-                                       if (cont){
-                                               continue;
+                                               if (c == '#') {
+                                                       if (handle_preprocessing_directive (false))
+                                                               break;
+                                               }
+                                               directive_expected = false;
                                        }
 
-                                       bool skipping = false;
-                                       for (;(c = getChar ()) != -1;){
-                                               if (c == '\n'){
-                                                       skipping = false;
-                                               } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
-                                                       continue;
-                                               else if (c != '#')
-                                                       skipping = true;
-                                               if (c == '#' && !skipping)
-                                                       goto start_again;
+                                       if (c != -1) {
+                                               tokens_seen = false;
+                                               continue;
                                        }
-                                       any_token_seen |= tokens_seen;
-                                       tokens_seen = false;
-                                       if (c == -1)
-                                               Report.Error (1027, Location, "Expected `#endif' directive");
-                                       continue;
+
+                                       return Token.EOF;
                                }
                                
                                if (c == '"') 
                                        return consume_string (false);
 
                                if (c == '\''){
-                                       c = getChar ();
+                                       c = get_char ();
                                        tokens_seen = true;
                                        if (c == '\''){
                                                error_details = "Empty character literal";
@@ -2176,14 +2671,14 @@ namespace Mono.CSharp
                                                return Token.ERROR;
                                        val = new System.Char ();
                                        val = (char) c;
-                                       c = getChar ();
+                                       c = get_char ();
 
                                        if (c != '\''){
                                                error_details = "Too many characters in character literal";
                                                Report.Error (1012, Location, error_details);
 
                                                // Try to recover, read until newline or next "'"
-                                               while ((c = getChar ()) != -1){
+                                               while ((c = get_char ()) != -1){
                                                        if (c == '\n'){
                                                                break;
                                                        }
@@ -2196,7 +2691,7 @@ namespace Mono.CSharp
                                }
                                
                                if (c == '@') {
-                                       c = getChar ();
+                                       c = get_char ();
                                        if (c == '"') {
                                                tokens_seen = true;
                                                return consume_string (true);
@@ -2221,10 +2716,10 @@ namespace Mono.CSharp
                private void handle_one_line_xml_comment ()
                {
                        int c;
-                       while ((c = peekChar ()) == ' ')
-                               getChar (); // skip heading whitespaces.
-                       while ((c = peekChar ()) != -1 && c != '\n' && c != '\r') {
-                               xml_comment_buffer.Append ((char) getChar ());
+                       while ((c = peek_char ()) == ' ')
+                               get_char (); // skip heading whitespaces.
+                       while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
+                               xml_comment_buffer.Append ((char) get_char ());
                        }
                        if (c == '\r' || c == '\n')
                                xml_comment_buffer.Append (Environment.NewLine);
@@ -2269,7 +2764,7 @@ namespace Mono.CSharp
                        if (current_comment_location.IsNull) {
                                // "-2" is for heading "//" or "/*"
                                current_comment_location =
-                                       new Location (ref_line, col - 2);
+                                       new Location (ref_line, hidden ? -1 : col - 2);
                        }
                }
 
@@ -2321,11 +2816,12 @@ namespace Mono.CSharp
                public void cleanup ()
                {
                        if (ifstack != null && ifstack.Count >= 1) {
+                               current_location = new Location (ref_line, hidden ? -1 : Col);
                                int state = (int) ifstack.Pop ();
                                if ((state & REGION) != 0)
                                        Report.Error (1038, Location, "#endregion directive expected");
-//                             else 
-//                                     Report.Error (1027, Location, "Expected `#endif' directive");
+                               else 
+                                       Report.Error (1027, Location, "Expected `#endif' directive");
                        }
                }
        }
@@ -2344,3 +2840,4 @@ namespace Mono.CSharp
                Error
        }
 }
+