X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mcs%2Fmcs%2Fcs-tokenizer.cs;h=5d1c92c54159e13f22995161cdc23ca7d69ac297;hb=f448841be113e8e9e19c9dca1fb9348fb7db3ee1;hp=ca0a27d8855b8fe732b7cd824ea09fd92aae3c5b;hpb=b3956926e598791fce4b637261ad4576592d99fd;p=mono.git diff --git a/mcs/mcs/cs-tokenizer.cs b/mcs/mcs/cs-tokenizer.cs index ca0a27d8855..5d1c92c5415 100644 --- a/mcs/mcs/cs-tokenizer.cs +++ b/mcs/mcs/cs-tokenizer.cs @@ -5,16 +5,16 @@ // Author: Miguel de Icaza (miguel@gnu.org) // Marek Safar (marek.safar@seznam.cz) // -// Licensed under the terms of the GNU GPL +// Dual licensed under the terms of the MIT X11 or GNU GPL // -// (C) 2001, 2002 Ximian, Inc (http://www.ximian.com) -// (C) 2004 Novell, Inc +// Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com) +// Copyright 2004-2008 Novell, Inc // // using System; using System.Text; -using System.Collections; +using System.Collections.Generic; using System.IO; using System.Globalization; using System.Reflection; @@ -27,25 +27,180 @@ namespace Mono.CSharp public class Tokenizer : yyParser.yyInput { + class KeywordEntry + { + public readonly int Token; + public KeywordEntry Next; + public readonly char[] Value; + + public KeywordEntry (string value, int token) + { + this.Value = value.ToCharArray (); + this.Token = token; + } + } + + sealed class IdentifiersComparer : IEqualityComparer + { + readonly int length; + + public IdentifiersComparer (int length) + { + this.length = length; + } + + public bool Equals (char[] x, char[] y) + { + for (int i = 0; i < length; ++i) + if (x [i] != y [i]) + return false; + + return true; + } + + public int GetHashCode (char[] obj) + { + int h = 0; + for (int i = 0; i < length; ++i) + h = (h << 5) - h + obj [i]; + + return h; + } + } + + // + // This class has to be used in the parser only, it reuses token + // details after each parse + // + public class LocatedToken + { + int row, column; + string value; + + static LocatedToken[] buffer; + static int pos; + + private LocatedToken () + { + } + + public static LocatedToken Create (int row, int column) + { + return Create (null, row, column); + } + + public static LocatedToken Create (string value, int row, int column) + { + // + // TODO: I am not very happy about the logic but it's the best + // what I could come up with for now. + // Ideally we should be using just tiny buffer (256 elements) which + // is enough to hold all details for currect stack and recycle elements + // poped from the stack but there is a trick needed to recycle + // them properly. + // + LocatedToken entry; + if (pos >= buffer.Length) { + entry = new LocatedToken (); + } else { + entry = buffer [pos]; + if (entry == null) { + entry = new LocatedToken (); + buffer [pos] = entry; + } + + ++pos; + } + entry.value = value; + entry.row = row; + entry.column = column; + return entry; + } + + // + // Used for token not required by expression evaluator + // + public static LocatedToken CreateOptional (int row, int col) + { +#if false + return Create (row, col); +#endif + return null; + } + + public static void Initialize () + { + if (buffer == null) + buffer = new LocatedToken [10000]; + pos = 0; + } + + public Location Location { + get { return new Location (row, column); } + } + + public string Value { + get { return value; } + } + } + SeekableStreamReader reader; SourceFile ref_name; - SourceFile file_name; + CompilationUnit file_name; + CompilerContext context; + bool hidden = false; int ref_line = 1; int line = 1; int col = 0; int previous_col; int current_token; + int tab_size; bool handle_get_set = false; bool handle_remove_add = false; - bool handle_assembly = false; bool handle_where = false; bool handle_typeof = false; bool lambda_arguments_parsing; - Location current_location; Location current_comment_location = Location.Null; - ArrayList escaped_identifiers = new ArrayList (); + List escaped_identifiers; + int parsing_generic_less_than; + + // + // Used mainly for parser optimizations. Some expressions for instance + // can appear only in block (including initializer, base initializer) + // scope only + // public int parsing_block; - internal int query_parsing; + internal bool query_parsing; + + // + // When parsing type only, useful for ambiguous nullable types + // + public int parsing_type; + + // + // Set when parsing generic declaration (type or method header) + // + public bool parsing_generic_declaration; + + // + // The value indicates that we have not reach any declaration or + // namespace yet + // + public int parsing_declaration; + + // + // The special character to inject on streams to trigger the EXPRESSION_PARSE + // token to be returned. It just happens to be a Unicode character that + // would never be part of a program (can not be an identifier). + // + // This character is only tested just before the tokenizer is about to report + // an error; So on the regular operation mode, this addition will have no + // impact on the tokenizer's performance. + // + + public const int EvalStatementParserCharacter = 0x2190; // Unicode Left Arrow + public const int EvalCompilationUnitParserCharacter = 0x2191; // Unicode Arrow + public const int EvalUsingDeclarationsParserCharacter = 0x2192; // Unicode Arrow // // XML documentation buffer. The save point is used to divide @@ -63,6 +218,14 @@ namespace Mono.CSharp // bool tokens_seen = false; + // + // Set to true once the GENERATE_COMPLETION token has bee + // returned. This helps produce one GENERATE_COMPLETION, + // as many COMPLETE_COMPLETION as necessary to complete the + // AST tree and one final EOF. + // + bool generated; + // // Whether a token has been seen on the file // This is needed because `define' is not allowed to be used @@ -70,66 +233,11 @@ namespace Mono.CSharp // bool any_token_seen = false; - static Hashtable token_values; static readonly char[] simple_whitespaces = new char[] { ' ', '\t' }; - private static Hashtable TokenValueName - { - get { - if (token_values == null) - token_values = GetTokenValueNameHash (); - - return token_values; - } - } - - private static Hashtable GetTokenValueNameHash () - { - Type t = typeof (Token); - FieldInfo [] fields = t.GetFields (); - Hashtable hash = new Hashtable (); - foreach (FieldInfo field in fields) { - if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int)) - hash.Add (field.GetValue (null), field.Name); - } - return hash; - } - - // - // Returns a verbose representation of the current location - // - public string location { - get { - string det; - - if (current_token == Token.ERROR) - det = "detail: " + error_details; - else - det = ""; - - // return "Line: "+line+" Col: "+col + "\n" + - // "VirtLine: "+ref_line + - // " Token: "+current_token + " " + det; - string current_token_name = TokenValueName [current_token] as string; - if (current_token_name == null) - current_token_name = current_token.ToString (); - - return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name, - ref_line, - col, - current_token_name, - det); - } - } - public bool PropertyParsing { get { return handle_get_set; } set { handle_get_set = value; } - } - - public bool AssemblyTargetParsing { - get { return handle_assembly; } - set { handle_assembly = value; } } public bool EventParsing { @@ -146,6 +254,11 @@ namespace Mono.CSharp get { return handle_typeof; } set { handle_typeof = value; } } + + public int TabSize { + get { return tab_size; } + set { tab_size = value; } + } public XmlCommentState doc_state { get { return xml_doc_state; } @@ -158,33 +271,46 @@ namespace Mono.CSharp } } + // + // This is used to trigger completion generation on the parser + public bool CompleteOnEOF; + + void AddEscapedIdentifier (Location loc) + { + if (escaped_identifiers == null) + escaped_identifiers = new List (); + + escaped_identifiers.Add (loc); + } + public bool IsEscapedIdentifier (Location loc) { - foreach (LocatedToken lt in escaped_identifiers) - if (lt.Location.Equals (loc)) - return true; + if (escaped_identifiers != null) { + foreach (Location lt in escaped_identifiers) + if (lt.Equals (loc)) + return true; + } + return false; } // // Class variables // - static CharArrayHashtable[] keywords; - static Hashtable keyword_strings; + static KeywordEntry[][] keywords; + static Dictionary keyword_strings; // TODO: HashSet static NumberStyles styles; static NumberFormatInfo csharp_format_info; // // Values for the associated token returned // - int putback_char; - Object val; + internal int putback_char; // Used by repl only + object val; // // Pre-processor // - Hashtable defines; - const int TAKING = 1; const int ELSE_SEEN = 4; const int PARENT_TAKING = 8; @@ -193,18 +319,20 @@ namespace Mono.CSharp // // pre-processor if stack state: // - Stack ifstack; + Stack ifstack; static System.Text.StringBuilder string_builder; const int max_id_size = 512; static char [] id_builder = new char [max_id_size]; - static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1]; + public static Dictionary[] identifiers = new Dictionary[max_id_size + 1]; const int max_number_size = 512; static char [] number_builder = new char [max_number_size]; static int number_pos; + + static StringBuilder static_cmd_arg = new System.Text.StringBuilder (); // // Details about the error encoutered by the tokenizer @@ -223,40 +351,46 @@ namespace Mono.CSharp } } - public int Col { - get { - return col; - } - } - // // This is used when the tokenizer needs to save // the current position as it needs to do some parsing // on its own to deamiguate a token in behalf of the // parser. // - Stack position_stack = new Stack (2); + Stack position_stack = new Stack (2); + class Position { public int position; + public int line; public int ref_line; public int col; + public bool hidden; public int putback_char; public int previous_col; - public Stack ifstack; + public Stack ifstack; public int parsing_generic_less_than; public int current_token; + public object val; public Position (Tokenizer t) { position = t.reader.Position; + line = t.line; ref_line = t.ref_line; col = t.col; + hidden = t.hidden; putback_char = t.putback_char; previous_col = t.previous_col; - if (t.ifstack != null && t.ifstack.Count != 0) - ifstack = (Stack)t.ifstack.Clone (); + if (t.ifstack != null && t.ifstack.Count != 0) { + // There is no simple way to clone Stack all + // methods reverse the order + var clone = t.ifstack.ToArray (); + Array.Reverse (clone); + ifstack = new Stack (clone); + } parsing_generic_less_than = t.parsing_generic_less_than; current_token = t.current_token; + val = t.val; } } @@ -267,16 +401,19 @@ namespace Mono.CSharp public void PopPosition () { - Position p = (Position) position_stack.Pop (); + Position p = position_stack.Pop (); reader.Position = p.position; ref_line = p.ref_line; + line = p.line; col = p.col; + hidden = p.hidden; putback_char = p.putback_char; previous_col = p.previous_col; ifstack = p.ifstack; parsing_generic_less_than = p.parsing_generic_less_than; current_token = p.current_token; + val = p.val; } // Do not reset the position, ignore it. @@ -287,23 +424,38 @@ namespace Mono.CSharp static void AddKeyword (string kw, int token) { - keyword_strings.Add (kw, kw); - if (keywords [kw.Length] == null) { - keywords [kw.Length] = new CharArrayHashtable (kw.Length); + keyword_strings.Add (kw, null); + + int length = kw.Length; + if (keywords [length] == null) { + keywords [length] = new KeywordEntry ['z' - '_' + 1]; + } + + int char_index = kw [0] - '_'; + KeywordEntry kwe = keywords [length] [char_index]; + if (kwe == null) { + keywords [length] [char_index] = new KeywordEntry (kw, token); + return; + } + + while (kwe.Next != null) { + kwe = kwe.Next; } - keywords [kw.Length] [kw.ToCharArray ()] = token; + + kwe.Next = new KeywordEntry (kw, token); } static void InitTokens () { - keyword_strings = new Hashtable (); - keywords = new CharArrayHashtable [64]; + keyword_strings = new Dictionary (); + + // 11 is the length of the longest keyword for now + keywords = new KeywordEntry [11] []; AddKeyword ("__arglist", Token.ARGLIST); AddKeyword ("abstract", Token.ABSTRACT); AddKeyword ("as", Token.AS); AddKeyword ("add", Token.ADD); - AddKeyword ("assembly", Token.ASSEMBLY); AddKeyword ("base", Token.BASE); AddKeyword ("bool", Token.BOOL); AddKeyword ("break", Token.BREAK); @@ -405,12 +557,7 @@ namespace Mono.CSharp // static Tokenizer () { - Reset (); - } - - public static void Reset () - { - InitTokens (); + InitTokens (); csharp_format_info = NumberFormatInfo.InvariantInfo; styles = NumberStyles.Float; @@ -419,181 +566,410 @@ namespace Mono.CSharp int GetKeyword (char[] id, int id_len) { - /* - * Keywords are stored in an array of hashtables grouped by their - * length. - */ - - if ((id_len >= keywords.Length) || (keywords [id_len] == null)) + // + // Keywords are stored in an array of arrays grouped by their + // length and then by the first character + // + if (id_len >= keywords.Length || keywords [id_len] == null) return -1; - object o = keywords [id_len] [id]; - if (o == null) + int first_index = id [0] - '_'; + if (first_index > 'z') return -1; - - int res = (int) o; - if (!handle_get_set && (res == Token.GET || res == Token.SET)) + KeywordEntry kwe = keywords [id_len] [first_index]; + if (kwe == null) return -1; - if (!handle_remove_add && (res == Token.REMOVE || res == Token.ADD)) - return -1; - if (!handle_assembly && res == Token.ASSEMBLY) + + int res; + do { + res = kwe.Token; + for (int i = 1; i < id_len; ++i) { + if (id [i] != kwe.Value [i]) { + res = 0; + kwe = kwe.Next; + break; + } + } + } while (res == 0 && kwe != null); + + if (res == 0) return -1; - - // - // A query expression is any expression that starts with `from identifier' - // followed by any token except ; , = - // - if (query_parsing == 0) { - if (res == Token.FROM && !lambda_arguments_parsing) { + + int next_token; + switch (res) { + case Token.GET: + case Token.SET: + if (!handle_get_set) + res = -1; + break; + case Token.REMOVE: + case Token.ADD: + if (!handle_remove_add) + res = -1; + break; + case Token.EXTERN: + if (parsing_declaration == 0) + res = Token.EXTERN_ALIAS; + break; + case Token.DEFAULT: + if (peek_token () == Token.COLON) { + token (); + res = Token.DEFAULT_COLON; + } + break; + case Token.WHERE: + if (!handle_where && !query_parsing) + res = -1; + break; + case Token.FROM: + // + // A query expression is any expression that starts with `from identifier' + // followed by any token except ; , = + // + if (!query_parsing) { + if (lambda_arguments_parsing) { + res = -1; + break; + } + PushPosition (); // HACK: to disable generics micro-parser, because PushPosition does not // store identifiers array parsing_generic_less_than = 1; switch (xtoken ()) { - case Token.IDENTIFIER: - case Token.INT: - case Token.BOOL: - case Token.BYTE: - case Token.CHAR: - case Token.DECIMAL: - case Token.FLOAT: - case Token.LONG: - case Token.OBJECT: - case Token.STRING: - case Token.UINT: - case Token.ULONG: - int next_token = xtoken (); - if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS) - goto default; - - ++query_parsing; - if (RootContext.Version <= LanguageVersion.ISO_2) - Report.FeatureIsNotAvailable (Location, "query expressions"); - break; - case Token.VOID: - Expression.Error_VoidInvalidInTheContext (Location); - break; - default: - PopPosition (); - // HACK: A token is not a keyword so we need to restore identifiers buffer - // which has been overwritten before we grabbed the identifier - id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm'; - return -1; + case Token.IDENTIFIER: + case Token.INT: + case Token.BOOL: + case Token.BYTE: + case Token.CHAR: + case Token.DECIMAL: + case Token.FLOAT: + case Token.LONG: + case Token.OBJECT: + case Token.STRING: + case Token.UINT: + case Token.ULONG: + next_token = xtoken (); + if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS) + goto default; + + res = Token.FROM_FIRST; + query_parsing = true; + if (RootContext.Version <= LanguageVersion.ISO_2) + Report.FeatureIsNotAvailable (Location, "query expressions"); + break; + case Token.VOID: + Expression.Error_VoidInvalidInTheContext (Location, Report); + break; + default: + PopPosition (); + // HACK: A token is not a keyword so we need to restore identifiers buffer + // which has been overwritten before we grabbed the identifier + id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm'; + return -1; } PopPosition (); + } + break; + case Token.JOIN: + case Token.ON: + case Token.EQUALS: + case Token.SELECT: + case Token.GROUP: + case Token.BY: + case Token.LET: + case Token.ORDERBY: + case Token.ASCENDING: + case Token.DESCENDING: + case Token.INTO: + if (!query_parsing) + res = -1; + break; + + case Token.USING: + case Token.NAMESPACE: + // TODO: some explanation needed + check_incorrect_doc_comment (); + break; + + case Token.PARTIAL: + if (parsing_block > 0) { + res = -1; + break; + } + + // Save current position and parse next token. + PushPosition (); + + next_token = token (); + bool ok = (next_token == Token.CLASS) || + (next_token == Token.STRUCT) || + (next_token == Token.INTERFACE) || + (next_token == Token.VOID); + + PopPosition (); + + if (ok) { + if (next_token == Token.VOID) { + if (RootContext.Version == LanguageVersion.ISO_1 || + RootContext.Version == LanguageVersion.ISO_2) + Report.FeatureIsNotAvailable (Location, "partial methods"); + } else if (RootContext.Version == LanguageVersion.ISO_1) + Report.FeatureIsNotAvailable (Location, "partial types"); + return res; } - if (res > Token.QUERY_FIRST_TOKEN && res < Token.QUERY_LAST_TOKEN) - return -1; + if (next_token < Token.LAST_KEYWORD) { + Report.Error (267, Location, + "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword"); + return token (); + } + + res = -1; + break; } - if (res == Token.WHERE && !handle_where && query_parsing == 0) - return -1; - return res; } public Location Location { - get { return current_location; } - } - - void define (string def) - { - if (!RootContext.AllDefines.Contains (def)){ - RootContext.AllDefines [def] = true; + get { + return new Location (ref_line, hidden ? -1 : col); } - if (defines.Contains (def)) - return; - defines [def] = true; } - - public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs) + + public Tokenizer (SeekableStreamReader input, CompilationUnit file, CompilerContext ctx) { this.ref_name = file; this.file_name = file; + this.context = ctx; reader = input; putback_char = -1; - if (defs != null){ - defines = new Hashtable (); - foreach (string def in defs) - define (def); - } - xml_comment_buffer = new StringBuilder (); + if (Environment.OSVersion.Platform == PlatformID.Win32NT) + tab_size = 4; + else + tab_size = 8; + // // FIXME: This could be `Location.Push' but we have to // find out why the MS compiler allows this // - Mono.CSharp.Location.Push (file); + Mono.CSharp.Location.Push (file, file); } - static bool is_identifier_start_character (char c) + static bool is_identifier_start_character (int c) { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter (c); + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c); } static bool is_identifier_part_character (char c) { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || - Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation; + if (c >= 'a' && c <= 'z') + return true; + + if (c >= 'A' && c <= 'Z') + return true; + + if (c == '_' || (c >= '0' && c <= '9')) + return true; + + if (c < 0x80) + return false; + + return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation; } public static bool IsKeyword (string s) { - return keyword_strings [s] != null; + return keyword_strings.ContainsKey (s); } // - // Tests whether '(' is beggining of lambda parameters - // - bool IsLambdaOpenParens () + // Open parens micro parser. Detects both lambda and cast ambiguity. + // + int TokenizeOpenParens () { - int ntoken; - while ((ntoken = xtoken ()) != Token.EOF) { - switch (ntoken) { - case Token.CLOSE_PARENS: - return xtoken () == Token.ARROW; - - case Token.STAR: - case Token.SEMICOLON: - case Token.OPEN_PARENS: - case Token.LITERAL_STRING: - return false; - } - } + int ptoken; + current_token = -1; - Error_TokenExpected (",' or `)"); - return false; - } + int bracket_level = 0; + bool is_type = false; + bool can_be_type = false; + + while (true) { + ptoken = current_token; + token (); - public static bool IsValidIdentifier (string s) - { - if (s == null || s.Length == 0) - return false; + switch (current_token) { + case Token.CLOSE_PARENS: + token (); + + // + // Expression inside parens is lambda, (int i) => + // + if (current_token == Token.ARROW) + return Token.OPEN_PARENS_LAMBDA; - if (!is_identifier_start_character (s [0])) - return false; - - for (int i = 1; i < s.Length; i ++) - if (! is_identifier_part_character (s [i])) - return false; - - return true; - } + // + // Expression inside parens is single type, (int[]) + // + if (is_type) + return Token.OPEN_PARENS_CAST; + + // + // Expression is possible cast, look at next token, (T)null + // + if (can_be_type) { + switch (current_token) { + case Token.OPEN_PARENS: + case Token.BANG: + case Token.TILDE: + case Token.IDENTIFIER: + case Token.LITERAL: + case Token.BASE: + case Token.CHECKED: + case Token.DELEGATE: + case Token.FALSE: + case Token.FIXED: + case Token.NEW: + case Token.NULL: + case Token.SIZEOF: + case Token.THIS: + case Token.THROW: + case Token.TRUE: + case Token.TYPEOF: + case Token.UNCHECKED: + case Token.UNSAFE: + case Token.DEFAULT: - bool parse_less_than () - { - start: - int the_token = token (); - if (the_token == Token.OPEN_BRACKET) { - do { - the_token = token (); - } while (the_token != Token.CLOSE_BRACKET); + // + // These can be part of a member access + // + case Token.INT: + case Token.UINT: + case Token.SHORT: + case Token.USHORT: + case Token.LONG: + case Token.ULONG: + case Token.DOUBLE: + case Token.FLOAT: + case Token.CHAR: + case Token.BYTE: + case Token.DECIMAL: + case Token.BOOL: + return Token.OPEN_PARENS_CAST; + } + } + return Token.OPEN_PARENS; + + case Token.DOT: + case Token.DOUBLE_COLON: + if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT) + goto default; + + continue; + + case Token.IDENTIFIER: + switch (ptoken) { + case Token.DOT: + case Token.OP_GENERICS_LT: + case Token.COMMA: + case Token.DOUBLE_COLON: + case -1: + if (bracket_level == 0) + can_be_type = true; + continue; + default: + can_be_type = is_type = false; + continue; + } + + case Token.OBJECT: + case Token.STRING: + case Token.BOOL: + case Token.DECIMAL: + case Token.FLOAT: + case Token.DOUBLE: + case Token.SBYTE: + case Token.BYTE: + case Token.SHORT: + case Token.USHORT: + case Token.INT: + case Token.UINT: + case Token.LONG: + case Token.ULONG: + case Token.CHAR: + case Token.VOID: + if (bracket_level == 0) + is_type = true; + continue; + + case Token.COMMA: + if (bracket_level == 0) { + bracket_level = 100; + can_be_type = is_type = false; + } + continue; + + case Token.OP_GENERICS_LT: + case Token.OPEN_BRACKET: + if (bracket_level++ == 0) + is_type = true; + continue; + + case Token.OP_GENERICS_GT: + case Token.CLOSE_BRACKET: + --bracket_level; + continue; + + case Token.INTERR_NULLABLE: + case Token.STAR: + if (bracket_level == 0) + is_type = true; + continue; + + case Token.REF: + case Token.OUT: + can_be_type = is_type = false; + continue; + + default: + return Token.OPEN_PARENS; + } + } + } + + public static bool IsValidIdentifier (string s) + { + if (s == null || s.Length == 0) + return false; + + if (!is_identifier_start_character (s [0])) + return false; + + for (int i = 1; i < s.Length; i ++) + if (! is_identifier_part_character (s [i])) + return false; + + return true; + } + + bool parse_less_than () + { + start: + int the_token = token (); + if (the_token == Token.OPEN_BRACKET) { + do { + the_token = token (); + } while (the_token != Token.CLOSE_BRACKET); + the_token = token (); + } else if (the_token == Token.IN || the_token == Token.OUT) { the_token = token (); } switch (the_token) { @@ -615,6 +991,8 @@ namespace Mono.CSharp case Token.CHAR: case Token.VOID: break; + case Token.OP_GENERICS_GT: + return true; default: return false; @@ -626,7 +1004,7 @@ namespace Mono.CSharp return true; else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON) goto start; - else if (the_token == Token.INTERR || the_token == Token.STAR) + else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR) goto again; else if (the_token == Token.OP_GENERICS_LT) { if (!parse_less_than ()) @@ -645,34 +1023,6 @@ namespace Mono.CSharp return false; } - public void PutbackNullable () - { - if (nullable_pos < 0) - throw new Exception (); - - current_token = -1; - val = null; - reader.Position = nullable_pos; - - putback_char = '?'; - } - - public void PutbackCloseParens () - { - putback_char = ')'; - } - - - int nullable_pos = -1; - - public void CheckNullable (bool is_nullable) - { - if (is_nullable) - nullable_pos = reader.Position; - else - nullable_pos = -1; - } - bool parse_generic_dimension (out int dimension) { dimension = 1; @@ -700,281 +1050,111 @@ namespace Mono.CSharp return the_token; } - int parsing_generic_less_than = 0; - - int is_punct (char c, ref bool doread) + // + // Tonizes `?' using custom disambiguous rules to return one + // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR + // + // Tricky expression look like: + // + // Foo ? a = x ? b : c; + // + int TokenizePossibleNullableType () { - int d; - int t; - - doread = false; - - switch (c){ - case '{': - val = Location; - return Token.OPEN_BRACE; - case '}': - val = Location; - return Token.CLOSE_BRACE; - case '[': - // To block doccomment inside attribute declaration. - if (doc_state == XmlCommentState.Allowed) - doc_state = XmlCommentState.NotAllowed; - return Token.OPEN_BRACKET; - case ']': - return Token.CLOSE_BRACKET; - case '(': - if (!lambda_arguments_parsing) { - lambda_arguments_parsing = true; - PushPosition (); - bool lambda_start = IsLambdaOpenParens (); - PopPosition (); - lambda_arguments_parsing = false; - if (lambda_start) { - if (RootContext.Version <= LanguageVersion.ISO_2) - Report.FeatureIsNotAvailable (Location, "lambda expressions"); - - return Token.OPEN_PARENS_LAMBDA; - } - } - return Token.OPEN_PARENS; - case ')': { - if (deambiguate_close_parens == 0) - return Token.CLOSE_PARENS; - - --deambiguate_close_parens; - - PushPosition (); - - int new_token = xtoken (); - - PopPosition (); - - if (new_token == Token.OPEN_PARENS) - return Token.CLOSE_PARENS_OPEN_PARENS; - else if (new_token == Token.MINUS) - return Token.CLOSE_PARENS_MINUS; - else if (IsCastToken (new_token)) - return Token.CLOSE_PARENS_CAST; - else - return Token.CLOSE_PARENS_NO_CAST; - } - - case ',': - return Token.COMMA; - case ';': - val = Location; - return Token.SEMICOLON; - case '~': - val = Location; - return Token.TILDE; - case '?': - d = peek_char (); - if (d == '?') { - get_char (); - return Token.OP_COALESCING; - } - return Token.INTERR; - } - - if (c == '<') { - if (parsing_generic_less_than++ > 0) - return Token.OP_GENERICS_LT; - - if (handle_typeof) { - int dimension; - PushPosition (); - if (parse_generic_dimension (out dimension)) { - val = dimension; - DiscardPosition (); - return Token.GENERIC_DIMENSION; - } - PopPosition (); - } - - // Save current position and parse next token. - PushPosition (); - bool is_generic_lt = parse_less_than (); - PopPosition (); - - if (is_generic_lt) { - return Token.OP_GENERICS_LT; - } else - parsing_generic_less_than = 0; - - d = peek_char (); - if (d == '<'){ - get_char (); - d = peek_char (); - - if (d == '='){ - doread = true; - return Token.OP_SHIFT_LEFT_ASSIGN; - } - return Token.OP_SHIFT_LEFT; - } else if (d == '='){ - doread = true; - return Token.OP_LE; - } - return Token.OP_LT; - } else if (c == '>') { - if (parsing_generic_less_than > 0) { - parsing_generic_less_than--; - return Token.OP_GENERICS_GT; - } - - d = peek_char (); - if (d == '>'){ - get_char (); - d = peek_char (); - - if (d == '='){ - doread = true; - return Token.OP_SHIFT_RIGHT_ASSIGN; - } - return Token.OP_SHIFT_RIGHT; - } else if (d == '='){ - doread = true; - return Token.OP_GE; - } - return Token.OP_GT; - } - - d = peek_char (); - if (c == '+'){ - - if (d == '+') { - val = Location; - t = Token.OP_INC; - } - else if (d == '=') - t = Token.OP_ADD_ASSIGN; - else { - val = Location; - return Token.PLUS; - } - doread = true; - return t; - } - if (c == '-'){ - if (d == '-') { - val = Location; - t = Token.OP_DEC; - } - else if (d == '=') - t = Token.OP_SUB_ASSIGN; - else if (d == '>') - t = Token.OP_PTR; - else { - val = Location; - return Token.MINUS; - } - doread = true; - return t; - } - - if (c == '!'){ - if (d == '='){ - doread = true; - return Token.OP_NE; - } - val = Location; - return Token.BANG; - } - - if (c == '='){ - if (d == '='){ - doread = true; - return Token.OP_EQ; - } - if (d == '>'){ - doread = true; - val = Location; - return Token.ARROW; - } - - return Token.ASSIGN; - } - - if (c == '&'){ - if (d == '&'){ - doread = true; - return Token.OP_AND; - } else if (d == '='){ - doread = true; - return Token.OP_AND_ASSIGN; - } - val = Location; - return Token.BITWISE_AND; - } - - if (c == '|'){ - if (d == '|'){ - doread = true; - return Token.OP_OR; - } else if (d == '='){ - doread = true; - return Token.OP_OR_ASSIGN; - } - return Token.BITWISE_OR; - } + if (parsing_block == 0 || parsing_type > 0) + return Token.INTERR_NULLABLE; - if (c == '*'){ - if (d == '='){ - doread = true; - return Token.OP_MULT_ASSIGN; - } - val = Location; - return Token.STAR; + int d = peek_char (); + if (d == '?') { + get_char (); + return Token.OP_COALESCING; } - if (c == '/'){ - if (d == '='){ - doread = true; - return Token.OP_DIV_ASSIGN; - } - return Token.DIV; + switch (current_token) { + case Token.CLOSE_PARENS: + case Token.TRUE: + case Token.FALSE: + case Token.NULL: + case Token.LITERAL: + return Token.INTERR; } - if (c == '%'){ - if (d == '='){ - doread = true; - return Token.OP_MOD_ASSIGN; - } - return Token.PERCENT; + if (d != ' ') { + if (d == ',' || d == ';' || d == '>') + return Token.INTERR_NULLABLE; + if (d == '*' || (d >= '0' && d <= '9')) + return Token.INTERR; } - if (c == '^'){ - if (d == '='){ - doread = true; - return Token.OP_XOR_ASSIGN; - } - return Token.CARRET; + PushPosition (); + current_token = Token.NONE; + int next_token; + switch (xtoken ()) { + case Token.LITERAL: + case Token.TRUE: + case Token.FALSE: + case Token.NULL: + case Token.THIS: + case Token.NEW: + next_token = Token.INTERR; + break; + + case Token.SEMICOLON: + case Token.COMMA: + case Token.CLOSE_PARENS: + case Token.OPEN_BRACKET: + case Token.OP_GENERICS_GT: + next_token = Token.INTERR_NULLABLE; + break; + + default: + next_token = -1; + break; } - if (c == ':'){ - if (d == ':'){ - doread = true; - return Token.DOUBLE_COLON; + if (next_token == -1) { + switch (xtoken ()) { + case Token.COMMA: + case Token.SEMICOLON: + case Token.OPEN_BRACE: + case Token.CLOSE_PARENS: + case Token.IN: + next_token = Token.INTERR_NULLABLE; + break; + + case Token.COLON: + next_token = Token.INTERR; + break; + + default: + int ntoken; + int interrs = 1; + int colons = 0; + // + // All shorcuts failed, do it hard way + // + while ((ntoken = xtoken ()) != Token.EOF) { + if (ntoken == Token.SEMICOLON) + break; + + if (ntoken == Token.COLON) { + if (++colons == interrs) + break; + continue; + } + + if (ntoken == Token.INTERR) { + ++interrs; + continue; + } + } + + next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR; + break; } - val = Location; - return Token.COLON; } - - return Token.ERROR; - } - - int deambiguate_close_parens = 0; - - public void Deambiguate_CloseParens (object expression) - { - putback (')'); - - // When any binary operation is used we are sure it is not a cast - if (expression is Binary) - return; - - deambiguate_close_parens++; + + PopPosition (); + return next_token; } bool decimal_digits (int c) @@ -1010,25 +1190,19 @@ namespace Mono.CSharp { return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f'); } - - static int real_type_suffix (int c) - { - int t; + static TypeCode real_type_suffix (int c) + { switch (c){ case 'F': case 'f': - t = Token.LITERAL_FLOAT; - break; + return TypeCode.Single; case 'D': case 'd': - t = Token.LITERAL_DOUBLE; - break; + return TypeCode.Double; case 'M': case 'm': - t= Token.LITERAL_DECIMAL; - break; + return TypeCode.Decimal; default: - return Token.NONE; + return TypeCode.Empty; } - return t; } int integer_type_suffix (ulong ul, int c) @@ -1055,16 +1229,8 @@ namespace Mono.CSharp // Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)"); } - // - // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so - // work around that. - // - //goto case 'L'; - if (is_long) - scanning = false; - is_long = true; - get_char (); - break; + + goto case 'L'; case 'L': if (is_long) @@ -1082,38 +1248,40 @@ namespace Mono.CSharp } if (is_long && is_unsigned){ - val = ul; - return Token.LITERAL_INTEGER; - } else if (is_unsigned){ + val = new ULongLiteral (ul, Location); + return Token.LITERAL; + } + + if (is_unsigned){ // uint if possible, or ulong else. if ((ul & 0xffffffff00000000) == 0) - val = (uint) ul; + val = new UIntLiteral ((uint) ul, Location); else - val = ul; + val = new ULongLiteral (ul, Location); } else if (is_long){ // long if possible, ulong otherwise if ((ul & 0x8000000000000000) != 0) - val = ul; + val = new ULongLiteral (ul, Location); else - val = (long) ul; + val = new LongLiteral ((long) ul, Location); } else { // int, uint, long or ulong in that order if ((ul & 0xffffffff00000000) == 0){ uint ui = (uint) ul; if ((ui & 0x80000000) != 0) - val = ui; + val = new UIntLiteral (ui, Location); else - val = (int) ui; + val = new IntLiteral ((int) ui, Location); } else { if ((ul & 0x8000000000000000) != 0) - val = ul; + val = new ULongLiteral (ul, Location); else - val = (long) ul; + val = new LongLiteral ((long) ul, Location); } } - return Token.LITERAL_INTEGER; + return Token.LITERAL; } // @@ -1142,51 +1310,49 @@ namespace Mono.CSharp } catch (OverflowException) { error_details = "Integral constant is too large"; Report.Error (1021, Location, error_details); - val = 0ul; - return Token.LITERAL_INTEGER; + val = new IntLiteral (0, Location); + return Token.LITERAL; } catch (FormatException) { Report.Error (1013, Location, "Invalid number"); - val = 0ul; - return Token.LITERAL_INTEGER; + val = new IntLiteral (0, Location); + return Token.LITERAL; } } - int adjust_real (int t) + int adjust_real (TypeCode t) { string s = new String (number_builder, 0, number_pos); const string error_details = "Floating-point constant is outside the range of type `{0}'"; switch (t){ - case Token.LITERAL_DECIMAL: + case TypeCode.Decimal: try { - val = System.Decimal.Parse (s, styles, csharp_format_info); + val = new DecimalLiteral (decimal.Parse (s, styles, csharp_format_info), Location); } catch (OverflowException) { - val = 0m; + val = new DecimalLiteral (0, Location); Report.Error (594, Location, error_details, "decimal"); } break; - case Token.LITERAL_FLOAT: + case TypeCode.Single: try { - val = float.Parse (s, styles, csharp_format_info); + val = new FloatLiteral (float.Parse (s, styles, csharp_format_info), Location); } catch (OverflowException) { - val = 0.0f; + val = new FloatLiteral (0, Location); Report.Error (594, Location, error_details, "float"); } break; - - case Token.LITERAL_DOUBLE: - case Token.NONE: - t = Token.LITERAL_DOUBLE; + default: try { - val = System.Double.Parse (s, styles, csharp_format_info); + val = new DoubleLiteral (double.Parse (s, styles, csharp_format_info), Location); } catch (OverflowException) { - val = 0.0; + val = new DoubleLiteral (0, Location); Report.Error (594, Location, error_details, "double"); } break; } - return t; + + return Token.LITERAL; } int handle_hex () @@ -1212,13 +1378,13 @@ namespace Mono.CSharp } catch (OverflowException){ error_details = "Integral constant is too large"; Report.Error (1021, Location, error_details); - val = 0ul; - return Token.LITERAL_INTEGER; + val = new IntLiteral (0, Location); + return Token.LITERAL; } catch (FormatException) { Report.Error (1013, Location, "Invalid number"); - val = 0ul; - return Token.LITERAL_INTEGER; + val = new IntLiteral (0, Location); + return Token.LITERAL; } return integer_type_suffix (ul, peek_char ()); @@ -1230,7 +1396,6 @@ namespace Mono.CSharp int is_number (int c) { bool is_real = false; - int type; number_pos = 0; @@ -1287,28 +1452,28 @@ namespace Mono.CSharp c = get_char (); } - type = real_type_suffix (c); - if (type == Token.NONE && !is_real){ + var type = real_type_suffix (c); + if (type == TypeCode.Empty && !is_real){ putback (c); return adjust_int (c); - } else - is_real = true; + } + + is_real = true; - if (type == Token.NONE){ + if (type == TypeCode.Empty){ putback (c); } if (is_real) return adjust_real (type); - Console.WriteLine ("This should not be reached"); throw new Exception ("Is Number should never reach this point"); } // // Accepts exactly count (4 or 8) hex, no more no less // - int getHex (int count, out bool error) + int getHex (int count, out int surrogate, out bool error) { int i; int total = 0; @@ -1317,9 +1482,10 @@ namespace Mono.CSharp get_char (); error = false; + surrogate = 0; for (i = 0; i < top; i++){ c = get_char (); - + if (c >= '0' && c <= '9') c = (int) c - (int) '0'; else if (c >= 'A' && c <= 'F') @@ -1340,18 +1506,33 @@ namespace Mono.CSharp break; } } + + if (top == 8) { + if (total > 0x0010FFFF) { + error = true; + return 0; + } + + if (total >= 0x00010000) { + surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00); + total = ((total - 0x00010000) / 0x0400 + 0xD800); + } + } + return total; } - int escape (int c) + int escape (int c, out int surrogate) { bool error; int d; int v; d = peek_char (); - if (c != '\\') + if (c != '\\') { + surrogate = 0; return c; + } switch (d){ case 'a': @@ -1377,28 +1558,39 @@ namespace Mono.CSharp case '\'': v = '\''; break; case 'x': - v = getHex (-1, out error); + v = getHex (-1, out surrogate, out error); if (error) goto default; return v; case 'u': - v = getHex (4, out error); - if (error) - goto default; - return v; case 'U': - v = getHex (8, out error); - if (error) - goto default; - return v; + return EscapeUnicode (d, out surrogate); default: + surrogate = 0; Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ()); return d; } + get_char (); + surrogate = 0; return v; } + int EscapeUnicode (int ch, out int surrogate) + { + bool error; + if (ch == 'U') { + ch = getHex (8, out surrogate, out error); + } else { + ch = getHex (4, out surrogate, out error); + } + + if (error) + Report.Error (1009, Location, "Unrecognized escape sequence"); + + return ch; + } + int get_char () { int x; @@ -1409,9 +1601,9 @@ namespace Mono.CSharp x = reader.Read (); if (x == '\n') { advance_line (); - } - else + } else { col++; + } return x; } @@ -1425,9 +1617,8 @@ namespace Mono.CSharp int peek_char () { - if (putback_char != -1) - return putback_char; - putback_char = reader.Read (); + if (putback_char == -1) + putback_char = reader.Read (); return putback_char; } @@ -1460,7 +1651,7 @@ namespace Mono.CSharp public bool advance () { - return peek_char () != -1; + return peek_char () != -1 || CompleteOnEOF; } public Object Value { @@ -1474,125 +1665,76 @@ namespace Mono.CSharp return val; } - static bool IsCastToken (int token) - { - switch (token) { - case Token.BANG: - case Token.TILDE: - case Token.IDENTIFIER: - case Token.LITERAL_INTEGER: - case Token.LITERAL_FLOAT: - case Token.LITERAL_DOUBLE: - case Token.LITERAL_DECIMAL: - case Token.LITERAL_CHARACTER: - case Token.LITERAL_STRING: - case Token.BASE: - case Token.CHECKED: - case Token.DELEGATE: - case Token.FALSE: - case Token.FIXED: - case Token.NEW: - case Token.NULL: - case Token.SIZEOF: - case Token.THIS: - case Token.THROW: - case Token.TRUE: - case Token.TYPEOF: - case Token.UNCHECKED: - case Token.UNSAFE: - case Token.DEFAULT: - - // - // These can be part of a member access - // - case Token.INT: - case Token.UINT: - case Token.SHORT: - case Token.USHORT: - case Token.LONG: - case Token.ULONG: - case Token.DOUBLE: - case Token.FLOAT: - case Token.CHAR: - return true; - - default: - return false; - } - } - public int token () { current_token = xtoken (); - - if (current_token != Token.DEFAULT) - return current_token; - - PushPosition(); - int c = xtoken(); - if (c == -1) - current_token = Token.ERROR; - else if (c == Token.OPEN_PARENS) - current_token = Token.DEFAULT_OPEN_PARENS; - else if (c == Token.COLON) - current_token = Token.DEFAULT_COLON; - else - PopPosition(); - return current_token; } - static StringBuilder static_cmd_arg = new System.Text.StringBuilder (); - void get_cmd_arg (out string cmd, out string arg) { int c; tokens_seen = false; arg = ""; - static_cmd_arg.Length = 0; // skip over white space - while ((c = get_char ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t'))) - ; - - while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){ - if (is_identifier_part_character ((char) c)){ - static_cmd_arg.Append ((char) c); - c = get_char (); - } else { - putback (c); - break; + do { + c = get_char (); + } while (c == '\r' || c == ' ' || c == '\t'); + + static_cmd_arg.Length = 0; + while (c != -1 && is_identifier_part_character ((char)c)) { + static_cmd_arg.Append ((char)c); + c = get_char (); + if (c == '\\') { + int peek = peek_char (); + if (peek == 'U' || peek == 'u') { + int surrogate; + c = EscapeUnicode (c, out surrogate); + if (surrogate != 0) { + if (is_identifier_part_character ((char) c)) + static_cmd_arg.Append ((char) c); + c = surrogate; + } + } } } cmd = static_cmd_arg.ToString (); - if (c == '\n' || c == '\r'){ - return; - } - // skip over white space - while ((c = get_char ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t'))) - ; + while (c == '\r' || c == ' ' || c == '\t') + c = get_char (); - if (c == '\n'){ - return; - } else if (c == '\r'){ - return; - } else if (c == -1){ - arg = ""; - return; - } - static_cmd_arg.Length = 0; - static_cmd_arg.Append ((char) c); - - while ((c = get_char ()) != -1 && (c != '\n') && (c != '\r')){ + int has_identifier_argument = 0; + + while (c != -1 && c != '\n' && c != '\r') { + if (c == '\\' && has_identifier_argument >= 0) { + if (has_identifier_argument != 0 || (cmd == "define" || cmd == "if" || cmd == "elif" || cmd == "undef")) { + has_identifier_argument = 1; + + int peek = peek_char (); + if (peek == 'U' || peek == 'u') { + int surrogate; + c = EscapeUnicode (c, out surrogate); + if (surrogate != 0) { + if (is_identifier_part_character ((char) c)) + static_cmd_arg.Append ((char) c); + c = surrogate; + } + } + } else { + has_identifier_argument = -1; + } + } static_cmd_arg.Append ((char) c); + c = get_char (); } - arg = static_cmd_arg.ToString (); + if (static_cmd_arg.Length != 0) + arg = static_cmd_arg.ToString (); } // @@ -1606,12 +1748,11 @@ namespace Mono.CSharp if (arg == "default"){ ref_line = line; ref_name = file_name; - Location.Push (ref_name); + hidden = false; + Location.Push (file_name, ref_name); return true; } else if (arg == "hidden"){ - // - // We ignore #line hidden - // + hidden = true; return true; } @@ -1625,12 +1766,13 @@ namespace Mono.CSharp char [] quotes = { '\"' }; string name = arg.Substring (pos). Trim (quotes); - ref_name = Location.LookupFile (name); - file_name.HasLineDirective = true; - ref_name.HasLineDirective = true; - Location.Push (ref_name); + ref_name = Location.LookupFile (file_name, name); + file_name.AddFile (ref_name); + hidden = false; + Location.Push (file_name, ref_name); } else { ref_line = System.Int32.Parse (arg); + hidden = false; } } catch { return false; @@ -1642,24 +1784,24 @@ namespace Mono.CSharp // // Handles #define and #undef // - void PreProcessDefinition (bool is_define, string arg, bool caller_is_taking) + void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking) { - if (arg.Length == 0 || arg == "true" || arg == "false"){ - Report.Error (1001, Location, "Missing identifer to pre-processor directive"); + if (ident.Length == 0 || ident == "true" || ident == "false"){ + Report.Error (1001, Location, "Missing identifier to pre-processor directive"); return; } - if (arg.IndexOfAny (simple_whitespaces) != -1){ + if (ident.IndexOfAny (simple_whitespaces) != -1){ Error_EndLineExpected (); return; } - if (!is_identifier_start_character (arg [0])) - Report.Error (1001, Location, "Identifier expected: " + arg); + if (!is_identifier_start_character (ident [0])) + Report.Error (1001, Location, "Identifier expected: {0}", ident); - foreach (char c in arg.Substring (1)){ + foreach (char c in ident.Substring (1)){ if (!is_identifier_part_character (c)){ - Report.Error (1001, Location, "Identifier expected: " + arg); + Report.Error (1001, Location, "Identifier expected: {0}", ident); return; } } @@ -1667,16 +1809,140 @@ namespace Mono.CSharp if (!caller_is_taking) return; - if (is_define){ - if (defines == null) - defines = new Hashtable (); - define (arg); - } else { - if (defines == null) + if (is_define) { + // + // #define ident + // + if (RootContext.IsConditionalDefined (ident)) return; - if (defines.Contains (arg)) - defines.Remove (arg); + + file_name.AddDefine (ident); + } else { + // + // #undef ident + // + file_name.AddUndefine (ident); + } + } + + static byte read_hex (string arg, int pos, out bool error) + { + error = false; + + int total; + char c = arg [pos]; + + if ((c >= '0') && (c <= '9')) + total = (int) c - (int) '0'; + else if ((c >= 'A') && (c <= 'F')) + total = (int) c - (int) 'A' + 10; + else if ((c >= 'a') && (c <= 'f')) + total = (int) c - (int) 'a' + 10; + else { + error = true; + return 0; } + + total *= 16; + c = arg [pos+1]; + + if ((c >= '0') && (c <= '9')) + total += (int) c - (int) '0'; + else if ((c >= 'A') && (c <= 'F')) + total += (int) c - (int) 'A' + 10; + else if ((c >= 'a') && (c <= 'f')) + total += (int) c - (int) 'a' + 10; + else { + error = true; + return 0; + } + + return (byte) total; + } + + /// + /// Handles #pragma checksum + /// + bool PreProcessPragmaChecksum (string arg) + { + if ((arg [0] != ' ') && (arg [0] != '\t')) + return false; + + arg = arg.Trim (simple_whitespaces); + if ((arg.Length < 2) || (arg [0] != '"')) + return false; + + StringBuilder file_sb = new StringBuilder (); + + int pos = 1; + char ch; + while ((ch = arg [pos++]) != '"') { + if (pos >= arg.Length) + return false; + + if (ch == '\\') { + if (pos+1 >= arg.Length) + return false; + ch = arg [pos++]; + } + + file_sb.Append (ch); + } + + if ((pos+2 >= arg.Length) || ((arg [pos] != ' ') && (arg [pos] != '\t'))) + return false; + + arg = arg.Substring (pos).Trim (simple_whitespaces); + if ((arg.Length < 42) || (arg [0] != '"') || (arg [1] != '{') || + (arg [10] != '-') || (arg [15] != '-') || (arg [20] != '-') || + (arg [25] != '-') || (arg [38] != '}') || (arg [39] != '"')) + return false; + + bool error; + byte[] guid_bytes = new byte [16]; + + for (int i = 0; i < 4; i++) { + guid_bytes [i] = read_hex (arg, 2+2*i, out error); + if (error) + return false; + } + for (int i = 0; i < 2; i++) { + guid_bytes [i+4] = read_hex (arg, 11+2*i, out error); + if (error) + return false; + guid_bytes [i+6] = read_hex (arg, 16+2*i, out error); + if (error) + return false; + guid_bytes [i+8] = read_hex (arg, 21+2*i, out error); + if (error) + return false; + } + + for (int i = 0; i < 6; i++) { + guid_bytes [i+10] = read_hex (arg, 26+2*i, out error); + if (error) + return false; + } + + arg = arg.Substring (40).Trim (simple_whitespaces); + if ((arg.Length < 34) || (arg [0] != '"') || (arg [33] != '"')) + return false; + + byte[] checksum_bytes = new byte [16]; + for (int i = 0; i < 16; i++) { + checksum_bytes [i] = read_hex (arg, 1+2*i, out error); + if (error) + return false; + } + + arg = arg.Substring (34).Trim (simple_whitespaces); + if (arg.Length > 0) + return false; + + SourceFile file = Location.LookupFile (file_name, file_sb.ToString ()); + file.SetChecksum (guid_bytes, checksum_bytes); + ref_name.AutoGenerated = true; + return true; } /// @@ -1687,6 +1953,7 @@ namespace Mono.CSharp const string warning = "warning"; const string w_disable = "warning disable"; const string w_restore = "warning restore"; + const string checksum = "checksum"; if (arg == w_disable) { Report.RegisterWarningRegion (Location).WarningDisable (Location.Row); @@ -1702,18 +1969,18 @@ namespace Mono.CSharp int[] codes = ParseNumbers (arg.Substring (w_disable.Length)); foreach (int code in codes) { if (code != 0) - Report.RegisterWarningRegion (Location).WarningDisable (Location, code); + Report.RegisterWarningRegion (Location).WarningDisable (Location, code, Report); } return; } if (arg.StartsWith (w_restore)) { int[] codes = ParseNumbers (arg.Substring (w_restore.Length)); - Hashtable w_table = Report.warning_ignore_table; + var w_table = Report.warning_ignore_table; foreach (int code in codes) { - if (w_table != null && w_table.Contains (code)) - Report.Warning (1635, 1, Location, String.Format ("Cannot restore warning `CS{0:0000}' because it was disabled globally", code)); - Report.RegisterWarningRegion (Location).WarningEnable (Location, code); + if (w_table != null && w_table.ContainsKey (code)) + Report.Warning (1635, 1, Location, "Cannot restore warning `CS{0:0000}' because it was disabled globally", code); + Report.RegisterWarningRegion (Location).WarningEnable (Location, code, Report); } return; } @@ -1723,6 +1990,12 @@ namespace Mono.CSharp return; } + if (arg.StartsWith (checksum)) { + if (!PreProcessPragmaChecksum (arg.Substring (checksum.Length))) + Warning_InvalidPragmaChecksum (); + return; + } + Report.Warning (1633, 1, Location, "Unrecognized #pragma directive"); } @@ -1748,13 +2021,8 @@ namespace Mono.CSharp return true; if (s == "false") return false; - - if (defines == null) - return false; - if (defines.Contains (s)) - return true; - return false; + return file_name.IsConditionalDefined (s); } bool pp_primary (ref string s) @@ -1921,12 +2189,7 @@ namespace Mono.CSharp { Report.Error ( 1028, Location, - "Unexpected processor directive (" + extra + ")"); - } - - void Error_TokenExpected (string token) - { - Report.Error (1026, Location, "Expected `{0}'", token); + "Unexpected processor directive ({0})", extra); } void Error_TokensSeen () @@ -1946,6 +2209,13 @@ namespace Mono.CSharp Report.Error (1025, Location, "Single-line comment or end-of-line expected"); } + void Warning_InvalidPragmaChecksum () + { + Report.Warning (1695, 1, Location, + "Invalid #pragma checksum syntax; should be " + + "#pragma checksum \"filename\" " + + "\"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\""); + } // // if true, then the code continues processing the code // if false, the code stays in a loop until another directive is @@ -1978,7 +2248,7 @@ namespace Mono.CSharp Error_UnexpectedDirective ("no #region for this #endregion"); return true; } - int pop = (int) ifstack.Pop (); + int pop = ifstack.Pop (); if ((pop & REGION) == 0) Report.Error (1027, Location, "Expected `#endif' directive"); @@ -1987,13 +2257,13 @@ namespace Mono.CSharp case "if": if (ifstack == null) - ifstack = new Stack (2); + ifstack = new Stack (2); int flags = region_directive ? REGION : 0; if (ifstack.Count == 0){ flags |= PARENT_TAKING; } else { - int state = (int) ifstack.Peek (); + int state = ifstack.Peek (); if ((state & TAKING) != 0) { flags |= PARENT_TAKING; } @@ -2011,7 +2281,7 @@ namespace Mono.CSharp Error_UnexpectedDirective ("no #if for this #endif"); return true; } else { - pop = (int) ifstack.Pop (); + pop = ifstack.Pop (); if ((pop & REGION) != 0) Report.Error (1038, Location, "#endregion directive expected"); @@ -2023,7 +2293,7 @@ namespace Mono.CSharp if (ifstack.Count == 0) return true; - int state = (int) ifstack.Peek (); + int state = ifstack.Peek (); return (state & TAKING) != 0; } @@ -2032,7 +2302,7 @@ namespace Mono.CSharp Error_UnexpectedDirective ("no #if for this #elif"); return true; } else { - int state = (int) ifstack.Pop (); + int state = ifstack.Pop (); if ((state & REGION) != 0) { Report.Error (1038, Location, "#endregion directive expected"); @@ -2063,7 +2333,7 @@ namespace Mono.CSharp Error_UnexpectedDirective ("no #if for this #else"); return true; } else { - int state = (int) ifstack.Peek (); + int state = ifstack.Peek (); if ((state & REGION) != 0) { Report.Error (1038, Location, "#endregion directive expected"); @@ -2096,21 +2366,21 @@ namespace Mono.CSharp return ret; } - case "define": - if (any_token_seen){ - Error_TokensSeen (); - return caller_is_taking; - } - PreProcessDefinition (true, arg, caller_is_taking); + case "define": + if (any_token_seen){ + Error_TokensSeen (); return caller_is_taking; + } + PreProcessDefinition (true, arg, caller_is_taking); + return caller_is_taking; - case "undef": - if (any_token_seen){ - Error_TokensSeen (); - return caller_is_taking; - } - PreProcessDefinition (false, arg, caller_is_taking); + case "undef": + if (any_token_seen){ + Error_TokensSeen (); return caller_is_taking; + } + PreProcessDefinition (false, arg, caller_is_taking); + return caller_is_taking; } // @@ -2121,7 +2391,7 @@ namespace Mono.CSharp switch (cmd){ case "error": - Report.Error (1029, Location, "#error: '" + arg + "'"); + Report.Error (1029, Location, "#error: '{0}'", arg); return true; case "warning": @@ -2155,12 +2425,6 @@ namespace Mono.CSharp int c; string_builder.Length = 0; - // - // No need to parse full string when parsing lambda arguments - // - if (lambda_arguments_parsing) - return Token.LITERAL_STRING; - while ((c = get_char ()) != -1){ if (c == '"'){ if (quoted && peek_char () == '"'){ @@ -2168,8 +2432,8 @@ namespace Mono.CSharp get_char (); continue; } else { - val = string_builder.ToString (); - return Token.LITERAL_STRING; + val = new StringLiteral (string_builder.ToString (), Location); + return Token.LITERAL; } } @@ -2179,9 +2443,14 @@ namespace Mono.CSharp } if (!quoted){ - c = escape (c); + int surrogate; + c = escape (c, out surrogate); if (c == -1) return Token.ERROR; + if (surrogate != 0) { + string_builder.Append ((char) c); + c = surrogate; + } } string_builder.Append ((char) c); } @@ -2196,87 +2465,76 @@ namespace Mono.CSharp if (doc_state == XmlCommentState.Allowed) doc_state = XmlCommentState.NotAllowed; - switch (res) { - case Token.USING: - case Token.NAMESPACE: - check_incorrect_doc_comment (); - break; - } - - if (res == Token.PARTIAL) { - if (parsing_block > 0) { - val = new LocatedToken (Location, "partial"); - return Token.IDENTIFIER; - } - - // Save current position and parse next token. - PushPosition (); - int next_token = token (); - bool ok = (next_token == Token.CLASS) || - (next_token == Token.STRUCT) || - (next_token == Token.INTERFACE) || - (next_token == Token.VOID); + return res; + } - PopPosition (); + int consume_identifier (int c, bool quoted) + { + // + // This method is very performance sensitive. It accounts + // for approximately 25% of all parser time + // - if (ok) { - if (next_token == Token.VOID) { - if (RootContext.Version <= LanguageVersion.ISO_2) - Report.FeatureIsNotAvailable (Location, "partial methods"); - } else if (RootContext.Version == LanguageVersion.ISO_1) - Report.FeatureIsNotAvailable (Location, "partial types"); + int pos = 0; + int column = col; - return res; + if (c == '\\') { + int surrogate; + c = escape (c, out surrogate); + if (surrogate != 0) { + id_builder [pos++] = (char) c; + c = surrogate; } - - if (next_token < Token.LAST_KEYWORD) - Report.Error (267, Location, "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword"); - - val = new LocatedToken (Location, "partial"); - return Token.IDENTIFIER; } - return res; - } + id_builder [pos++] = (char) c; - private int consume_identifier (int s, bool quoted) - { - int pos = 1; - int c = -1; - - id_builder [0] = (char) s; + try { + while (true) { + c = reader.Read (); + + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) { + id_builder [pos++] = (char) c; + continue; + } - current_location = new Location (ref_line, Col); + if (c < 0x80) { + if (c == '\\') { + int surrogate; + c = escape (c, out surrogate); + if (surrogate != 0) { + if (is_identifier_part_character ((char) c)) + id_builder[pos++] = (char) c; + c = surrogate; + } - while ((c = get_char ()) != -1) { - loop: - if (is_identifier_part_character ((char) c)){ - if (pos == max_id_size){ - Report.Error (645, Location, "Identifier too long (limit is 512 chars)"); - return Token.ERROR; + continue; + } + } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) { + id_builder [pos++] = (char) c; + continue; } - - id_builder [pos++] = (char) c; -// putback_char = -1; - } else if (c == '\\') { - c = escape (c); - goto loop; - } else { -// putback_char = c; - putback (c); + + putback_char = c; break; } + } catch (IndexOutOfRangeException) { + Report.Error (645, Location, "Identifier too long (limit is 512 chars)"); + col += pos - 1; + return Token.ERROR; } + col += pos - 1; + // // Optimization: avoids doing the keyword lookup - // on uppercase letters and _ + // on uppercase letters // - if (!quoted && (s >= 'a' || s == '_')){ + if (id_builder [0] >= '_' && !quoted) { int keyword = GetKeyword (id_builder, pos); if (keyword != -1) { - val = Location; + val = LocatedToken.Create (null, ref_line, column); return keyword; } } @@ -2285,60 +2543,68 @@ namespace Mono.CSharp // Keep identifiers in an array of hashtables to avoid needless // allocations // - - if (identifiers [pos] != null) { - val = identifiers [pos][id_builder]; - if (val != null) { - val = new LocatedToken (Location, (string) val); + var identifiers_group = identifiers [pos]; + string s; + if (identifiers_group != null) { + if (identifiers_group.TryGetValue (id_builder, out s)) { + val = LocatedToken.Create (s, ref_line, column); if (quoted) - escaped_identifiers.Add (val); + AddEscapedIdentifier (((LocatedToken) val).Location); return Token.IDENTIFIER; } - } - else - identifiers [pos] = new CharArrayHashtable (pos); - - val = new String (id_builder, 0, pos); - if (RootContext.Version == LanguageVersion.ISO_1) { - for (int i = 1; i < id_builder.Length; i += 3) { - if (id_builder [i] == '_' && (id_builder [i - 1] == '_' || id_builder [i + 1] == '_')) { - Report.Error (1638, Location, - "`{0}': Any identifier with double underscores cannot be used when ISO language version mode is specified", val.ToString ()); - break; - } - } + } else { + // TODO: this should be number of files dependant + // corlib compilation peaks at 1000 and System.Core at 150 + int capacity = pos > 20 ? 10 : 100; + identifiers_group = new Dictionary (capacity, new IdentifiersComparer (pos)); + identifiers [pos] = identifiers_group; } char [] chars = new char [pos]; Array.Copy (id_builder, chars, pos); - identifiers [pos] [chars] = val; + s = new string (id_builder, 0, pos); + identifiers_group.Add (chars, s); - val = new LocatedToken (Location, (string) val); + val = LocatedToken.Create (s, ref_line, column); if (quoted) - escaped_identifiers.Add (val); + AddEscapedIdentifier (((LocatedToken) val).Location); + return Token.IDENTIFIER; } public int xtoken () { - int t; - bool doread = false; - int c; + int d, c; // Whether we have seen comments on the current line bool comments_seen = false; - val = null; - for (;(c = get_char ()) != -1;) { - if (c == '\t'){ - col = ((col + 8) / 8) * 8; + while ((c = get_char ()) != -1) { + switch (c) { + case '\t': + col = ((col + tab_size) / tab_size) * tab_size; continue; - } - - if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0 || c == 0) + + case ' ': + case '\f': + case '\v': + case 0xa0: + case 0: + case 0xFEFF: // Ignore BOM anywhere in the file continue; - if (c == '\r') { +/* This is required for compatibility with .NET + case 0xEF: + if (peek_char () == 0xBB) { + PushPosition (); + get_char (); + if (get_char () == 0xBF) + continue; + PopPosition (); + } + break; +*/ + case '\r': if (peek_char () != '\n') advance_line (); else @@ -2348,12 +2614,213 @@ namespace Mono.CSharp tokens_seen = false; comments_seen = false; continue; - } - // Handle double-slash comments. - if (c == '/'){ - int d = peek_char (); - + case '\\': + tokens_seen = true; + return consume_identifier (c); + + case '{': + val = LocatedToken.Create (ref_line, col); + return Token.OPEN_BRACE; + case '}': + val = LocatedToken.Create (ref_line, col); + return Token.CLOSE_BRACE; + case '[': + // To block doccomment inside attribute declaration. + if (doc_state == XmlCommentState.Allowed) + doc_state = XmlCommentState.NotAllowed; + val = LocatedToken.CreateOptional (ref_line, col); + return Token.OPEN_BRACKET; + case ']': + val = LocatedToken.CreateOptional (ref_line, col); + return Token.CLOSE_BRACKET; + case '(': + val = LocatedToken.Create (ref_line, col); + // + // An expression versions of parens can appear in block context only + // + if (parsing_block != 0 && !lambda_arguments_parsing) { + + // + // Optmize most common case where we know that parens + // is not special + // + switch (current_token) { + case Token.IDENTIFIER: + case Token.IF: + case Token.FOR: + case Token.FOREACH: + case Token.TYPEOF: + case Token.WHILE: + case Token.USING: + case Token.DEFAULT: + case Token.DELEGATE: + case Token.OP_GENERICS_GT: + return Token.OPEN_PARENS; + } + + // Optimize using peek + int xx = peek_char (); + switch (xx) { + case '(': + case '\'': + case '"': + case '0': + case '1': + return Token.OPEN_PARENS; + } + + lambda_arguments_parsing = true; + PushPosition (); + d = TokenizeOpenParens (); + PopPosition (); + lambda_arguments_parsing = false; + return d; + } + + return Token.OPEN_PARENS; + case ')': + val = LocatedToken.CreateOptional (ref_line, col); + return Token.CLOSE_PARENS; + case ',': + val = LocatedToken.CreateOptional (ref_line, col); + return Token.COMMA; + case ';': + val = LocatedToken.CreateOptional (ref_line, col); + return Token.SEMICOLON; + case '~': + val = LocatedToken.Create (ref_line, col); + return Token.TILDE; + case '?': + val = LocatedToken.Create (ref_line, col); + return TokenizePossibleNullableType (); + case '<': + val = LocatedToken.Create (ref_line, col); + if (parsing_generic_less_than++ > 0) + return Token.OP_GENERICS_LT; + + return TokenizeLessThan (); + + case '>': + val = LocatedToken.Create (ref_line, col); + d = peek_char (); + + if (d == '='){ + get_char (); + return Token.OP_GE; + } + + if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) { + parsing_generic_less_than--; + return Token.OP_GENERICS_GT; + } + + if (d == '>') { + get_char (); + d = peek_char (); + + if (d == '=') { + get_char (); + return Token.OP_SHIFT_RIGHT_ASSIGN; + } + return Token.OP_SHIFT_RIGHT; + } + + return Token.OP_GT; + + case '+': + val = LocatedToken.Create (ref_line, col); + d = peek_char (); + if (d == '+') { + d = Token.OP_INC; + } else if (d == '=') { + d = Token.OP_ADD_ASSIGN; + } else { + return Token.PLUS; + } + get_char (); + return d; + + case '-': + val = LocatedToken.Create (ref_line, col); + d = peek_char (); + if (d == '-') { + d = Token.OP_DEC; + } else if (d == '=') + d = Token.OP_SUB_ASSIGN; + else if (d == '>') + d = Token.OP_PTR; + else { + return Token.MINUS; + } + get_char (); + return d; + + case '!': + val = LocatedToken.Create (ref_line, col); + if (peek_char () == '='){ + get_char (); + return Token.OP_NE; + } + return Token.BANG; + + case '=': + val = LocatedToken.Create (ref_line, col); + d = peek_char (); + if (d == '='){ + get_char (); + return Token.OP_EQ; + } + if (d == '>'){ + get_char (); + return Token.ARROW; + } + + return Token.ASSIGN; + + case '&': + val = LocatedToken.Create (ref_line, col); + d = peek_char (); + if (d == '&'){ + get_char (); + return Token.OP_AND; + } + if (d == '='){ + get_char (); + return Token.OP_AND_ASSIGN; + } + return Token.BITWISE_AND; + + case '|': + val = LocatedToken.Create (ref_line, col); + d = peek_char (); + if (d == '|'){ + get_char (); + return Token.OP_OR; + } + if (d == '='){ + get_char (); + return Token.OP_OR_ASSIGN; + } + return Token.BITWISE_OR; + + case '*': + val = LocatedToken.Create (ref_line, col); + if (peek_char () == '='){ + get_char (); + return Token.OP_MULT_ASSIGN; + } + return Token.STAR; + + case '/': + d = peek_char (); + if (d == '='){ + val = LocatedToken.Create (ref_line, col); + get_char (); + return Token.OP_DIV_ASSIGN; + } + + // Handle double-slash comments. if (d == '/'){ get_char (); if (RootContext.Documentation != null && peek_char () == '/') { @@ -2367,9 +2834,8 @@ namespace Mono.CSharp warn_incorrect_doc_comment (); } } - while ((d = get_char ()) != -1 && (d != '\n') && d != '\r') - if (d == '\n'){ - } + while ((d = get_char ()) != -1 && (d != '\n') && d != '\r'); + any_token_seen |= tokens_seen; tokens_seen = false; comments_seen = false; @@ -2397,8 +2863,6 @@ namespace Mono.CSharp xml_comment_buffer.Append (Environment.NewLine); } - Location start_location = Location; - while ((d = get_char ()) != -1){ if (d == '*' && peek_char () == '/'){ get_char (); @@ -2419,53 +2883,57 @@ namespace Mono.CSharp } } if (!comments_seen) - Report.Error (1035, start_location, "End-of-file found, '*/' expected"); + Report.Error (1035, Location, "End-of-file found, '*/' expected"); if (docAppend) update_formatted_doc_comment (current_comment_start); continue; } - goto is_punct_label; - } + return Token.DIV; - - if (c == '\\' || is_identifier_start_character ((char)c)){ - tokens_seen = true; - return consume_identifier (c); - } + case '%': + val = LocatedToken.Create (ref_line, col); + if (peek_char () == '='){ + get_char (); + return Token.OP_MOD_ASSIGN; + } + return Token.PERCENT; - is_punct_label: - current_location = new Location (ref_line, Col); - if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){ - tokens_seen = true; - if (doread){ + case '^': + val = LocatedToken.Create (ref_line, col); + if (peek_char () == '='){ get_char (); + return Token.OP_XOR_ASSIGN; } - return t; - } + return Token.CARRET; - // white space - if (c == '\n'){ + case ':': + val = LocatedToken.Create (ref_line, col); + if (peek_char () == ':') { + get_char (); + return Token.DOUBLE_COLON; + } + return Token.COLON; + + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + tokens_seen = true; + return is_number (c); + + case '\n': // white space any_token_seen |= tokens_seen; tokens_seen = false; comments_seen = false; continue; - } - if (c >= '0' && c <= '9'){ + case '.': tokens_seen = true; - return is_number (c); - } - - if (c == '.'){ - tokens_seen = true; - int peek = peek_char (); - if (peek >= '0' && peek <= '9') + d = peek_char (); + if (d >= '0' && d <= '9') return is_number (c); return Token.DOT; - } - if (c == '#') { + case '#': if (tokens_seen || comments_seen) { Eror_WrongPreprocessorLocation (); return Token.ERROR; @@ -2503,67 +2971,141 @@ namespace Mono.CSharp } return Token.EOF; - } - if (c == '"') + case '"': return consume_string (false); - if (c == '\''){ - c = get_char (); - tokens_seen = true; - if (c == '\''){ - error_details = "Empty character literal"; - Report.Error (1011, Location, error_details); - return Token.ERROR; - } - if (c == '\r' || c == '\n') { - Report.Error (1010, Location, "Newline in constant"); - return Token.ERROR; - } - c = escape (c); - if (c == -1) - return Token.ERROR; - val = new System.Char (); - val = (char) c; - c = get_char (); - - if (c != '\''){ - error_details = "Too many characters in character literal"; - Report.Error (1012, Location, error_details); - - // Try to recover, read until newline or next "'" - while ((c = get_char ()) != -1){ - if (c == '\n'){ - break; - } - else if (c == '\'') - break; - } - return Token.ERROR; - } - return Token.LITERAL_CHARACTER; - } + case '\'': + return TokenizeBackslash (); - if (c == '@') { + case '@': c = get_char (); if (c == '"') { tokens_seen = true; return consume_string (true); - } else if (is_identifier_start_character ((char) c)){ + } + + if (is_identifier_start_character (c)){ return consume_identifier (c, true); - } else { - Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @"); } + + Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @"); + return Token.ERROR; + + case EvalStatementParserCharacter: + return Token.EVAL_STATEMENT_PARSER; + case EvalCompilationUnitParserCharacter: + return Token.EVAL_COMPILATION_UNIT_PARSER; + case EvalUsingDeclarationsParserCharacter: + return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER; + } + + if (is_identifier_start_character (c)) { + tokens_seen = true; + return consume_identifier (c); } error_details = ((char)c).ToString (); - return Token.ERROR; } + if (CompleteOnEOF){ + if (generated) + return Token.COMPLETE_COMPLETION; + + generated = true; + return Token.GENERATE_COMPLETION; + } + + return Token.EOF; } + int TokenizeBackslash () + { + int c = get_char (); + tokens_seen = true; + if (c == '\'') { + error_details = "Empty character literal"; + Report.Error (1011, Location, error_details); + return Token.ERROR; + } + if (c == '\r' || c == '\n') { + Report.Error (1010, Location, "Newline in constant"); + return Token.ERROR; + } + + int d; + c = escape (c, out d); + if (c == -1) + return Token.ERROR; + if (d != 0) + throw new NotImplementedException (); + + val = new CharLiteral ((char) c, Location); + c = get_char (); + + if (c != '\'') { + Report.Error (1012, Location, "Too many characters in character literal"); + + // Try to recover, read until newline or next "'" + while ((c = get_char ()) != -1) { + if (c == '\n' || c == '\'') + break; + } + return Token.ERROR; + } + + return Token.LITERAL; + } + + int TokenizeLessThan () + { + int d; + if (handle_typeof) { + PushPosition (); + if (parse_generic_dimension (out d)) { + val = d; + DiscardPosition (); + return Token.GENERIC_DIMENSION; + } + PopPosition (); + } + + // Save current position and parse next token. + PushPosition (); + if (parse_less_than ()) { + if (parsing_generic_declaration && token () != Token.DOT) { + d = Token.OP_GENERICS_LT_DECL; + } else { + d = Token.OP_GENERICS_LT; + } + PopPosition (); + return d; + } + + PopPosition (); + parsing_generic_less_than = 0; + + d = peek_char (); + if (d == '<') { + get_char (); + d = peek_char (); + + if (d == '=') { + get_char (); + return Token.OP_SHIFT_LEFT_ASSIGN; + } + return Token.OP_SHIFT_LEFT; + } + + if (d == '=') { + get_char (); + return Token.OP_LE; + } + return Token.OP_LT; + } + // // Handles one line xml comment // @@ -2618,7 +3160,7 @@ namespace Mono.CSharp if (current_comment_location.IsNull) { // "-2" is for heading "//" or "/*" current_comment_location = - new Location (ref_line, col - 2); + new Location (ref_line, hidden ? -1 : col - 2); } } @@ -2661,6 +3203,10 @@ namespace Mono.CSharp return null; } + Report Report { + get { return context.Report; } + } + void reset_doc_comment () { xml_comment_buffer.Length = 0; @@ -2670,8 +3216,7 @@ namespace Mono.CSharp public void cleanup () { if (ifstack != null && ifstack.Count >= 1) { - current_location = new Location (ref_line, Col); - int state = (int) ifstack.Pop (); + int state = ifstack.Pop (); if ((state & REGION) != 0) Report.Error (1038, Location, "#endregion directive expected"); else