X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mcs%2Fmcs%2Fcs-tokenizer.cs;h=5d1c92c54159e13f22995161cdc23ca7d69ac297;hb=f448841be113e8e9e19c9dca1fb9348fb7db3ee1;hp=4f3f1e53bbb3a84151d458f130191498f62c4e5e;hpb=2f3db2f36a5e2a7b236878343cedb55051c208c1;p=mono.git diff --git a/mcs/mcs/cs-tokenizer.cs b/mcs/mcs/cs-tokenizer.cs index 4f3f1e53bbb..5d1c92c5415 100644 --- a/mcs/mcs/cs-tokenizer.cs +++ b/mcs/mcs/cs-tokenizer.cs @@ -14,7 +14,7 @@ using System; using System.Text; -using System.Collections; +using System.Collections.Generic; using System.IO; using System.Globalization; using System.Reflection; @@ -27,23 +27,142 @@ namespace Mono.CSharp public class Tokenizer : yyParser.yyInput { + class KeywordEntry + { + public readonly int Token; + public KeywordEntry Next; + public readonly char[] Value; + + public KeywordEntry (string value, int token) + { + this.Value = value.ToCharArray (); + this.Token = token; + } + } + + sealed class IdentifiersComparer : IEqualityComparer + { + readonly int length; + + public IdentifiersComparer (int length) + { + this.length = length; + } + + public bool Equals (char[] x, char[] y) + { + for (int i = 0; i < length; ++i) + if (x [i] != y [i]) + return false; + + return true; + } + + public int GetHashCode (char[] obj) + { + int h = 0; + for (int i = 0; i < length; ++i) + h = (h << 5) - h + obj [i]; + + return h; + } + } + + // + // This class has to be used in the parser only, it reuses token + // details after each parse + // + public class LocatedToken + { + int row, column; + string value; + + static LocatedToken[] buffer; + static int pos; + + private LocatedToken () + { + } + + public static LocatedToken Create (int row, int column) + { + return Create (null, row, column); + } + + public static LocatedToken Create (string value, int row, int column) + { + // + // TODO: I am not very happy about the logic but it's the best + // what I could come up with for now. + // Ideally we should be using just tiny buffer (256 elements) which + // is enough to hold all details for currect stack and recycle elements + // poped from the stack but there is a trick needed to recycle + // them properly. + // + LocatedToken entry; + if (pos >= buffer.Length) { + entry = new LocatedToken (); + } else { + entry = buffer [pos]; + if (entry == null) { + entry = new LocatedToken (); + buffer [pos] = entry; + } + + ++pos; + } + entry.value = value; + entry.row = row; + entry.column = column; + return entry; + } + + // + // Used for token not required by expression evaluator + // + public static LocatedToken CreateOptional (int row, int col) + { +#if false + return Create (row, col); +#endif + return null; + } + + public static void Initialize () + { + if (buffer == null) + buffer = new LocatedToken [10000]; + pos = 0; + } + + public Location Location { + get { return new Location (row, column); } + } + + public string Value { + get { return value; } + } + } + SeekableStreamReader reader; SourceFile ref_name; CompilationUnit file_name; + CompilerContext context; bool hidden = false; int ref_line = 1; int line = 1; int col = 0; int previous_col; int current_token; + int tab_size; bool handle_get_set = false; bool handle_remove_add = false; bool handle_where = false; bool handle_typeof = false; bool lambda_arguments_parsing; - Location current_location; Location current_comment_location = Location.Null; - ArrayList escaped_identifiers = new ArrayList (); + List escaped_identifiers; + int parsing_generic_less_than; // // Used mainly for parser optimizations. Some expressions for instance @@ -51,7 +170,12 @@ namespace Mono.CSharp // scope only // public int parsing_block; - internal int query_parsing; + internal bool query_parsing; + + // + // When parsing type only, useful for ambiguous nullable types + // + public int parsing_type; // // Set when parsing generic declaration (type or method header) @@ -94,6 +218,14 @@ namespace Mono.CSharp // bool tokens_seen = false; + // + // Set to true once the GENERATE_COMPLETION token has bee + // returned. This helps produce one GENERATE_COMPLETION, + // as many COMPLETE_COMPLETION as necessary to complete the + // AST tree and one final EOF. + // + bool generated; + // // Whether a token has been seen on the file // This is needed because `define' is not allowed to be used @@ -101,62 +233,12 @@ namespace Mono.CSharp // bool any_token_seen = false; - static Hashtable token_values; static readonly char[] simple_whitespaces = new char[] { ' ', '\t' }; - private static Hashtable TokenValueName - { - get { - if (token_values == null) - token_values = GetTokenValueNameHash (); - - return token_values; - } - } - - private static Hashtable GetTokenValueNameHash () - { - Type t = typeof (Token); - FieldInfo [] fields = t.GetFields (); - Hashtable hash = new Hashtable (); - foreach (FieldInfo field in fields) { - if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int)) - hash.Add (field.GetValue (null), field.Name); - } - return hash; - } - - // - // Returns a verbose representation of the current location - // - public string location { - get { - string det; - - if (current_token == Token.ERROR) - det = "detail: " + error_details; - else - det = ""; - - // return "Line: "+line+" Col: "+col + "\n" + - // "VirtLine: "+ref_line + - // " Token: "+current_token + " " + det; - string current_token_name = TokenValueName [current_token] as string; - if (current_token_name == null) - current_token_name = current_token.ToString (); - - return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name, - ref_line, - col, - current_token_name, - det); - } - } - public bool PropertyParsing { get { return handle_get_set; } set { handle_get_set = value; } - } + } public bool EventParsing { get { return handle_remove_add; } @@ -172,6 +254,11 @@ namespace Mono.CSharp get { return handle_typeof; } set { handle_typeof = value; } } + + public int TabSize { + get { return tab_size; } + set { tab_size = value; } + } public XmlCommentState doc_state { get { return xml_doc_state; } @@ -184,19 +271,34 @@ namespace Mono.CSharp } } + // + // This is used to trigger completion generation on the parser + public bool CompleteOnEOF; + + void AddEscapedIdentifier (Location loc) + { + if (escaped_identifiers == null) + escaped_identifiers = new List (); + + escaped_identifiers.Add (loc); + } + public bool IsEscapedIdentifier (Location loc) { - foreach (LocatedToken lt in escaped_identifiers) - if (lt.Location.Equals (loc)) - return true; + if (escaped_identifiers != null) { + foreach (Location lt in escaped_identifiers) + if (lt.Equals (loc)) + return true; + } + return false; } // // Class variables // - static CharArrayHashtable[] keywords; - static Hashtable keyword_strings; + static KeywordEntry[][] keywords; + static Dictionary keyword_strings; // TODO: HashSet static NumberStyles styles; static NumberFormatInfo csharp_format_info; @@ -204,7 +306,7 @@ namespace Mono.CSharp // Values for the associated token returned // internal int putback_char; // Used by repl only - Object val; + object val; // // Pre-processor @@ -217,18 +319,20 @@ namespace Mono.CSharp // // pre-processor if stack state: // - Stack ifstack; + Stack ifstack; static System.Text.StringBuilder string_builder; const int max_id_size = 512; static char [] id_builder = new char [max_id_size]; - static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1]; + public static Dictionary[] identifiers = new Dictionary[max_id_size + 1]; const int max_number_size = 512; static char [] number_builder = new char [max_number_size]; static int number_pos; + + static StringBuilder static_cmd_arg = new System.Text.StringBuilder (); // // Details about the error encoutered by the tokenizer @@ -247,19 +351,14 @@ namespace Mono.CSharp } } - public int Col { - get { - return col; - } - } - // // This is used when the tokenizer needs to save // the current position as it needs to do some parsing // on its own to deamiguate a token in behalf of the // parser. // - Stack position_stack = new Stack (2); + Stack position_stack = new Stack (2); + class Position { public int position; public int line; @@ -268,9 +367,10 @@ namespace Mono.CSharp public bool hidden; public int putback_char; public int previous_col; - public Stack ifstack; + public Stack ifstack; public int parsing_generic_less_than; public int current_token; + public object val; public Position (Tokenizer t) { @@ -281,10 +381,16 @@ namespace Mono.CSharp hidden = t.hidden; putback_char = t.putback_char; previous_col = t.previous_col; - if (t.ifstack != null && t.ifstack.Count != 0) - ifstack = (Stack)t.ifstack.Clone (); + if (t.ifstack != null && t.ifstack.Count != 0) { + // There is no simple way to clone Stack all + // methods reverse the order + var clone = t.ifstack.ToArray (); + Array.Reverse (clone); + ifstack = new Stack (clone); + } parsing_generic_less_than = t.parsing_generic_less_than; current_token = t.current_token; + val = t.val; } } @@ -295,7 +401,7 @@ namespace Mono.CSharp public void PopPosition () { - Position p = (Position) position_stack.Pop (); + Position p = position_stack.Pop (); reader.Position = p.position; ref_line = p.ref_line; @@ -307,6 +413,7 @@ namespace Mono.CSharp ifstack = p.ifstack; parsing_generic_less_than = p.parsing_generic_less_than; current_token = p.current_token; + val = p.val; } // Do not reset the position, ignore it. @@ -317,17 +424,33 @@ namespace Mono.CSharp static void AddKeyword (string kw, int token) { - keyword_strings.Add (kw, kw); - if (keywords [kw.Length] == null) { - keywords [kw.Length] = new CharArrayHashtable (kw.Length); + keyword_strings.Add (kw, null); + + int length = kw.Length; + if (keywords [length] == null) { + keywords [length] = new KeywordEntry ['z' - '_' + 1]; + } + + int char_index = kw [0] - '_'; + KeywordEntry kwe = keywords [length] [char_index]; + if (kwe == null) { + keywords [length] [char_index] = new KeywordEntry (kw, token); + return; + } + + while (kwe.Next != null) { + kwe = kwe.Next; } - keywords [kw.Length] [kw.ToCharArray ()] = token; + + kwe.Next = new KeywordEntry (kw, token); } static void InitTokens () { - keyword_strings = new Hashtable (); - keywords = new CharArrayHashtable [64]; + keyword_strings = new Dictionary (); + + // 11 is the length of the longest keyword for now + keywords = new KeywordEntry [11] []; AddKeyword ("__arglist", Token.ARGLIST); AddKeyword ("abstract", Token.ABSTRACT); @@ -434,12 +557,7 @@ namespace Mono.CSharp // static Tokenizer () { - Reset (); - } - - public static void Reset () - { - InitTokens (); + InitTokens (); csharp_format_info = NumberFormatInfo.InvariantInfo; styles = NumberStyles.Float; @@ -448,19 +566,37 @@ namespace Mono.CSharp int GetKeyword (char[] id, int id_len) { - /* - * Keywords are stored in an array of hashtables grouped by their - * length. - */ + // + // Keywords are stored in an array of arrays grouped by their + // length and then by the first character + // + if (id_len >= keywords.Length || keywords [id_len] == null) + return -1; - if ((id_len >= keywords.Length) || (keywords [id_len] == null)) + int first_index = id [0] - '_'; + if (first_index > 'z') return -1; - object o = keywords [id_len] [id]; - if (o == null) + KeywordEntry kwe = keywords [id_len] [first_index]; + if (kwe == null) return -1; - - int res = (int) o; + + int res; + do { + res = kwe.Token; + for (int i = 1; i < id_len; ++i) { + if (id [i] != kwe.Value [i]) { + res = 0; + kwe = kwe.Next; + break; + } + } + } while (res == 0 && kwe != null); + + if (res == 0) + return -1; + + int next_token; switch (res) { case Token.GET: case Token.SET: @@ -483,7 +619,7 @@ namespace Mono.CSharp } break; case Token.WHERE: - if (!handle_where && query_parsing == 0) + if (!handle_where && !query_parsing) res = -1; break; case Token.FROM: @@ -491,7 +627,7 @@ namespace Mono.CSharp // A query expression is any expression that starts with `from identifier' // followed by any token except ; , = // - if (query_parsing == 0) { + if (!query_parsing) { if (lambda_arguments_parsing) { res = -1; break; @@ -514,16 +650,17 @@ namespace Mono.CSharp case Token.STRING: case Token.UINT: case Token.ULONG: - int next_token = xtoken (); + next_token = xtoken (); if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS) goto default; - ++query_parsing; + res = Token.FROM_FIRST; + query_parsing = true; if (RootContext.Version <= LanguageVersion.ISO_2) Report.FeatureIsNotAvailable (Location, "query expressions"); break; case Token.VOID: - Expression.Error_VoidInvalidInTheContext (Location); + Expression.Error_VoidInvalidInTheContext (Location, Report); break; default: PopPosition (); @@ -546,8 +683,51 @@ namespace Mono.CSharp case Token.ASCENDING: case Token.DESCENDING: case Token.INTO: - if (query_parsing == 0) + if (!query_parsing) + res = -1; + break; + + case Token.USING: + case Token.NAMESPACE: + // TODO: some explanation needed + check_incorrect_doc_comment (); + break; + + case Token.PARTIAL: + if (parsing_block > 0) { res = -1; + break; + } + + // Save current position and parse next token. + PushPosition (); + + next_token = token (); + bool ok = (next_token == Token.CLASS) || + (next_token == Token.STRUCT) || + (next_token == Token.INTERFACE) || + (next_token == Token.VOID); + + PopPosition (); + + if (ok) { + if (next_token == Token.VOID) { + if (RootContext.Version == LanguageVersion.ISO_1 || + RootContext.Version == LanguageVersion.ISO_2) + Report.FeatureIsNotAvailable (Location, "partial methods"); + } else if (RootContext.Version == LanguageVersion.ISO_1) + Report.FeatureIsNotAvailable (Location, "partial types"); + + return res; + } + + if (next_token < Token.LAST_KEYWORD) { + Report.Error (267, Location, + "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword"); + return token (); + } + + res = -1; break; } @@ -555,19 +735,27 @@ namespace Mono.CSharp } public Location Location { - get { return current_location; } + get { + return new Location (ref_line, hidden ? -1 : col); + } } - public Tokenizer (SeekableStreamReader input, CompilationUnit file) + public Tokenizer (SeekableStreamReader input, CompilationUnit file, CompilerContext ctx) { this.ref_name = file; this.file_name = file; + this.context = ctx; reader = input; putback_char = -1; xml_comment_buffer = new StringBuilder (); + if (Environment.OSVersion.Platform == PlatformID.Win32NT) + tab_size = 4; + else + tab_size = 8; + // // FIXME: This could be `Location.Push' but we have to // find out why the MS compiler allows this @@ -575,81 +763,75 @@ namespace Mono.CSharp Mono.CSharp.Location.Push (file, file); } - static bool is_identifier_start_character (char c) + static bool is_identifier_start_character (int c) { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter (c); + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c); } static bool is_identifier_part_character (char c) { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || - Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation; + if (c >= 'a' && c <= 'z') + return true; + + if (c >= 'A' && c <= 'Z') + return true; + + if (c == '_' || (c >= '0' && c <= '9')) + return true; + + if (c < 0x80) + return false; + + return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation; } public static bool IsKeyword (string s) { - return keyword_strings [s] != null; + return keyword_strings.ContainsKey (s); } // // Open parens micro parser. Detects both lambda and cast ambiguity. - // - + // int TokenizeOpenParens () { - int ntoken = -1; int ptoken; - bool cast_posible = false; - bool cast_not = false; - bool has_star = false; + current_token = -1; + + int bracket_level = 0; + bool is_type = false; + bool can_be_type = false; while (true) { - ptoken = ntoken; - ntoken = xtoken (); + ptoken = current_token; + token (); - switch (ntoken) { + switch (current_token) { case Token.CLOSE_PARENS: - ntoken = xtoken (); + token (); // - // Token is a lambda + // Expression inside parens is lambda, (int i) => // - if (ntoken == Token.ARROW) { - if (RootContext.Version <= LanguageVersion.ISO_2) - Report.FeatureIsNotAvailable (Location, "lambda expressions"); - + if (current_token == Token.ARROW) return Token.OPEN_PARENS_LAMBDA; - } // - // Token is possible cast, parser will decide later + // Expression inside parens is single type, (int[]) // - if (cast_posible && !cast_not) { - switch (ntoken) { - // - // Indirection is a special - // cast: (int*)&a; - // binary: (C)&a; - // - case Token.BITWISE_AND: - case Token.OP_INC: - case Token.OP_DEC: - case Token.PLUS: - case Token.MINUS: - if (!has_star) - break; - return Token.OPEN_PARENS_CAST; + if (is_type) + return Token.OPEN_PARENS_CAST; + // + // Expression is possible cast, look at next token, (T)null + // + if (can_be_type) { + switch (current_token) { case Token.OPEN_PARENS: case Token.BANG: case Token.TILDE: case Token.IDENTIFIER: - case Token.LITERAL_INTEGER: - case Token.LITERAL_FLOAT: - case Token.LITERAL_DOUBLE: - case Token.LITERAL_DECIMAL: - case Token.LITERAL_CHARACTER: - case Token.LITERAL_STRING: + case Token.LITERAL: case Token.BASE: case Token.CHECKED: case Token.DELEGATE: @@ -687,8 +869,10 @@ namespace Mono.CSharp return Token.OPEN_PARENS; case Token.DOT: + case Token.DOUBLE_COLON: if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT) goto default; + continue; case Token.IDENTIFIER: @@ -698,17 +882,14 @@ namespace Mono.CSharp case Token.COMMA: case Token.DOUBLE_COLON: case -1: - cast_posible = true; + if (bracket_level == 0) + can_be_type = true; continue; default: - cast_not = true; + can_be_type = is_type = false; continue; } - case Token.STAR: - has_star = true; - continue; - case Token.OBJECT: case Token.STRING: case Token.BOOL: @@ -725,21 +906,39 @@ namespace Mono.CSharp case Token.ULONG: case Token.CHAR: case Token.VOID: - case Token.OP_GENERICS_GT: + if (bracket_level == 0) + is_type = true; + continue; + + case Token.COMMA: + if (bracket_level == 0) { + bracket_level = 100; + can_be_type = is_type = false; + } + continue; + case Token.OP_GENERICS_LT: - case Token.INTERR_NULLABLE: case Token.OPEN_BRACKET: + if (bracket_level++ == 0) + is_type = true; + continue; + + case Token.OP_GENERICS_GT: case Token.CLOSE_BRACKET: - case Token.COMMA: - case Token.DOUBLE_COLON: - cast_posible = true; + --bracket_level; + continue; + + case Token.INTERR_NULLABLE: + case Token.STAR: + if (bracket_level == 0) + is_type = true; continue; case Token.REF: case Token.OUT: - cast_not = true; + can_be_type = is_type = false; continue; - + default: return Token.OPEN_PARENS; } @@ -770,6 +969,8 @@ namespace Mono.CSharp the_token = token (); } while (the_token != Token.CLOSE_BRACKET); the_token = token (); + } else if (the_token == Token.IN || the_token == Token.OUT) { + the_token = token (); } switch (the_token) { case Token.IDENTIFIER: @@ -790,7 +991,6 @@ namespace Mono.CSharp case Token.CHAR: case Token.VOID: break; - case Token.OP_GENERICS_GT: return true; @@ -850,305 +1050,51 @@ namespace Mono.CSharp return the_token; } - int parsing_generic_less_than = 0; - - int is_punct (char c, ref bool doread) + // + // Tonizes `?' using custom disambiguous rules to return one + // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR + // + // Tricky expression look like: + // + // Foo ? a = x ? b : c; + // + int TokenizePossibleNullableType () { - int d; - int t; + if (parsing_block == 0 || parsing_type > 0) + return Token.INTERR_NULLABLE; - doread = false; + int d = peek_char (); + if (d == '?') { + get_char (); + return Token.OP_COALESCING; + } - switch (c){ - case '{': - val = Location; - return Token.OPEN_BRACE; - case '}': - val = Location; - return Token.CLOSE_BRACE; - case '[': - // To block doccomment inside attribute declaration. - if (doc_state == XmlCommentState.Allowed) - doc_state = XmlCommentState.NotAllowed; - return Token.OPEN_BRACKET; - case ']': - return Token.CLOSE_BRACKET; - case '(': - val = Location; - // - // An expression versions of parens can appear in block context only - // - if (parsing_block != 0 && !lambda_arguments_parsing) { - lambda_arguments_parsing = true; - PushPosition (); - t = TokenizeOpenParens (); - PopPosition (); - lambda_arguments_parsing = false; - return t; - } + switch (current_token) { + case Token.CLOSE_PARENS: + case Token.TRUE: + case Token.FALSE: + case Token.NULL: + case Token.LITERAL: + return Token.INTERR; + } - return Token.OPEN_PARENS; - case ')': - return Token.CLOSE_PARENS; - case ',': - return Token.COMMA; - case ';': - val = Location; - return Token.SEMICOLON; - case '~': - val = Location; - return Token.TILDE; - case '?': - return TokenizePossibleNullableType (); - } - - if (c == '<') { - if (parsing_generic_less_than++ > 0) - return Token.OP_GENERICS_LT; - - if (handle_typeof) { - int dimension; - PushPosition (); - if (parse_generic_dimension (out dimension)) { - val = dimension; - DiscardPosition (); - return Token.GENERIC_DIMENSION; - } - PopPosition (); - } - - // Save current position and parse next token. - PushPosition (); - bool is_generic_lt = parse_less_than (); - if (is_generic_lt) { - int rt; - if (parsing_generic_declaration && token () != Token.DOT) { - rt = Token.OP_GENERICS_LT_DECL; - } else { - rt = Token.OP_GENERICS_LT; - } - PopPosition (); - return rt; - } - - PopPosition (); - parsing_generic_less_than = 0; - - d = peek_char (); - if (d == '<'){ - get_char (); - d = peek_char (); - - if (d == '='){ - doread = true; - return Token.OP_SHIFT_LEFT_ASSIGN; - } - return Token.OP_SHIFT_LEFT; - } else if (d == '='){ - doread = true; - return Token.OP_LE; - } - return Token.OP_LT; - } else if (c == '>') { - d = peek_char (); - - if (d == '='){ - doread = true; - return Token.OP_GE; - } - - if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) { - parsing_generic_less_than--; - return Token.OP_GENERICS_GT; - } - - if (d == '>') { - get_char (); - d = peek_char (); - - if (d == '=') { - doread = true; - return Token.OP_SHIFT_RIGHT_ASSIGN; - } - return Token.OP_SHIFT_RIGHT; - } - - return Token.OP_GT; - } - - d = peek_char (); - if (c == '+'){ - - if (d == '+') { - val = Location; - t = Token.OP_INC; - } - else if (d == '=') - t = Token.OP_ADD_ASSIGN; - else { - val = Location; - return Token.PLUS; - } - doread = true; - return t; - } - if (c == '-'){ - if (d == '-') { - val = Location; - t = Token.OP_DEC; - } - else if (d == '=') - t = Token.OP_SUB_ASSIGN; - else if (d == '>') - t = Token.OP_PTR; - else { - val = Location; - return Token.MINUS; - } - doread = true; - return t; - } - - if (c == '!'){ - if (d == '='){ - doread = true; - return Token.OP_NE; - } - val = Location; - return Token.BANG; - } - - if (c == '='){ - if (d == '='){ - doread = true; - return Token.OP_EQ; - } - if (d == '>'){ - doread = true; - val = Location; - return Token.ARROW; - } - - return Token.ASSIGN; - } - - if (c == '&'){ - if (d == '&'){ - doread = true; - return Token.OP_AND; - } else if (d == '='){ - doread = true; - return Token.OP_AND_ASSIGN; - } - val = Location; - return Token.BITWISE_AND; - } - - if (c == '|'){ - if (d == '|'){ - doread = true; - return Token.OP_OR; - } else if (d == '='){ - doread = true; - return Token.OP_OR_ASSIGN; - } - return Token.BITWISE_OR; - } - - if (c == '*'){ - if (d == '='){ - doread = true; - return Token.OP_MULT_ASSIGN; - } - val = Location; - return Token.STAR; - } - - if (c == '/'){ - if (d == '='){ - doread = true; - return Token.OP_DIV_ASSIGN; - } - return Token.DIV; - } - - if (c == '%'){ - if (d == '='){ - doread = true; - return Token.OP_MOD_ASSIGN; - } - return Token.PERCENT; - } - - if (c == '^'){ - if (d == '='){ - doread = true; - return Token.OP_XOR_ASSIGN; - } - return Token.CARRET; - } - - if (c == ':'){ - if (d == ':'){ - doread = true; - return Token.DOUBLE_COLON; - } - val = Location; - return Token.COLON; - } - - return Token.ERROR; - } - - // - // Tonizes `?' using custom disambiguous rules to return one - // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR - // - // Tricky expression look like: - // - // Foo ? a = x ? b : c; - // - int TokenizePossibleNullableType () - { - if (parsing_block == 0) - return Token.INTERR_NULLABLE; - - int d = peek_char (); - if (d == '?') { - get_char (); - return Token.OP_COALESCING; - } - - switch (current_token) { - case Token.CLOSE_PARENS: - case Token.TRUE: - case Token.FALSE: - case Token.NULL: - case Token.LITERAL_INTEGER: - case Token.LITERAL_STRING: - return Token.INTERR; - } - - if (d != ' ') { - if (d == ',' || d == ';' || d == '>') - return Token.INTERR_NULLABLE; - if (d == '*' || (d >= '0' && d <= '9')) - return Token.INTERR; - } + if (d != ' ') { + if (d == ',' || d == ';' || d == '>') + return Token.INTERR_NULLABLE; + if (d == '*' || (d >= '0' && d <= '9')) + return Token.INTERR; + } PushPosition (); + current_token = Token.NONE; int next_token; switch (xtoken ()) { - case Token.LITERAL_INTEGER: - case Token.LITERAL_STRING: - case Token.LITERAL_CHARACTER: - case Token.LITERAL_DECIMAL: - case Token.LITERAL_DOUBLE: - case Token.LITERAL_FLOAT: + case Token.LITERAL: case Token.TRUE: case Token.FALSE: case Token.NULL: case Token.THIS: + case Token.NEW: next_token = Token.INTERR; break; @@ -1245,24 +1191,18 @@ namespace Mono.CSharp return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f'); } - static int real_type_suffix (int c) + static TypeCode real_type_suffix (int c) { - int t; - switch (c){ case 'F': case 'f': - t = Token.LITERAL_FLOAT; - break; + return TypeCode.Single; case 'D': case 'd': - t = Token.LITERAL_DOUBLE; - break; + return TypeCode.Double; case 'M': case 'm': - t= Token.LITERAL_DECIMAL; - break; + return TypeCode.Decimal; default: - return Token.NONE; + return TypeCode.Empty; } - return t; } int integer_type_suffix (ulong ul, int c) @@ -1289,16 +1229,8 @@ namespace Mono.CSharp // Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)"); } - // - // This goto statement causes the MS CLR 2.0 beta 1 csc to report an error, so - // work around that. - // - //goto case 'L'; - if (is_long) - scanning = false; - is_long = true; - get_char (); - break; + + goto case 'L'; case 'L': if (is_long) @@ -1316,38 +1248,40 @@ namespace Mono.CSharp } if (is_long && is_unsigned){ - val = ul; - return Token.LITERAL_INTEGER; - } else if (is_unsigned){ + val = new ULongLiteral (ul, Location); + return Token.LITERAL; + } + + if (is_unsigned){ // uint if possible, or ulong else. if ((ul & 0xffffffff00000000) == 0) - val = (uint) ul; + val = new UIntLiteral ((uint) ul, Location); else - val = ul; + val = new ULongLiteral (ul, Location); } else if (is_long){ // long if possible, ulong otherwise if ((ul & 0x8000000000000000) != 0) - val = ul; + val = new ULongLiteral (ul, Location); else - val = (long) ul; + val = new LongLiteral ((long) ul, Location); } else { // int, uint, long or ulong in that order if ((ul & 0xffffffff00000000) == 0){ uint ui = (uint) ul; if ((ui & 0x80000000) != 0) - val = ui; + val = new UIntLiteral (ui, Location); else - val = (int) ui; + val = new IntLiteral ((int) ui, Location); } else { if ((ul & 0x8000000000000000) != 0) - val = ul; + val = new ULongLiteral (ul, Location); else - val = (long) ul; + val = new LongLiteral ((long) ul, Location); } } - return Token.LITERAL_INTEGER; + return Token.LITERAL; } // @@ -1376,51 +1310,49 @@ namespace Mono.CSharp } catch (OverflowException) { error_details = "Integral constant is too large"; Report.Error (1021, Location, error_details); - val = 0ul; - return Token.LITERAL_INTEGER; + val = new IntLiteral (0, Location); + return Token.LITERAL; } catch (FormatException) { Report.Error (1013, Location, "Invalid number"); - val = 0ul; - return Token.LITERAL_INTEGER; + val = new IntLiteral (0, Location); + return Token.LITERAL; } } - int adjust_real (int t) + int adjust_real (TypeCode t) { string s = new String (number_builder, 0, number_pos); const string error_details = "Floating-point constant is outside the range of type `{0}'"; switch (t){ - case Token.LITERAL_DECIMAL: + case TypeCode.Decimal: try { - val = System.Decimal.Parse (s, styles, csharp_format_info); + val = new DecimalLiteral (decimal.Parse (s, styles, csharp_format_info), Location); } catch (OverflowException) { - val = 0m; + val = new DecimalLiteral (0, Location); Report.Error (594, Location, error_details, "decimal"); } break; - case Token.LITERAL_FLOAT: + case TypeCode.Single: try { - val = float.Parse (s, styles, csharp_format_info); + val = new FloatLiteral (float.Parse (s, styles, csharp_format_info), Location); } catch (OverflowException) { - val = 0.0f; + val = new FloatLiteral (0, Location); Report.Error (594, Location, error_details, "float"); } break; - - case Token.LITERAL_DOUBLE: - case Token.NONE: - t = Token.LITERAL_DOUBLE; + default: try { - val = System.Double.Parse (s, styles, csharp_format_info); + val = new DoubleLiteral (double.Parse (s, styles, csharp_format_info), Location); } catch (OverflowException) { - val = 0.0; + val = new DoubleLiteral (0, Location); Report.Error (594, Location, error_details, "double"); } break; } - return t; + + return Token.LITERAL; } int handle_hex () @@ -1446,13 +1378,13 @@ namespace Mono.CSharp } catch (OverflowException){ error_details = "Integral constant is too large"; Report.Error (1021, Location, error_details); - val = 0ul; - return Token.LITERAL_INTEGER; + val = new IntLiteral (0, Location); + return Token.LITERAL; } catch (FormatException) { Report.Error (1013, Location, "Invalid number"); - val = 0ul; - return Token.LITERAL_INTEGER; + val = new IntLiteral (0, Location); + return Token.LITERAL; } return integer_type_suffix (ul, peek_char ()); @@ -1464,7 +1396,6 @@ namespace Mono.CSharp int is_number (int c) { bool is_real = false; - int type; number_pos = 0; @@ -1521,21 +1452,21 @@ namespace Mono.CSharp c = get_char (); } - type = real_type_suffix (c); - if (type == Token.NONE && !is_real){ + var type = real_type_suffix (c); + if (type == TypeCode.Empty && !is_real){ putback (c); return adjust_int (c); - } else - is_real = true; + } - if (type == Token.NONE){ + is_real = true; + + if (type == TypeCode.Empty){ putback (c); } if (is_real) return adjust_real (type); - Console.WriteLine ("This should not be reached"); throw new Exception ("Is Number should never reach this point"); } @@ -1686,9 +1617,8 @@ namespace Mono.CSharp int peek_char () { - if (putback_char != -1) - return putback_char; - putback_char = reader.Read (); + if (putback_char == -1) + putback_char = reader.Read (); return putback_char; } @@ -1721,7 +1651,7 @@ namespace Mono.CSharp public bool advance () { - return peek_char () != -1; + return peek_char () != -1 || CompleteOnEOF; } public Object Value { @@ -1741,8 +1671,6 @@ namespace Mono.CSharp return current_token; } - static StringBuilder static_cmd_arg = new System.Text.StringBuilder (); - void get_cmd_arg (out string cmd, out string arg) { int c; @@ -1780,17 +1708,25 @@ namespace Mono.CSharp c = get_char (); static_cmd_arg.Length = 0; + int has_identifier_argument = 0; + while (c != -1 && c != '\n' && c != '\r') { - if (c == '\\') { - int peek = peek_char (); - if (peek == 'U' || peek == 'u') { - int surrogate; - c = EscapeUnicode (c, out surrogate); - if (surrogate != 0) { - if (is_identifier_part_character ((char) c)) - static_cmd_arg.Append ((char) c); - c = surrogate; + if (c == '\\' && has_identifier_argument >= 0) { + if (has_identifier_argument != 0 || (cmd == "define" || cmd == "if" || cmd == "elif" || cmd == "undef")) { + has_identifier_argument = 1; + + int peek = peek_char (); + if (peek == 'U' || peek == 'u') { + int surrogate; + c = EscapeUnicode (c, out surrogate); + if (surrogate != 0) { + if (is_identifier_part_character ((char) c)) + static_cmd_arg.Append ((char) c); + c = surrogate; + } } + } else { + has_identifier_argument = -1; } } static_cmd_arg.Append ((char) c); @@ -2033,18 +1969,18 @@ namespace Mono.CSharp int[] codes = ParseNumbers (arg.Substring (w_disable.Length)); foreach (int code in codes) { if (code != 0) - Report.RegisterWarningRegion (Location).WarningDisable (Location, code); + Report.RegisterWarningRegion (Location).WarningDisable (Location, code, Report); } return; } if (arg.StartsWith (w_restore)) { int[] codes = ParseNumbers (arg.Substring (w_restore.Length)); - Hashtable w_table = Report.warning_ignore_table; + var w_table = Report.warning_ignore_table; foreach (int code in codes) { - if (w_table != null && w_table.Contains (code)) - Report.Warning (1635, 1, Location, String.Format ("Cannot restore warning `CS{0:0000}' because it was disabled globally", code)); - Report.RegisterWarningRegion (Location).WarningEnable (Location, code); + if (w_table != null && w_table.ContainsKey (code)) + Report.Warning (1635, 1, Location, "Cannot restore warning `CS{0:0000}' because it was disabled globally", code); + Report.RegisterWarningRegion (Location).WarningEnable (Location, code, Report); } return; } @@ -2312,7 +2248,7 @@ namespace Mono.CSharp Error_UnexpectedDirective ("no #region for this #endregion"); return true; } - int pop = (int) ifstack.Pop (); + int pop = ifstack.Pop (); if ((pop & REGION) == 0) Report.Error (1027, Location, "Expected `#endif' directive"); @@ -2321,13 +2257,13 @@ namespace Mono.CSharp case "if": if (ifstack == null) - ifstack = new Stack (2); + ifstack = new Stack (2); int flags = region_directive ? REGION : 0; if (ifstack.Count == 0){ flags |= PARENT_TAKING; } else { - int state = (int) ifstack.Peek (); + int state = ifstack.Peek (); if ((state & TAKING) != 0) { flags |= PARENT_TAKING; } @@ -2345,7 +2281,7 @@ namespace Mono.CSharp Error_UnexpectedDirective ("no #if for this #endif"); return true; } else { - pop = (int) ifstack.Pop (); + pop = ifstack.Pop (); if ((pop & REGION) != 0) Report.Error (1038, Location, "#endregion directive expected"); @@ -2357,7 +2293,7 @@ namespace Mono.CSharp if (ifstack.Count == 0) return true; - int state = (int) ifstack.Peek (); + int state = ifstack.Peek (); return (state & TAKING) != 0; } @@ -2366,7 +2302,7 @@ namespace Mono.CSharp Error_UnexpectedDirective ("no #if for this #elif"); return true; } else { - int state = (int) ifstack.Pop (); + int state = ifstack.Pop (); if ((state & REGION) != 0) { Report.Error (1038, Location, "#endregion directive expected"); @@ -2397,7 +2333,7 @@ namespace Mono.CSharp Error_UnexpectedDirective ("no #if for this #else"); return true; } else { - int state = (int) ifstack.Peek (); + int state = ifstack.Peek (); if ((state & REGION) != 0) { Report.Error (1038, Location, "#endregion directive expected"); @@ -2430,21 +2366,21 @@ namespace Mono.CSharp return ret; } - case "define": - if (any_token_seen){ - Error_TokensSeen (); - return caller_is_taking; - } - PreProcessDefinition (true, arg, caller_is_taking); + case "define": + if (any_token_seen){ + Error_TokensSeen (); return caller_is_taking; + } + PreProcessDefinition (true, arg, caller_is_taking); + return caller_is_taking; - case "undef": - if (any_token_seen){ - Error_TokensSeen (); - return caller_is_taking; - } - PreProcessDefinition (false, arg, caller_is_taking); + case "undef": + if (any_token_seen){ + Error_TokensSeen (); return caller_is_taking; + } + PreProcessDefinition (false, arg, caller_is_taking); + return caller_is_taking; } // @@ -2489,12 +2425,6 @@ namespace Mono.CSharp int c; string_builder.Length = 0; - // - // No need to parse full string when parsing lambda arguments - // - if (lambda_arguments_parsing) - return Token.LITERAL_STRING; - while ((c = get_char ()) != -1){ if (c == '"'){ if (quoted && peek_char () == '"'){ @@ -2502,8 +2432,8 @@ namespace Mono.CSharp get_char (); continue; } else { - val = string_builder.ToString (); - return Token.LITERAL_STRING; + val = new StringLiteral (string_builder.ToString (), Location); + return Token.LITERAL; } } @@ -2535,57 +2465,19 @@ namespace Mono.CSharp if (doc_state == XmlCommentState.Allowed) doc_state = XmlCommentState.NotAllowed; - switch (res) { - case Token.USING: - case Token.NAMESPACE: - check_incorrect_doc_comment (); - break; - } - - if (res == Token.PARTIAL) { - if (parsing_block > 0) { - val = new LocatedToken (Location, "partial"); - return Token.IDENTIFIER; - } - - // Save current position and parse next token. - PushPosition (); - - int next_token = token (); - bool ok = (next_token == Token.CLASS) || - (next_token == Token.STRUCT) || - (next_token == Token.INTERFACE) || - (next_token == Token.VOID); - - PopPosition (); - - if (ok) { - if (next_token == Token.VOID) { - if (RootContext.Version == LanguageVersion.ISO_1 || - RootContext.Version == LanguageVersion.ISO_2) - Report.FeatureIsNotAvailable (Location, "partial methods"); - } else if (RootContext.Version == LanguageVersion.ISO_1) - Report.FeatureIsNotAvailable (Location, "partial types"); - - return res; - } - - if (next_token < Token.LAST_KEYWORD) { - Report.Error (267, Location, - "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword"); - return token (); - } - - val = new LocatedToken (Location, "partial"); - return Token.IDENTIFIER; - } return res; } - private int consume_identifier (int c, bool quoted) + int consume_identifier (int c, bool quoted) { + // + // This method is very performance sensitive. It accounts + // for approximately 25% of all parser time + // + int pos = 0; + int column = col; if (c == '\\') { int surrogate; @@ -2597,34 +2489,44 @@ namespace Mono.CSharp } id_builder [pos++] = (char) c; - current_location = new Location (ref_line, hidden ? -1 : Col); - while ((c = get_char ()) != -1) { - loop: - if (is_identifier_part_character ((char) c)){ - if (pos == max_id_size){ - Report.Error (645, Location, "Identifier too long (limit is 512 chars)"); - return Token.ERROR; + try { + while (true) { + c = reader.Read (); + + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) { + id_builder [pos++] = (char) c; + continue; } - - id_builder [pos++] = (char) c; -// putback_char = -1; - } else if (c == '\\') { - int surrogate; - c = escape (c, out surrogate); - if (surrogate != 0) { - if (is_identifier_part_character ((char) c)) - id_builder [pos++] = (char) c; - c = surrogate; + + if (c < 0x80) { + if (c == '\\') { + int surrogate; + c = escape (c, out surrogate); + if (surrogate != 0) { + if (is_identifier_part_character ((char) c)) + id_builder[pos++] = (char) c; + c = surrogate; + } + + continue; + } + } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) { + id_builder [pos++] = (char) c; + continue; } - goto loop; - } else { -// putback_char = c; - putback (c); + + putback_char = c; break; } + } catch (IndexOutOfRangeException) { + Report.Error (645, Location, "Identifier too long (limit is 512 chars)"); + col += pos - 1; + return Token.ERROR; } + col += pos - 1; + // // Optimization: avoids doing the keyword lookup // on uppercase letters @@ -2632,7 +2534,7 @@ namespace Mono.CSharp if (id_builder [0] >= '_' && !quoted) { int keyword = GetKeyword (id_builder, pos); if (keyword != -1) { - val = Location; + val = LocatedToken.Create (null, ref_line, column); return keyword; } } @@ -2641,60 +2543,68 @@ namespace Mono.CSharp // Keep identifiers in an array of hashtables to avoid needless // allocations // - - if (identifiers [pos] != null) { - val = identifiers [pos][id_builder]; - if (val != null) { - val = new LocatedToken (Location, (string) val); + var identifiers_group = identifiers [pos]; + string s; + if (identifiers_group != null) { + if (identifiers_group.TryGetValue (id_builder, out s)) { + val = LocatedToken.Create (s, ref_line, column); if (quoted) - escaped_identifiers.Add (val); + AddEscapedIdentifier (((LocatedToken) val).Location); return Token.IDENTIFIER; } - } - else - identifiers [pos] = new CharArrayHashtable (pos); - - val = new String (id_builder, 0, pos); - if (RootContext.Version == LanguageVersion.ISO_1) { - for (int i = 1; i < id_builder.Length; i += 3) { - if (id_builder [i] == '_' && (id_builder [i - 1] == '_' || id_builder [i + 1] == '_')) { - Report.Error (1638, Location, - "`{0}': Any identifier with double underscores cannot be used when ISO language version mode is specified", val.ToString ()); - break; - } - } + } else { + // TODO: this should be number of files dependant + // corlib compilation peaks at 1000 and System.Core at 150 + int capacity = pos > 20 ? 10 : 100; + identifiers_group = new Dictionary (capacity, new IdentifiersComparer (pos)); + identifiers [pos] = identifiers_group; } char [] chars = new char [pos]; Array.Copy (id_builder, chars, pos); - identifiers [pos] [chars] = val; + s = new string (id_builder, 0, pos); + identifiers_group.Add (chars, s); - val = new LocatedToken (Location, (string) val); + val = LocatedToken.Create (s, ref_line, column); if (quoted) - escaped_identifiers.Add (val); + AddEscapedIdentifier (((LocatedToken) val).Location); + return Token.IDENTIFIER; } public int xtoken () { - int t; - bool doread = false; - int c; + int d, c; // Whether we have seen comments on the current line bool comments_seen = false; - val = null; - for (;(c = get_char ()) != -1;) { - if (c == '\t'){ - col = ((col + 8) / 8) * 8; + while ((c = get_char ()) != -1) { + switch (c) { + case '\t': + col = ((col + tab_size) / tab_size) * tab_size; continue; - } - - if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0 || c == 0) + + case ' ': + case '\f': + case '\v': + case 0xa0: + case 0: + case 0xFEFF: // Ignore BOM anywhere in the file continue; - if (c == '\r') { +/* This is required for compatibility with .NET + case 0xEF: + if (peek_char () == 0xBB) { + PushPosition (); + get_char (); + if (get_char () == 0xBF) + continue; + PopPosition (); + } + break; +*/ + case '\r': if (peek_char () != '\n') advance_line (); else @@ -2704,12 +2614,213 @@ namespace Mono.CSharp tokens_seen = false; comments_seen = false; continue; - } - // Handle double-slash comments. - if (c == '/'){ - int d = peek_char (); - + case '\\': + tokens_seen = true; + return consume_identifier (c); + + case '{': + val = LocatedToken.Create (ref_line, col); + return Token.OPEN_BRACE; + case '}': + val = LocatedToken.Create (ref_line, col); + return Token.CLOSE_BRACE; + case '[': + // To block doccomment inside attribute declaration. + if (doc_state == XmlCommentState.Allowed) + doc_state = XmlCommentState.NotAllowed; + val = LocatedToken.CreateOptional (ref_line, col); + return Token.OPEN_BRACKET; + case ']': + val = LocatedToken.CreateOptional (ref_line, col); + return Token.CLOSE_BRACKET; + case '(': + val = LocatedToken.Create (ref_line, col); + // + // An expression versions of parens can appear in block context only + // + if (parsing_block != 0 && !lambda_arguments_parsing) { + + // + // Optmize most common case where we know that parens + // is not special + // + switch (current_token) { + case Token.IDENTIFIER: + case Token.IF: + case Token.FOR: + case Token.FOREACH: + case Token.TYPEOF: + case Token.WHILE: + case Token.USING: + case Token.DEFAULT: + case Token.DELEGATE: + case Token.OP_GENERICS_GT: + return Token.OPEN_PARENS; + } + + // Optimize using peek + int xx = peek_char (); + switch (xx) { + case '(': + case '\'': + case '"': + case '0': + case '1': + return Token.OPEN_PARENS; + } + + lambda_arguments_parsing = true; + PushPosition (); + d = TokenizeOpenParens (); + PopPosition (); + lambda_arguments_parsing = false; + return d; + } + + return Token.OPEN_PARENS; + case ')': + val = LocatedToken.CreateOptional (ref_line, col); + return Token.CLOSE_PARENS; + case ',': + val = LocatedToken.CreateOptional (ref_line, col); + return Token.COMMA; + case ';': + val = LocatedToken.CreateOptional (ref_line, col); + return Token.SEMICOLON; + case '~': + val = LocatedToken.Create (ref_line, col); + return Token.TILDE; + case '?': + val = LocatedToken.Create (ref_line, col); + return TokenizePossibleNullableType (); + case '<': + val = LocatedToken.Create (ref_line, col); + if (parsing_generic_less_than++ > 0) + return Token.OP_GENERICS_LT; + + return TokenizeLessThan (); + + case '>': + val = LocatedToken.Create (ref_line, col); + d = peek_char (); + + if (d == '='){ + get_char (); + return Token.OP_GE; + } + + if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) { + parsing_generic_less_than--; + return Token.OP_GENERICS_GT; + } + + if (d == '>') { + get_char (); + d = peek_char (); + + if (d == '=') { + get_char (); + return Token.OP_SHIFT_RIGHT_ASSIGN; + } + return Token.OP_SHIFT_RIGHT; + } + + return Token.OP_GT; + + case '+': + val = LocatedToken.Create (ref_line, col); + d = peek_char (); + if (d == '+') { + d = Token.OP_INC; + } else if (d == '=') { + d = Token.OP_ADD_ASSIGN; + } else { + return Token.PLUS; + } + get_char (); + return d; + + case '-': + val = LocatedToken.Create (ref_line, col); + d = peek_char (); + if (d == '-') { + d = Token.OP_DEC; + } else if (d == '=') + d = Token.OP_SUB_ASSIGN; + else if (d == '>') + d = Token.OP_PTR; + else { + return Token.MINUS; + } + get_char (); + return d; + + case '!': + val = LocatedToken.Create (ref_line, col); + if (peek_char () == '='){ + get_char (); + return Token.OP_NE; + } + return Token.BANG; + + case '=': + val = LocatedToken.Create (ref_line, col); + d = peek_char (); + if (d == '='){ + get_char (); + return Token.OP_EQ; + } + if (d == '>'){ + get_char (); + return Token.ARROW; + } + + return Token.ASSIGN; + + case '&': + val = LocatedToken.Create (ref_line, col); + d = peek_char (); + if (d == '&'){ + get_char (); + return Token.OP_AND; + } + if (d == '='){ + get_char (); + return Token.OP_AND_ASSIGN; + } + return Token.BITWISE_AND; + + case '|': + val = LocatedToken.Create (ref_line, col); + d = peek_char (); + if (d == '|'){ + get_char (); + return Token.OP_OR; + } + if (d == '='){ + get_char (); + return Token.OP_OR_ASSIGN; + } + return Token.BITWISE_OR; + + case '*': + val = LocatedToken.Create (ref_line, col); + if (peek_char () == '='){ + get_char (); + return Token.OP_MULT_ASSIGN; + } + return Token.STAR; + + case '/': + d = peek_char (); + if (d == '='){ + val = LocatedToken.Create (ref_line, col); + get_char (); + return Token.OP_DIV_ASSIGN; + } + + // Handle double-slash comments. if (d == '/'){ get_char (); if (RootContext.Documentation != null && peek_char () == '/') { @@ -2752,8 +2863,6 @@ namespace Mono.CSharp xml_comment_buffer.Append (Environment.NewLine); } - Location start_location = Location; - while ((d = get_char ()) != -1){ if (d == '*' && peek_char () == '/'){ get_char (); @@ -2774,53 +2883,57 @@ namespace Mono.CSharp } } if (!comments_seen) - Report.Error (1035, start_location, "End-of-file found, '*/' expected"); + Report.Error (1035, Location, "End-of-file found, '*/' expected"); if (docAppend) update_formatted_doc_comment (current_comment_start); continue; } - goto is_punct_label; - } + return Token.DIV; - - if (c == '\\' || is_identifier_start_character ((char)c)){ - tokens_seen = true; - return consume_identifier (c); - } + case '%': + val = LocatedToken.Create (ref_line, col); + if (peek_char () == '='){ + get_char (); + return Token.OP_MOD_ASSIGN; + } + return Token.PERCENT; - is_punct_label: - current_location = new Location (ref_line, hidden ? -1 : Col); - if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){ - tokens_seen = true; - if (doread){ + case '^': + val = LocatedToken.Create (ref_line, col); + if (peek_char () == '='){ get_char (); + return Token.OP_XOR_ASSIGN; } - return t; - } + return Token.CARRET; - // white space - if (c == '\n'){ + case ':': + val = LocatedToken.Create (ref_line, col); + if (peek_char () == ':') { + get_char (); + return Token.DOUBLE_COLON; + } + return Token.COLON; + + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + tokens_seen = true; + return is_number (c); + + case '\n': // white space any_token_seen |= tokens_seen; tokens_seen = false; comments_seen = false; continue; - } - - if (c >= '0' && c <= '9'){ - tokens_seen = true; - return is_number (c); - } - if (c == '.'){ + case '.': tokens_seen = true; - int peek = peek_char (); - if (peek >= '0' && peek <= '9') + d = peek_char (); + if (d >= '0' && d <= '9') return is_number (c); return Token.DOT; - } - if (c == '#') { + case '#': if (tokens_seen || comments_seen) { Eror_WrongPreprocessorLocation (); return Token.ERROR; @@ -2858,78 +2971,141 @@ namespace Mono.CSharp } return Token.EOF; - } - if (c == '"') + case '"': return consume_string (false); - if (c == '\''){ - c = get_char (); - tokens_seen = true; - if (c == '\''){ - error_details = "Empty character literal"; - Report.Error (1011, Location, error_details); - return Token.ERROR; - } - if (c == '\r' || c == '\n') { - Report.Error (1010, Location, "Newline in constant"); - return Token.ERROR; - } - - int surrogate; - c = escape (c, out surrogate); - if (c == -1) - return Token.ERROR; - if (surrogate != 0) - throw new NotImplementedException (); - - val = (char) c; - c = get_char (); - - if (c != '\''){ - error_details = "Too many characters in character literal"; - Report.Error (1012, Location, error_details); - - // Try to recover, read until newline or next "'" - while ((c = get_char ()) != -1){ - if (c == '\n'){ - break; - } - else if (c == '\'') - break; - } - return Token.ERROR; - } - return Token.LITERAL_CHARACTER; - } + case '\'': + return TokenizeBackslash (); - if (c == '@') { + case '@': c = get_char (); if (c == '"') { tokens_seen = true; return consume_string (true); - } else if (is_identifier_start_character ((char) c)){ + } + + if (is_identifier_start_character (c)){ return consume_identifier (c, true); - } else { - Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @"); } - } - if (c == EvalStatementParserCharacter) + Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @"); + return Token.ERROR; + + case EvalStatementParserCharacter: return Token.EVAL_STATEMENT_PARSER; - if (c == EvalCompilationUnitParserCharacter) + case EvalCompilationUnitParserCharacter: return Token.EVAL_COMPILATION_UNIT_PARSER; - if (c == EvalUsingDeclarationsParserCharacter) + case EvalUsingDeclarationsParserCharacter: return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER; - + } + + if (is_identifier_start_character (c)) { + tokens_seen = true; + return consume_identifier (c); + } + error_details = ((char)c).ToString (); - return Token.ERROR; } + if (CompleteOnEOF){ + if (generated) + return Token.COMPLETE_COMPLETION; + + generated = true; + return Token.GENERATE_COMPLETION; + } + + return Token.EOF; } + int TokenizeBackslash () + { + int c = get_char (); + tokens_seen = true; + if (c == '\'') { + error_details = "Empty character literal"; + Report.Error (1011, Location, error_details); + return Token.ERROR; + } + if (c == '\r' || c == '\n') { + Report.Error (1010, Location, "Newline in constant"); + return Token.ERROR; + } + + int d; + c = escape (c, out d); + if (c == -1) + return Token.ERROR; + if (d != 0) + throw new NotImplementedException (); + + val = new CharLiteral ((char) c, Location); + c = get_char (); + + if (c != '\'') { + Report.Error (1012, Location, "Too many characters in character literal"); + + // Try to recover, read until newline or next "'" + while ((c = get_char ()) != -1) { + if (c == '\n' || c == '\'') + break; + } + return Token.ERROR; + } + + return Token.LITERAL; + } + + int TokenizeLessThan () + { + int d; + if (handle_typeof) { + PushPosition (); + if (parse_generic_dimension (out d)) { + val = d; + DiscardPosition (); + return Token.GENERIC_DIMENSION; + } + PopPosition (); + } + + // Save current position and parse next token. + PushPosition (); + if (parse_less_than ()) { + if (parsing_generic_declaration && token () != Token.DOT) { + d = Token.OP_GENERICS_LT_DECL; + } else { + d = Token.OP_GENERICS_LT; + } + PopPosition (); + return d; + } + + PopPosition (); + parsing_generic_less_than = 0; + + d = peek_char (); + if (d == '<') { + get_char (); + d = peek_char (); + + if (d == '=') { + get_char (); + return Token.OP_SHIFT_LEFT_ASSIGN; + } + return Token.OP_SHIFT_LEFT; + } + + if (d == '=') { + get_char (); + return Token.OP_LE; + } + return Token.OP_LT; + } + // // Handles one line xml comment // @@ -3027,6 +3203,10 @@ namespace Mono.CSharp return null; } + Report Report { + get { return context.Report; } + } + void reset_doc_comment () { xml_comment_buffer.Length = 0; @@ -3036,8 +3216,7 @@ namespace Mono.CSharp public void cleanup () { if (ifstack != null && ifstack.Count >= 1) { - current_location = new Location (ref_line, hidden ? -1 : Col); - int state = (int) ifstack.Pop (); + int state = ifstack.Pop (); if ((state & REGION) != 0) Report.Error (1038, Location, "#endregion directive expected"); else