X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mcs%2Fmcs%2Fcs-tokenizer.cs;h=7475a3055fabcd00e8a69901c526ec90f8046212;hb=e06a9eb8b5f4cc4f1d3a20e6edec14b26509da5f;hp=6bec5b51d51e0e17506b43764136c0139caee2c6;hpb=f78e6f8fee273c6c80c8c36e7e1b2bbd8392b8cb;p=mono.git diff --git a/mcs/mcs/cs-tokenizer.cs b/mcs/mcs/cs-tokenizer.cs index 6bec5b51d51..7475a3055fa 100644 --- a/mcs/mcs/cs-tokenizer.cs +++ b/mcs/mcs/cs-tokenizer.cs @@ -17,6 +17,7 @@ using System.Text; using System.Collections.Generic; using System.Globalization; using System.Diagnostics; +using System.Collections; namespace Mono.CSharp { @@ -68,32 +69,62 @@ namespace Mono.CSharp } // - // This class has to be used in the parser only, it reuses token - // details after each parse + // This class has to be used by parser only, it reuses token + // details after each file parse completion // public class LocatedToken { - int row, column; - string value; + public int row, column; + public string value; + public SourceFile file; - static LocatedToken[] buffer = new LocatedToken[0]; - static int pos; - - private LocatedToken () + public LocatedToken () { } - public static LocatedToken Create (int row, int column) + public LocatedToken (string value, Location loc) { - return Create (null, row, column); + this.value = value; + file = loc.SourceFile; + row = loc.Row; + column = loc.Column; } - public static LocatedToken Create (string value, Location loc) + public override string ToString () { - return Create (value, loc.Row, loc.Column); + return string.Format ("Token '{0}' at {1},{2}", Value, row, column); } - public static LocatedToken Create (string value, int row, int column) + public Location Location { + get { return new Location (file, row, column); } + } + + public string Value { + get { return value; } + } + } + + public class LocatedTokenBuffer + { + readonly LocatedToken[] buffer; + public int pos; + + public LocatedTokenBuffer () + { + buffer = new LocatedToken[0]; + } + + public LocatedTokenBuffer (LocatedToken[] buffer) + { + this.buffer = buffer ?? new LocatedToken[0]; + } + + public LocatedToken Create (SourceFile file, int row, int column) + { + return Create (null, file, row, column); + } + + public LocatedToken Create (string value, SourceFile file, int row, int column) { // // TODO: I am not very happy about the logic but it's the best @@ -107,15 +138,16 @@ namespace Mono.CSharp if (pos >= buffer.Length) { entry = new LocatedToken (); } else { - entry = buffer [pos]; + entry = buffer[pos]; if (entry == null) { entry = new LocatedToken (); - buffer [pos] = entry; + buffer[pos] = entry; } ++pos; } entry.value = value; + entry.file = file; entry.row = row; entry.column = column; return entry; @@ -125,31 +157,9 @@ namespace Mono.CSharp // Used for token not required by expression evaluator // [Conditional ("FULL_AST")] - public static void CreateOptional (int row, int col, ref object token) + public void CreateOptional (SourceFile file, int row, int col, ref object token) { - token = Create (row, col); - } - - public static void Initialize () - { -#if !FULL_AST - if (buffer.Length == 0) - buffer = new LocatedToken [15000]; -#endif - pos = 0; - } - - public override string ToString () - { - return string.Format ("Token '{0}' at {1},{2}", Value, row, column); - } - - public Location Location { - get { return new Location (row, column); } - } - - public string Value { - get { return value; } + token = Create (file, row, col); } } @@ -194,6 +204,7 @@ namespace Mono.CSharp List escaped_identifiers; int parsing_generic_less_than; readonly bool doc_processing; + readonly LocatedTokenBuffer ltb; // // Used mainly for parser optimizations. Some expressions for instance @@ -356,28 +367,15 @@ namespace Mono.CSharp // Stack ifstack; - const int max_id_size = 512; - const int max_number_size = 512; - -#if FULL_AST - readonly char [] id_builder = new char [max_id_size]; - - Dictionary[] identifiers = new Dictionary[max_id_size + 1]; + public const int MaxIdentifierLength = 512; + public const int MaxNumberLength = 512; - char [] number_builder = new char [max_number_size]; + readonly char[] id_builder; + readonly Dictionary[] identifiers; + readonly char[] number_builder; int number_pos; - char[] value_builder = new char[256]; -#else - static readonly char [] id_builder = new char [max_id_size]; - - static Dictionary[] identifiers = new Dictionary[max_id_size + 1]; - - static char [] number_builder = new char [max_number_size]; - static int number_pos; - - static char[] value_builder = new char[256]; -#endif + char[] value_builder = new char[64]; public int Line { get { @@ -428,11 +426,15 @@ namespace Mono.CSharp } } - public Tokenizer (SeekableStreamReader input, CompilationSourceFile file) + public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, ParserSession session) { this.source_file = file; this.context = file.Compiler; this.current_source = file.SourceFile; + this.identifiers = session.Identifiers; + this.id_builder = session.IDBuilder; + this.number_builder = session.NumberBuilder; + this.ltb = new LocatedTokenBuffer (session.LocatedTokens); reader = input; @@ -442,8 +444,6 @@ namespace Mono.CSharp doc_processing = context.Settings.DocumentationFile != null; tab_size = context.Settings.TabSize; - - Mono.CSharp.Location.Push (current_source); } public void PushPosition () @@ -827,11 +827,13 @@ namespace Mono.CSharp PushPosition (); xtoken (); if (xtoken () != Token.ARROW) - res = -1; + goto default; PopPosition (); break; default: + // peek_token could overwrite id_buffer + id_builder [0] = 'a'; id_builder [1] = 's'; id_builder [2] = 'y'; id_builder [3] = 'n'; id_builder [4] = 'c'; res = -1; break; } @@ -890,7 +892,7 @@ namespace Mono.CSharp public Location Location { get { - return new Location (ref_line, col); + return new Location (current_source, ref_line, col); } } @@ -1103,9 +1105,14 @@ namespace Mono.CSharp start: int the_token = token (); if (the_token == Token.OPEN_BRACKET) { - do { + while (true) { the_token = token (); - } while (the_token != Token.CLOSE_BRACKET); + if (the_token == Token.EOF) + return true; + + if (the_token == Token.CLOSE_BRACKET) + break; + } the_token = token (); } else if (the_token == Token.IN || the_token == Token.OUT) { the_token = token (); @@ -1318,7 +1325,7 @@ namespace Mono.CSharp bool seen_digits = false; if (c != -1){ - if (number_pos == max_number_size) + if (number_pos == MaxNumberLength) Error_NumericConstantTooLong (); number_builder [number_pos++] = (char) c; } @@ -1329,7 +1336,7 @@ namespace Mono.CSharp // while ((d = peek_char2 ()) != -1){ if (d >= '0' && d <= '9'){ - if (number_pos == max_number_size) + if (number_pos == MaxNumberLength) Error_NumericConstantTooLong (); number_builder [number_pos++] = (char) d; get_char (); @@ -1594,23 +1601,23 @@ namespace Mono.CSharp if (c == 'e' || c == 'E'){ is_real = true; - if (number_pos == max_number_size) + if (number_pos == MaxNumberLength) Error_NumericConstantTooLong (); number_builder [number_pos++] = (char) c; c = get_char (); if (c == '+'){ - if (number_pos == max_number_size) + if (number_pos == MaxNumberLength) Error_NumericConstantTooLong (); number_builder [number_pos++] = '+'; c = -1; } else if (c == '-') { - if (number_pos == max_number_size) + if (number_pos == MaxNumberLength) Error_NumericConstantTooLong (); number_builder [number_pos++] = '-'; c = -1; } else { - if (number_pos == max_number_size) + if (number_pos == MaxNumberLength) Error_NumericConstantTooLong (); number_builder [number_pos++] = '+'; } @@ -1943,7 +1950,7 @@ namespace Mono.CSharp } if (pos != 0) { - if (pos > max_id_size) + if (pos > MaxIdentifierLength) arg = new string (value_builder, 0, pos); else arg = InternIdentifier (value_builder, pos); @@ -1976,7 +1983,6 @@ namespace Mono.CSharp } ref_line = line; - Location.Push (current_source); return true; } @@ -2063,7 +2069,6 @@ namespace Mono.CSharp if (new_file_name != null) { current_source = context.LookupFile (source_file, new_file_name); source_file.AddIncludeFile (current_source); - Location.Push (current_source); } if (!hidden_block_start.IsNull) { @@ -2161,6 +2166,8 @@ namespace Mono.CSharp // // The syntax is ` "foo.txt" "{guid}" "hash"' // + // guid is predefined hash algorithm guid {406ea660-64cf-4c82-b6f0-42d48172a799} for md5 + // int c = get_char (); if (c != '"') @@ -2223,6 +2230,7 @@ namespace Mono.CSharp // Any length of checksum List checksum_bytes = new List (16); + var checksum_location = Location; c = peek_char (); while (c != '"' && c != -1) { checksum_bytes.Add (read_hex (out error)); @@ -2238,14 +2246,23 @@ namespace Mono.CSharp return false; } - file.SetChecksum (guid_bytes, checksum_bytes.ToArray ()); - current_source.AutoGenerated = true; + if (context.Settings.GenerateDebugInfo) { + var chsum = checksum_bytes.ToArray (); + + if (file.HasChecksum) { + if (!ArrayComparer.IsEqual (file.Checksum, chsum)) { + // TODO: Report.SymbolRelatedToPreviousError + Report.Warning (1697, 1, checksum_location, "Different checksum values specified for file `{0}'", file.Name); + } + } + + file.SetChecksum (guid_bytes, chsum); + current_source.AutoGenerated = true; + } + return true; } -#if !FULL_AST - static -#endif bool IsTokenIdentifierEqual (char[] identifier) { for (int i = 0; i < identifier.Length; ++i) { @@ -2979,22 +2996,19 @@ namespace Mono.CSharp if (id_builder [0] >= '_' && !quoted) { int keyword = GetKeyword (id_builder, pos); if (keyword != -1) { - val = LocatedToken.Create (keyword == Token.AWAIT ? "await" : null, ref_line, column); + val = ltb.Create (keyword == Token.AWAIT ? "await" : null, current_source, ref_line, column); return keyword; } } string s = InternIdentifier (id_builder, pos); - val = LocatedToken.Create (s, ref_line, column); + val = ltb.Create (s, current_source, ref_line, column); if (quoted && parsing_attribute_section) AddEscapedIdentifier (((LocatedToken) val).Location); return Token.IDENTIFIER; } -#if !FULL_AST - static -#endif string InternIdentifier (char[] charBuffer, int length) { // @@ -3059,17 +3073,17 @@ namespace Mono.CSharp return consume_identifier (c); case '{': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); return Token.OPEN_BRACE; case '}': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); return Token.CLOSE_BRACE; case '[': // To block doccomment inside attribute declaration. if (doc_state == XmlCommentState.Allowed) doc_state = XmlCommentState.NotAllowed; - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); if (parsing_block == 0 || lambda_arguments_parsing) return Token.OPEN_BRACKET; @@ -3095,10 +3109,10 @@ namespace Mono.CSharp return Token.OPEN_BRACKET_EXPR; } case ']': - LocatedToken.CreateOptional (ref_line, col, ref val); + ltb.CreateOptional (current_source, ref_line, col, ref val); return Token.CLOSE_BRACKET; case '(': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); // // An expression versions of parens can appear in block context only // @@ -3143,29 +3157,29 @@ namespace Mono.CSharp return Token.OPEN_PARENS; case ')': - LocatedToken.CreateOptional (ref_line, col, ref val); + ltb.CreateOptional (current_source, ref_line, col, ref val); return Token.CLOSE_PARENS; case ',': - LocatedToken.CreateOptional (ref_line, col, ref val); + ltb.CreateOptional (current_source, ref_line, col, ref val); return Token.COMMA; case ';': - LocatedToken.CreateOptional (ref_line, col, ref val); + ltb.CreateOptional (current_source, ref_line, col, ref val); return Token.SEMICOLON; case '~': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); return Token.TILDE; case '?': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); return TokenizePossibleNullableType (); case '<': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); if (parsing_generic_less_than++ > 0) return Token.OP_GENERICS_LT; return TokenizeLessThan (); case '>': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); d = peek_char (); if (d == '='){ @@ -3192,7 +3206,7 @@ namespace Mono.CSharp return Token.OP_GT; case '+': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); d = peek_char (); if (d == '+') { d = Token.OP_INC; @@ -3205,7 +3219,7 @@ namespace Mono.CSharp return d; case '-': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); d = peek_char (); if (d == '-') { d = Token.OP_DEC; @@ -3220,7 +3234,7 @@ namespace Mono.CSharp return d; case '!': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); if (peek_char () == '='){ get_char (); return Token.OP_NE; @@ -3228,7 +3242,7 @@ namespace Mono.CSharp return Token.BANG; case '=': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); d = peek_char (); if (d == '='){ get_char (); @@ -3242,7 +3256,7 @@ namespace Mono.CSharp return Token.ASSIGN; case '&': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); d = peek_char (); if (d == '&'){ get_char (); @@ -3255,7 +3269,7 @@ namespace Mono.CSharp return Token.BITWISE_AND; case '|': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); d = peek_char (); if (d == '|'){ get_char (); @@ -3268,7 +3282,7 @@ namespace Mono.CSharp return Token.BITWISE_OR; case '*': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); if (peek_char () == '='){ get_char (); return Token.OP_MULT_ASSIGN; @@ -3278,7 +3292,7 @@ namespace Mono.CSharp case '/': d = peek_char (); if (d == '='){ - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); get_char (); return Token.OP_DIV_ASSIGN; } @@ -3357,11 +3371,11 @@ namespace Mono.CSharp update_formatted_doc_comment (current_comment_start); continue; } - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); return Token.DIV; case '%': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); if (peek_char () == '='){ get_char (); return Token.OP_MOD_ASSIGN; @@ -3369,7 +3383,7 @@ namespace Mono.CSharp return Token.PERCENT; case '^': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); if (peek_char () == '='){ get_char (); return Token.OP_XOR_ASSIGN; @@ -3377,7 +3391,7 @@ namespace Mono.CSharp return Token.CARRET; case ':': - val = LocatedToken.Create (ref_line, col); + val = ltb.Create (current_source, ref_line, col); if (peek_char () == ':') { get_char (); return Token.DOUBLE_COLON; @@ -3401,7 +3415,7 @@ namespace Mono.CSharp if (d >= '0' && d <= '9') return is_number (c); - LocatedToken.CreateOptional (ref_line, col, ref val); + ltb.CreateOptional (current_source, ref_line, col, ref val); return Token.DOT; case '#':