X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mcs%2Fmcs%2Fcs-tokenizer.cs;h=2dccc46414da9f88d981f042d186a30fb6433351;hb=8e33e0d088c564d039dd791b7cc3e121ff87b3e4;hp=d2cd6fe55411c274188b582d9bd73a24dc532ee9;hpb=282c5e6f6751f042db2e4a66eaaf0b73962ecf20;p=mono.git diff --git a/mcs/mcs/cs-tokenizer.cs b/mcs/mcs/cs-tokenizer.cs index d2cd6fe5541..2dccc46414d 100644 --- a/mcs/mcs/cs-tokenizer.cs +++ b/mcs/mcs/cs-tokenizer.cs @@ -3,19 +3,18 @@ // This also implements the preprocessor // // Author: Miguel de Icaza (miguel@gnu.org) -// Marek Safar (marek.safar@seznam.cz) +// Marek Safar (marek.safar@gmail.com) // // Dual licensed under the terms of the MIT X11 or GNU GPL // // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com) // Copyright 2004-2008 Novell, Inc -// +// Copyright 2011 Xamarin, Inc (http://www.xamarin.com) // using System; using System.Text; using System.Collections.Generic; -using System.IO; using System.Globalization; using System.Diagnostics; @@ -88,6 +87,11 @@ namespace Mono.CSharp { return Create (null, row, column); } + + public static LocatedToken Create (string value, Location loc) + { + return Create (value, loc.Row, loc.Column); + } public static LocatedToken Create (string value, int row, int column) { @@ -142,7 +146,7 @@ namespace Mono.CSharp } } - enum PreprocessorDirective + public enum PreprocessorDirective { Invalid = 0, @@ -165,7 +169,7 @@ namespace Mono.CSharp SeekableStreamReader reader; SourceFile ref_name; - CompilationUnit file_name; + CompilationSourceFile file_name; CompilerContext context; bool hidden = false; int ref_line = 1; @@ -179,9 +183,9 @@ namespace Mono.CSharp bool handle_where = false; bool handle_typeof = false; bool lambda_arguments_parsing; - Location current_comment_location = Location.Null; List escaped_identifiers; int parsing_generic_less_than; + readonly bool doc_processing; // // Used mainly for parser optimizations. Some expressions for instance @@ -200,6 +204,7 @@ namespace Mono.CSharp // Set when parsing generic declaration (type or method header) // public bool parsing_generic_declaration; + public bool parsing_generic_declaration_doc; // // The value indicates that we have not reach any declaration or @@ -207,19 +212,24 @@ namespace Mono.CSharp // public int parsing_declaration; + public bool parsing_attribute_section; + + public bool parsing_modifiers; + // - // The special character to inject on streams to trigger the EXPRESSION_PARSE - // token to be returned. It just happens to be a Unicode character that - // would never be part of a program (can not be an identifier). + // The special characters to inject on streams to run the unit parser + // in the special expression mode. Using private characters from + // Plane Sixteen (U+100000 to U+10FFFD) // // This character is only tested just before the tokenizer is about to report // an error; So on the regular operation mode, this addition will have no // impact on the tokenizer's performance. // - public const int EvalStatementParserCharacter = 0x2190; // Unicode Left Arrow - public const int EvalCompilationUnitParserCharacter = 0x2191; // Unicode Arrow - public const int EvalUsingDeclarationsParserCharacter = 0x2192; // Unicode Arrow + public const int EvalStatementParserCharacter = 0x100000; + public const int EvalCompilationUnitParserCharacter = 0x100001; + public const int EvalUsingDeclarationsParserCharacter = 0x100002; + public const int DocumentationXref = 0x100003; // // XML documentation buffer. The save point is used to divide @@ -250,7 +260,22 @@ namespace Mono.CSharp // This is needed because `define' is not allowed to be used // after a token has been seen. // - bool any_token_seen = false; + bool any_token_seen; + + // + // Class variables + // + static readonly KeywordEntry[][] keywords; + static readonly KeywordEntry[][] keywords_preprocessor; + static readonly Dictionary keyword_strings; // TODO: HashSet + static readonly NumberStyles styles; + static readonly NumberFormatInfo csharp_format_info; + + // Pragma arguments + static readonly char[] pragma_warning = "warning".ToCharArray (); + static readonly char[] pragma_warning_disable = "disable".ToCharArray (); + static readonly char[] pragma_warning_restore = "restore".ToCharArray (); + static readonly char[] pragma_checksum = "checksum".ToCharArray (); static readonly char[] simple_whitespaces = new char[] { ' ', '\t' }; @@ -302,26 +327,11 @@ namespace Mono.CSharp escaped_identifiers.Add (loc); } - public bool IsEscapedIdentifier (MemberName name) + public bool IsEscapedIdentifier (ATypeNameExpression name) { return escaped_identifiers != null && escaped_identifiers.Contains (name.Location); } - // - // Class variables - // - static KeywordEntry[][] keywords; - static KeywordEntry[][] keywords_preprocessor; - static Dictionary keyword_strings; // TODO: HashSet - static NumberStyles styles; - static NumberFormatInfo csharp_format_info; - - // Pragma arguments - static readonly char[] pragma_warning = "warning".ToCharArray (); - static readonly char[] pragma_warning_disable = "disable".ToCharArray (); - static readonly char[] pragma_warning_restore = "restore".ToCharArray (); - static readonly char[] pragma_checksum = "checksum".ToCharArray (); - // // Values for the associated token returned // @@ -344,7 +354,7 @@ namespace Mono.CSharp static System.Text.StringBuilder string_builder; const int max_id_size = 512; - static char [] id_builder = new char [max_id_size]; + static readonly char [] id_builder = new char [max_id_size]; public static Dictionary[] identifiers = new Dictionary[max_id_size + 1]; @@ -352,8 +362,8 @@ namespace Mono.CSharp static char [] number_builder = new char [max_number_size]; static int number_pos; - static StringBuilder static_cmd_arg = new System.Text.StringBuilder (); - + static char[] value_builder = new char[256]; + public int Line { get { return ref_line; @@ -402,6 +412,26 @@ namespace Mono.CSharp val = t.val; } } + + public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, CompilerContext ctx) + { + this.ref_name = file; + this.file_name = file; + this.context = ctx; + reader = input; + + putback_char = -1; + + xml_comment_buffer = new StringBuilder (); + doc_processing = ctx.Settings.DocumentationFile != null; + + if (Environment.OSVersion.Platform == PlatformID.Win32NT) + tab_size = 4; + else + tab_size = 8; + + Mono.CSharp.Location.Push (file, file); + } public void PushPosition () { @@ -464,14 +494,20 @@ namespace Mono.CSharp kwe.Next = new KeywordEntry (kw, token); } - static void InitTokens () + // + // Class initializer + // + static Tokenizer () { keyword_strings = new Dictionary (); // 11 is the length of the longest keyword for now - keywords = new KeywordEntry [11] []; + keywords = new KeywordEntry[11][]; AddKeyword ("__arglist", Token.ARGLIST); + AddKeyword ("__makeref", Token.MAKEREF); + AddKeyword ("__reftype", Token.REFTYPE); + AddKeyword ("__refvalue", Token.REFVALUE); AddKeyword ("abstract", Token.ABSTRACT); AddKeyword ("as", Token.AS); AddKeyword ("add", Token.ADD); @@ -555,7 +591,6 @@ namespace Mono.CSharp AddKeyword ("while", Token.WHILE); AddKeyword ("partial", Token.PARTIAL); AddKeyword ("where", Token.WHERE); - AddKeyword ("async", Token.ASYNC); // LINQ keywords AddKeyword ("from", Token.FROM); @@ -571,6 +606,10 @@ namespace Mono.CSharp AddKeyword ("descending", Token.DESCENDING); AddKeyword ("into", Token.INTO); + // Contextual async keywords + AddKeyword ("async", Token.ASYNC); + AddKeyword ("await", Token.AWAIT); + keywords_preprocessor = new KeywordEntry[10][]; AddPreprocessorKeyword ("region", PreprocessorDirective.Region); @@ -585,14 +624,7 @@ namespace Mono.CSharp AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning); AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma); AddPreprocessorKeyword ("line", PreprocessorDirective.Line); - } - // - // Class initializer - // - static Tokenizer () - { - InitTokens (); csharp_format_info = NumberFormatInfo.InvariantInfo; styles = NumberStyles.Float; @@ -691,8 +723,8 @@ namespace Mono.CSharp res = Token.FROM_FIRST; query_parsing = true; - if (RootContext.Version <= LanguageVersion.ISO_2) - Report.FeatureIsNotAvailable (Location, "query expressions"); + if (context.Settings.Version <= LanguageVersion.ISO_2) + Report.FeatureIsNotAvailable (context, Location, "query expressions"); break; case Token.VOID: Expression.Error_VoidInvalidInTheContext (Location, Report); @@ -747,11 +779,10 @@ namespace Mono.CSharp if (ok) { if (next_token == Token.VOID) { - if (RootContext.Version == LanguageVersion.ISO_1 || - RootContext.Version == LanguageVersion.ISO_2) - Report.FeatureIsNotAvailable (Location, "partial methods"); - } else if (RootContext.Version == LanguageVersion.ISO_1) - Report.FeatureIsNotAvailable (Location, "partial types"); + if (context.Settings.Version <= LanguageVersion.ISO_2) + Report.FeatureIsNotAvailable (context, Location, "partial methods"); + } else if (context.Settings.Version == LanguageVersion.ISO_1) + Report.FeatureIsNotAvailable (context, Location, "partial types"); return res; } @@ -766,13 +797,51 @@ namespace Mono.CSharp break; case Token.ASYNC: - if (parsing_block > 0 || RootContext.Version != LanguageVersion.Future) { + if (parsing_modifiers) { + // + // Skip attributes section or constructor called async + // + if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) { + res = -1; + } else { + // async is keyword + } + } else if (parsing_block > 0) { + switch (peek_token ()) { + case Token.DELEGATE: + case Token.OPEN_PARENS_LAMBDA: + // async is keyword + break; + case Token.IDENTIFIER: + PushPosition (); + xtoken (); + if (xtoken () != Token.ARROW) + res = -1; + + PopPosition (); + break; + default: + res = -1; + break; + } + } else { res = -1; - break; } + + if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) { + Report.FeatureIsNotAvailable (context, Location, "asynchronous functions"); + } + + break; + + case Token.AWAIT: + if (parsing_block == 0) + res = -1; + break; } + return res; } @@ -814,29 +883,6 @@ namespace Mono.CSharp } } - public Tokenizer (SeekableStreamReader input, CompilationUnit file, CompilerContext ctx) - { - this.ref_name = file; - this.file_name = file; - this.context = ctx; - reader = input; - - putback_char = -1; - - xml_comment_buffer = new StringBuilder (); - - if (Environment.OSVersion.Platform == PlatformID.Win32NT) - tab_size = 4; - else - tab_size = 8; - - // - // FIXME: This could be `Location.Push' but we have to - // find out why the MS compiler allows this - // - Mono.CSharp.Location.Push (file, file); - } - static bool is_identifier_start_character (int c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c); @@ -1072,6 +1118,8 @@ namespace Mono.CSharp case Token.VOID: break; case Token.OP_GENERICS_GT: + case Token.IN: + case Token.OUT: return true; default: @@ -1183,6 +1231,7 @@ namespace Mono.CSharp case Token.CLOSE_PARENS: case Token.OPEN_BRACKET: case Token.OP_GENERICS_GT: + case Token.INTERR: next_token = Token.INTERR_NULLABLE; break; @@ -1285,7 +1334,7 @@ namespace Mono.CSharp } } - int integer_type_suffix (ulong ul, int c) + ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc) { bool is_unsigned = false; bool is_long = false; @@ -1328,40 +1377,38 @@ namespace Mono.CSharp } if (is_long && is_unsigned){ - val = new ULongLiteral (ul, Location); - return Token.LITERAL; + return new ULongLiteral (context.BuiltinTypes, ul, loc); } if (is_unsigned){ // uint if possible, or ulong else. if ((ul & 0xffffffff00000000) == 0) - val = new UIntLiteral ((uint) ul, Location); + return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc); else - val = new ULongLiteral (ul, Location); + return new ULongLiteral (context.BuiltinTypes, ul, loc); } else if (is_long){ // long if possible, ulong otherwise if ((ul & 0x8000000000000000) != 0) - val = new ULongLiteral (ul, Location); + return new ULongLiteral (context.BuiltinTypes, ul, loc); else - val = new LongLiteral ((long) ul, Location); + return new LongLiteral (context.BuiltinTypes, (long) ul, loc); } else { // int, uint, long or ulong in that order if ((ul & 0xffffffff00000000) == 0){ uint ui = (uint) ul; if ((ui & 0x80000000) != 0) - val = new UIntLiteral (ui, Location); + return new UIntLiteral (context.BuiltinTypes, ui, loc); else - val = new IntLiteral ((int) ui, Location); + return new IntLiteral (context.BuiltinTypes, (int) ui, loc); } else { if ((ul & 0x8000000000000000) != 0) - val = new ULongLiteral (ul, Location); + return new ULongLiteral (context.BuiltinTypes, ul, loc); else - val = new LongLiteral ((long) ul, Location); + return new LongLiteral (context.BuiltinTypes, (long) ul, loc); } } - return Token.LITERAL; } // @@ -1369,7 +1416,7 @@ namespace Mono.CSharp // we need to convert to a special type, and then choose // the best representation for the integer // - int adjust_int (int c) + ILiteralConstant adjust_int (int c, Location loc) { try { if (number_pos > 9){ @@ -1378,63 +1425,58 @@ namespace Mono.CSharp for (int i = 1; i < number_pos; i++){ ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0'))); } - return integer_type_suffix (ul, c); + + return integer_type_suffix (ul, c, loc); } else { uint ui = (uint) (number_builder [0] - '0'); for (int i = 1; i < number_pos; i++){ ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0'))); } - return integer_type_suffix (ui, c); + + return integer_type_suffix (ui, c, loc); } } catch (OverflowException) { Error_NumericConstantTooLong (); - val = new IntLiteral (0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } catch (FormatException) { Report.Error (1013, Location, "Invalid number"); - val = new IntLiteral (0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } } - int adjust_real (TypeCode t) + ILiteralConstant adjust_real (TypeCode t, Location loc) { - string s = new String (number_builder, 0, number_pos); + string s = new string (number_builder, 0, number_pos); const string error_details = "Floating-point constant is outside the range of type `{0}'"; switch (t){ case TypeCode.Decimal: try { - val = new DecimalLiteral (decimal.Parse (s, styles, csharp_format_info), Location); + return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc); } catch (OverflowException) { - val = new DecimalLiteral (0, Location); Report.Error (594, Location, error_details, "decimal"); + return new DecimalLiteral (context.BuiltinTypes, 0, loc); } - break; case TypeCode.Single: try { - val = new FloatLiteral (float.Parse (s, styles, csharp_format_info), Location); + return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc); } catch (OverflowException) { - val = new FloatLiteral (0, Location); Report.Error (594, Location, error_details, "float"); + return new FloatLiteral (context.BuiltinTypes, 0, loc); } - break; default: try { - val = new DoubleLiteral (double.Parse (s, styles, csharp_format_info), Location); + return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc); } catch (OverflowException) { - val = new DoubleLiteral (0, Location); - Report.Error (594, Location, error_details, "double"); + Report.Error (594, loc, error_details, "double"); + return new DoubleLiteral (context.BuiltinTypes, 0, loc); } - break; } - - return Token.LITERAL; } - int handle_hex () + ILiteralConstant handle_hex (Location loc) { int d; ulong ul; @@ -1449,23 +1491,22 @@ namespace Mono.CSharp } string s = new String (number_builder, 0, number_pos); + try { if (number_pos <= 8) ul = System.UInt32.Parse (s, NumberStyles.HexNumber); else ul = System.UInt64.Parse (s, NumberStyles.HexNumber); + + return integer_type_suffix (ul, peek_char (), loc); } catch (OverflowException){ Error_NumericConstantTooLong (); - val = new IntLiteral (0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } catch (FormatException) { Report.Error (1013, Location, "Invalid number"); - val = new IntLiteral (0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } - - return integer_type_suffix (ul, peek_char ()); } // @@ -1473,16 +1514,26 @@ namespace Mono.CSharp // int is_number (int c) { - bool is_real = false; + ILiteralConstant res; +#if FULL_AST + int read_start = reader.Position - 1; +#endif number_pos = 0; + var loc = Location; if (c >= '0' && c <= '9'){ if (c == '0'){ int peek = peek_char (); - if (peek == 'x' || peek == 'X') - return handle_hex (); + if (peek == 'x' || peek == 'X') { + val = res = handle_hex (loc); +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1); +#endif + + return Token.LITERAL; + } } decimal_digits (c); c = get_char (); @@ -1492,6 +1543,7 @@ namespace Mono.CSharp // We need to handle the case of // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER) // + bool is_real = false; if (c == '.'){ if (decimal_digits ('.')){ is_real = true; @@ -1499,7 +1551,12 @@ namespace Mono.CSharp } else { putback ('.'); number_pos--; - return adjust_int (-1); + val = res = adjust_int (-1, loc); + +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1); +#endif + return Token.LITERAL; } } @@ -1507,7 +1564,7 @@ namespace Mono.CSharp is_real = true; if (number_pos == max_number_size) Error_NumericConstantTooLong (); - number_builder [number_pos++] = 'e'; + number_builder [number_pos++] = (char) c; c = get_char (); if (c == '+'){ @@ -1531,21 +1588,26 @@ namespace Mono.CSharp } var type = real_type_suffix (c); - if (type == TypeCode.Empty && !is_real){ + if (type == TypeCode.Empty && !is_real) { putback (c); - return adjust_int (c); - } + res = adjust_int (c, loc); + } else { + is_real = true; - is_real = true; + if (type == TypeCode.Empty) { + putback (c); + } - if (type == TypeCode.Empty){ - putback (c); + res = adjust_real (type, loc); } - - if (is_real) - return adjust_real (type); - throw new Exception ("Is Number should never reach this point"); + val = res; + +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty ? 1 : 0)); +#endif + + return Token.LITERAL; } // @@ -1675,9 +1737,18 @@ namespace Mono.CSharp if (putback_char != -1) { x = putback_char; putback_char = -1; - } else + } else { x = reader.Read (); - if (x == '\n') { + } + + if (x == '\r') { + if (peek_char () == '\n') { + putback_char = -1; + } + + x = '\n'; + advance_line (); + } else if (x == '\n') { advance_line (); } else { col++; @@ -1754,7 +1825,7 @@ namespace Mono.CSharp // skip over white space do { c = get_char (); - } while (c == '\r' || c == ' ' || c == '\t'); + } while (c == ' ' || c == '\t'); int pos = 0; @@ -1792,13 +1863,13 @@ namespace Mono.CSharp return cmd; // skip over white space - while (c == '\r' || c == ' ' || c == '\t') + while (c == ' ' || c == '\t') c = get_char (); - static_cmd_arg.Length = 0; int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument); + int pos = 0; - while (c != -1 && c != '\n' && c != '\r') { + while (c != -1 && c != '\n') { if (c == '\\' && has_identifier_argument >= 0) { if (has_identifier_argument != 0) { has_identifier_argument = 1; @@ -1808,27 +1879,44 @@ namespace Mono.CSharp int surrogate; c = EscapeUnicode (c, out surrogate); if (surrogate != 0) { - if (is_identifier_part_character ((char) c)) - static_cmd_arg.Append ((char) c); + if (is_identifier_part_character ((char) c)) { + if (pos == value_builder.Length) + Array.Resize (ref value_builder, pos * 2); + + value_builder[pos++] = (char) c; + } c = surrogate; } } } else { has_identifier_argument = -1; } + } else if (c == '/' && peek_char () == '/') { + // + // Eat single-line comments + // + get_char (); + do { + c = get_char (); + } while (c != -1 && c != '\n'); + + break; } - static_cmd_arg.Append ((char) c); + + if (pos == value_builder.Length) + Array.Resize (ref value_builder, pos * 2); + + value_builder[pos++] = (char) c; c = get_char (); } - if (static_cmd_arg.Length != 0) { - arg = static_cmd_arg.ToString (); - - // Eat any trailing whitespaces and single-line comments - if (arg.IndexOf ("//") != -1) { - arg = arg.Substring (0, arg.IndexOf ("//")); - } + if (pos != 0) { + if (pos > max_id_size) + arg = new string (value_builder, 0, pos); + else + arg = InternIdentifier (value_builder, pos); + // Eat any trailing whitespaces arg = arg.Trim (simple_whitespaces); } @@ -1864,8 +1952,8 @@ namespace Mono.CSharp char [] quotes = { '\"' }; string name = arg.Substring (pos). Trim (quotes); - ref_name = Location.LookupFile (file_name, name); - file_name.AddFile (ref_name); + ref_name = context.LookupFile (file_name, name); + file_name.AddIncludeFile (ref_name); hidden = false; Location.Push (file_name, ref_name); } else { @@ -1911,7 +1999,7 @@ namespace Mono.CSharp // // #define ident // - if (RootContext.IsConditionalDefined (ident)) + if (context.Settings.IsConditionalSymbolDefined (ident)) return; file_name.AddDefine (ident); @@ -1989,7 +2077,7 @@ namespace Mono.CSharp if (c != ' ') return false; - SourceFile file = Location.LookupFile (file_name, string_builder.ToString ()); + SourceFile file = context.LookupFile (file_name, string_builder.ToString ()); if (get_char () != '"' || get_char () != '{') return false; @@ -2095,7 +2183,7 @@ namespace Mono.CSharp c = get_char (); // skip over white space - while (c == '\r' || c == ' ' || c == '\t') + while (c == ' ' || c == '\t') c = get_char (); if (c == ',') { @@ -2103,7 +2191,7 @@ namespace Mono.CSharp } // skip over white space - while (c == '\r' || c == ' ' || c == '\t') + while (c == ' ' || c == '\t') c = get_char (); } else { number = -1; @@ -2152,7 +2240,7 @@ namespace Mono.CSharp bool disable = IsTokenIdentifierEqual (pragma_warning_disable); if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) { // skip over white space - while (c == '\r' || c == ' ' || c == '\t') + while (c == ' ' || c == '\t') c = get_char (); var loc = Location; @@ -2183,7 +2271,7 @@ namespace Mono.CSharp Report.RegisterWarningRegion (loc).WarningEnable (loc, code, Report); } } - } while (code >= 0 && c != '\n'); + } while (code >= 0 && c != '\n' && c != -1); } return; @@ -2216,7 +2304,7 @@ namespace Mono.CSharp if (s == "false") return false; - return file_name.IsConditionalDefined (s); + return file_name.IsConditionalDefined (context, s); } bool pp_primary (ref string s) @@ -2402,6 +2490,18 @@ namespace Mono.CSharp { Report.Error (1025, Location, "Single-line comment or end-of-line expected"); } + + // + // Raises a warning when tokenizer found documentation comment + // on unexpected place + // + void WarningMisplacedComment (Location loc) + { + if (doc_state != XmlCommentState.Error) { + doc_state = XmlCommentState.Error; + Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element"); + } + } // // if true, then the code continues processing the code @@ -2451,7 +2551,7 @@ namespace Mono.CSharp } } - if (caller_is_taking && eval (arg)) { + if (eval (arg) && caller_is_taking) { ifstack.Push (flags | TAKING); return true; } @@ -2585,8 +2685,8 @@ namespace Mono.CSharp return true; case PreprocessorDirective.Pragma: - if (RootContext.Version == LanguageVersion.ISO_1) { - Report.FeatureIsNotAvailable (Location, "#pragma"); + if (context.Settings.Version == LanguageVersion.ISO_1) { + Report.FeatureIsNotAvailable (context, Location, "#pragma"); } ParsePragmaDirective (arg); @@ -2606,31 +2706,62 @@ namespace Mono.CSharp private int consume_string (bool quoted) { int c; - string_builder.Length = 0; + int pos = 0; + Location start_location = Location; + if (quoted) + start_location = start_location - 1; + +#if FULL_AST + int reader_pos = reader.Position; +#endif while (true){ c = get_char (); if (c == '"') { if (quoted && peek_char () == '"') { - string_builder.Append ((char) c); + if (pos == value_builder.Length) + Array.Resize (ref value_builder, pos * 2); + + value_builder[pos++] = (char) c; get_char (); continue; } - val = new StringLiteral (string_builder.ToString (), Location); + string s; + if (pos == 0) + s = string.Empty; + else if (pos <= 4) + s = InternIdentifier (value_builder, pos); + else + s = new string (value_builder, 0, pos); + + ILiteralConstant res = new StringLiteral (context.BuiltinTypes, s, start_location); + val = res; +#if FULL_AST + res.ParsedValue = quoted ? + reader.ReadChars (reader_pos - 2, reader.Position - 1) : + reader.ReadChars (reader_pos - 1, reader.Position); +#endif + return Token.LITERAL; } if (c == '\n') { - if (!quoted) + if (!quoted) { Report.Error (1010, Location, "Newline in constant"); + val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location); + return Token.LITERAL; + } } else if (c == '\\' && !quoted) { int surrogate; c = escape (c, out surrogate); if (c == -1) return Token.ERROR; if (surrogate != 0) { - string_builder.Append ((char) c); + if (pos == value_builder.Length) + Array.Resize (ref value_builder, pos * 2); + + value_builder[pos++] = (char) c; c = surrogate; } } else if (c == -1) { @@ -2638,7 +2769,10 @@ namespace Mono.CSharp return Token.EOF; } - string_builder.Append ((char) c); + if (pos == value_builder.Length) + Array.Resize (ref value_builder, pos * 2); + + value_builder[pos++] = (char) c; } } @@ -2661,6 +2795,8 @@ namespace Mono.CSharp int pos = 0; int column = col; + if (quoted) + --column; if (c == '\\') { int surrogate; @@ -2686,9 +2822,10 @@ namespace Mono.CSharp if (c == '\\') { int surrogate; c = escape (c, out surrogate); + if (is_identifier_part_character ((char) c)) + id_builder[pos++] = (char) c; + if (surrogate != 0) { - if (is_identifier_part_character ((char) c)) - id_builder[pos++] = (char) c; c = surrogate; } @@ -2717,43 +2854,45 @@ namespace Mono.CSharp if (id_builder [0] >= '_' && !quoted) { int keyword = GetKeyword (id_builder, pos); if (keyword != -1) { - val = LocatedToken.Create (null, ref_line, column); + val = LocatedToken.Create (keyword == Token.AWAIT ? "await" : null, ref_line, column); return keyword; } } + string s = InternIdentifier (id_builder, pos); + val = LocatedToken.Create (s, ref_line, column); + if (quoted && parsing_attribute_section) + AddEscapedIdentifier (((LocatedToken) val).Location); + + return Token.IDENTIFIER; + } + + static string InternIdentifier (char[] charBuffer, int length) + { // // Keep identifiers in an array of hashtables to avoid needless // allocations // - var identifiers_group = identifiers [pos]; + var identifiers_group = identifiers[length]; string s; if (identifiers_group != null) { - if (identifiers_group.TryGetValue (id_builder, out s)) { - val = LocatedToken.Create (s, ref_line, column); - if (quoted) - AddEscapedIdentifier (((LocatedToken) val).Location); - return Token.IDENTIFIER; + if (identifiers_group.TryGetValue (charBuffer, out s)) { + return s; } } else { // TODO: this should be number of files dependant // corlib compilation peaks at 1000 and System.Core at 150 - int capacity = pos > 20 ? 10 : 100; - identifiers_group = new Dictionary (capacity, new IdentifiersComparer (pos)); - identifiers [pos] = identifiers_group; + int capacity = length > 20 ? 10 : 100; + identifiers_group = new Dictionary (capacity, new IdentifiersComparer (length)); + identifiers[length] = identifiers_group; } - char [] chars = new char [pos]; - Array.Copy (id_builder, chars, pos); + char[] chars = new char[length]; + Array.Copy (charBuffer, chars, length); - s = new string (id_builder, 0, pos); + s = new string (charBuffer, 0, length); identifiers_group.Add (chars, s); - - val = LocatedToken.Create (s, ref_line, column); - if (quoted) - AddEscapedIdentifier (((LocatedToken) val).Location); - - return Token.IDENTIFIER; + return s; } public int xtoken () @@ -2787,17 +2926,6 @@ namespace Mono.CSharp } break; */ - case '\r': - if (peek_char () != '\n') - advance_line (); - else - get_char (); - - any_token_seen |= tokens_seen; - tokens_seen = false; - comments_seen = false; - continue; - case '\\': tokens_seen = true; return consume_identifier (c); @@ -3030,18 +3158,23 @@ namespace Mono.CSharp // Handle double-slash comments. if (d == '/'){ get_char (); - if (RootContext.Documentation != null && peek_char () == '/') { - get_char (); - // Don't allow ////. - if ((d = peek_char ()) != '/') { - update_comment_location (); - if (doc_state == XmlCommentState.Allowed) - handle_one_line_xml_comment (); - else if (doc_state == XmlCommentState.NotAllowed) - warn_incorrect_doc_comment (); + if (doc_processing) { + if (peek_char () == '/') { + get_char (); + // Don't allow ////. + if ((d = peek_char ()) != '/') { + if (doc_state == XmlCommentState.Allowed) + handle_one_line_xml_comment (); + else if (doc_state == XmlCommentState.NotAllowed) + WarningMisplacedComment (Location - 3); + } + } else { + if (xml_comment_buffer.Length > 0) + doc_state = XmlCommentState.NotAllowed; } } - while ((d = get_char ()) != -1 && (d != '\n') && d != '\r'); + + while ((d = get_char ()) != -1 && d != '\n'); any_token_seen |= tokens_seen; tokens_seen = false; @@ -3050,9 +3183,8 @@ namespace Mono.CSharp } else if (d == '*'){ get_char (); bool docAppend = false; - if (RootContext.Documentation != null && peek_char () == '*') { + if (doc_processing && peek_char () == '*') { get_char (); - update_comment_location (); // But when it is /**/, just do nothing. if (peek_char () == '/') { get_char (); @@ -3060,8 +3192,9 @@ namespace Mono.CSharp } if (doc_state == XmlCommentState.Allowed) docAppend = true; - else if (doc_state == XmlCommentState.NotAllowed) - warn_incorrect_doc_comment (); + else if (doc_state == XmlCommentState.NotAllowed) { + WarningMisplacedComment (Location - 2); + } } int current_comment_start = 0; @@ -3165,7 +3298,7 @@ namespace Mono.CSharp continue; } - if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' ) + if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' ) continue; if (c == '#') { @@ -3208,6 +3341,8 @@ namespace Mono.CSharp return Token.EVAL_COMPILATION_UNIT_PARSER; case EvalUsingDeclarationsParserCharacter: return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER; + case DocumentationXref: + return Token.DOC_SEE; } if (is_identifier_start_character (c)) { @@ -3215,6 +3350,9 @@ namespace Mono.CSharp return consume_identifier (c); } + if (char.IsWhiteSpace ((char) c)) + continue; + Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ()); } @@ -3232,16 +3370,20 @@ namespace Mono.CSharp int TokenizeBackslash () { +#if FULL_AST + int read_start = reader.Position; +#endif + Location start_location = Location; int c = get_char (); tokens_seen = true; if (c == '\'') { - val = new CharLiteral ((char) c, Location); - Report.Error (1011, Location, "Empty character literal"); + val = new CharLiteral (context.BuiltinTypes, (char) c, start_location); + Report.Error (1011, start_location, "Empty character literal"); return Token.LITERAL; } - if (c == '\r' || c == '\n') { - Report.Error (1010, Location, "Newline in constant"); + if (c == '\n') { + Report.Error (1010, start_location, "Newline in constant"); return Token.ERROR; } @@ -3252,11 +3394,12 @@ namespace Mono.CSharp if (d != 0) throw new NotImplementedException (); - val = new CharLiteral ((char) c, Location); + ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location); + val = res; c = get_char (); if (c != '\'') { - Report.Error (1012, Location, "Too many characters in character literal"); + Report.Error (1012, start_location, "Too many characters in character literal"); // Try to recover, read until newline or next "'" while ((c = get_char ()) != -1) { @@ -3265,6 +3408,10 @@ namespace Mono.CSharp } } +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position); +#endif + return Token.LITERAL; } @@ -3284,7 +3431,7 @@ namespace Mono.CSharp // Save current position and parse next token. PushPosition (); if (parse_less_than ()) { - if (parsing_generic_declaration && token () != Token.DOT) { + if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) { d = Token.OP_GENERICS_LT_DECL; } else { d = Token.OP_GENERICS_LT; @@ -3361,18 +3508,6 @@ namespace Mono.CSharp xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines)); } - // - // Updates current comment location. - // - private void update_comment_location () - { - if (current_comment_location.IsNull) { - // "-2" is for heading "//" or "/*" - current_comment_location = - new Location (ref_line, hidden ? -1 : col - 2); - } - } - // // Checks if there was incorrect doc comments and raise // warnings. @@ -3380,22 +3515,7 @@ namespace Mono.CSharp public void check_incorrect_doc_comment () { if (xml_comment_buffer.Length > 0) - warn_incorrect_doc_comment (); - } - - // - // Raises a warning when tokenizer found incorrect doccomment - // markup. - // - private void warn_incorrect_doc_comment () - { - if (doc_state != XmlCommentState.Error) { - doc_state = XmlCommentState.Error; - // in csc, it is 'XML comment is not placed on - // a valid language element'. But that does not - // make sense. - Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element"); - } + WarningMisplacedComment (Location); } // @@ -3419,7 +3539,6 @@ namespace Mono.CSharp void reset_doc_comment () { xml_comment_buffer.Length = 0; - current_comment_location = Location.Null; } public void cleanup ()