X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mcs%2Fmcs%2Fcs-tokenizer.cs;h=2dccc46414da9f88d981f042d186a30fb6433351;hb=8e33e0d088c564d039dd791b7cc3e121ff87b3e4;hp=b4e1ef8955f2efc57c19822b94e9a8a18f75c762;hpb=e038c33277c88cf500fa096df295255f7abb01f8;p=mono.git diff --git a/mcs/mcs/cs-tokenizer.cs b/mcs/mcs/cs-tokenizer.cs index b4e1ef8955f..2dccc46414d 100644 --- a/mcs/mcs/cs-tokenizer.cs +++ b/mcs/mcs/cs-tokenizer.cs @@ -3,13 +3,13 @@ // This also implements the preprocessor // // Author: Miguel de Icaza (miguel@gnu.org) -// Marek Safar (marek.safar@seznam.cz) +// Marek Safar (marek.safar@gmail.com) // // Dual licensed under the terms of the MIT X11 or GNU GPL // // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com) // Copyright 2004-2008 Novell, Inc -// +// Copyright 2011 Xamarin, Inc (http://www.xamarin.com) // using System; @@ -87,6 +87,11 @@ namespace Mono.CSharp { return Create (null, row, column); } + + public static LocatedToken Create (string value, Location loc) + { + return Create (value, loc.Row, loc.Column); + } public static LocatedToken Create (string value, int row, int column) { @@ -141,7 +146,7 @@ namespace Mono.CSharp } } - enum PreprocessorDirective + public enum PreprocessorDirective { Invalid = 0, @@ -199,6 +204,7 @@ namespace Mono.CSharp // Set when parsing generic declaration (type or method header) // public bool parsing_generic_declaration; + public bool parsing_generic_declaration_doc; // // The value indicates that we have not reach any declaration or @@ -208,19 +214,22 @@ namespace Mono.CSharp public bool parsing_attribute_section; + public bool parsing_modifiers; + // - // The special character to inject on streams to trigger the EXPRESSION_PARSE - // token to be returned. It just happens to be a Unicode character that - // would never be part of a program (can not be an identifier). + // The special characters to inject on streams to run the unit parser + // in the special expression mode. Using private characters from + // Plane Sixteen (U+100000 to U+10FFFD) // // This character is only tested just before the tokenizer is about to report // an error; So on the regular operation mode, this addition will have no // impact on the tokenizer's performance. // - public const int EvalStatementParserCharacter = 0x2190; // Unicode Left Arrow - public const int EvalCompilationUnitParserCharacter = 0x2191; // Unicode Arrow - public const int EvalUsingDeclarationsParserCharacter = 0x2192; // Unicode Arrow + public const int EvalStatementParserCharacter = 0x100000; + public const int EvalCompilationUnitParserCharacter = 0x100001; + public const int EvalUsingDeclarationsParserCharacter = 0x100002; + public const int DocumentationXref = 0x100003; // // XML documentation buffer. The save point is used to divide @@ -318,7 +327,7 @@ namespace Mono.CSharp escaped_identifiers.Add (loc); } - public bool IsEscapedIdentifier (MemberName name) + public bool IsEscapedIdentifier (ATypeNameExpression name) { return escaped_identifiers != null && escaped_identifiers.Contains (name.Location); } @@ -414,17 +423,13 @@ namespace Mono.CSharp putback_char = -1; xml_comment_buffer = new StringBuilder (); - doc_processing = ctx.Settings.Documentation != null; + doc_processing = ctx.Settings.DocumentationFile != null; if (Environment.OSVersion.Platform == PlatformID.Win32NT) tab_size = 4; else tab_size = 8; - // - // FIXME: This could be `Location.Push' but we have to - // find out why the MS compiler allows this - // Mono.CSharp.Location.Push (file, file); } @@ -586,7 +591,6 @@ namespace Mono.CSharp AddKeyword ("while", Token.WHILE); AddKeyword ("partial", Token.PARTIAL); AddKeyword ("where", Token.WHERE); - AddKeyword ("async", Token.ASYNC); // LINQ keywords AddKeyword ("from", Token.FROM); @@ -602,6 +606,10 @@ namespace Mono.CSharp AddKeyword ("descending", Token.DESCENDING); AddKeyword ("into", Token.INTO); + // Contextual async keywords + AddKeyword ("async", Token.ASYNC); + AddKeyword ("await", Token.AWAIT); + keywords_preprocessor = new KeywordEntry[10][]; AddPreprocessorKeyword ("region", PreprocessorDirective.Region); @@ -789,13 +797,51 @@ namespace Mono.CSharp break; case Token.ASYNC: - if (parsing_block > 0 || context.Settings.Version != LanguageVersion.Future) { + if (parsing_modifiers) { + // + // Skip attributes section or constructor called async + // + if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) { + res = -1; + } else { + // async is keyword + } + } else if (parsing_block > 0) { + switch (peek_token ()) { + case Token.DELEGATE: + case Token.OPEN_PARENS_LAMBDA: + // async is keyword + break; + case Token.IDENTIFIER: + PushPosition (); + xtoken (); + if (xtoken () != Token.ARROW) + res = -1; + + PopPosition (); + break; + default: + res = -1; + break; + } + } else { res = -1; - break; } + + if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) { + Report.FeatureIsNotAvailable (context, Location, "asynchronous functions"); + } + + break; + + case Token.AWAIT: + if (parsing_block == 0) + res = -1; + break; } + return res; } @@ -1072,6 +1118,8 @@ namespace Mono.CSharp case Token.VOID: break; case Token.OP_GENERICS_GT: + case Token.IN: + case Token.OUT: return true; default: @@ -1183,6 +1231,7 @@ namespace Mono.CSharp case Token.CLOSE_PARENS: case Token.OPEN_BRACKET: case Token.OP_GENERICS_GT: + case Token.INTERR: next_token = Token.INTERR_NULLABLE; break; @@ -1285,7 +1334,7 @@ namespace Mono.CSharp } } - int integer_type_suffix (ulong ul, int c) + ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc) { bool is_unsigned = false; bool is_long = false; @@ -1328,40 +1377,38 @@ namespace Mono.CSharp } if (is_long && is_unsigned){ - val = new ULongLiteral (context.BuiltinTypes, ul, Location); - return Token.LITERAL; + return new ULongLiteral (context.BuiltinTypes, ul, loc); } if (is_unsigned){ // uint if possible, or ulong else. if ((ul & 0xffffffff00000000) == 0) - val = new UIntLiteral (context.BuiltinTypes, (uint) ul, Location); + return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc); else - val = new ULongLiteral (context.BuiltinTypes, ul, Location); + return new ULongLiteral (context.BuiltinTypes, ul, loc); } else if (is_long){ // long if possible, ulong otherwise if ((ul & 0x8000000000000000) != 0) - val = new ULongLiteral (context.BuiltinTypes, ul, Location); + return new ULongLiteral (context.BuiltinTypes, ul, loc); else - val = new LongLiteral (context.BuiltinTypes, (long) ul, Location); + return new LongLiteral (context.BuiltinTypes, (long) ul, loc); } else { // int, uint, long or ulong in that order if ((ul & 0xffffffff00000000) == 0){ uint ui = (uint) ul; if ((ui & 0x80000000) != 0) - val = new UIntLiteral (context.BuiltinTypes, ui, Location); + return new UIntLiteral (context.BuiltinTypes, ui, loc); else - val = new IntLiteral (context.BuiltinTypes, (int) ui, Location); + return new IntLiteral (context.BuiltinTypes, (int) ui, loc); } else { if ((ul & 0x8000000000000000) != 0) - val = new ULongLiteral (context.BuiltinTypes, ul, Location); + return new ULongLiteral (context.BuiltinTypes, ul, loc); else - val = new LongLiteral (context.BuiltinTypes, (long) ul, Location); + return new LongLiteral (context.BuiltinTypes, (long) ul, loc); } } - return Token.LITERAL; } // @@ -1369,7 +1416,7 @@ namespace Mono.CSharp // we need to convert to a special type, and then choose // the best representation for the integer // - int adjust_int (int c) + ILiteralConstant adjust_int (int c, Location loc) { try { if (number_pos > 9){ @@ -1378,63 +1425,58 @@ namespace Mono.CSharp for (int i = 1; i < number_pos; i++){ ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0'))); } - return integer_type_suffix (ul, c); + + return integer_type_suffix (ul, c, loc); } else { uint ui = (uint) (number_builder [0] - '0'); for (int i = 1; i < number_pos; i++){ ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0'))); } - return integer_type_suffix (ui, c); + + return integer_type_suffix (ui, c, loc); } } catch (OverflowException) { Error_NumericConstantTooLong (); - val = new IntLiteral (context.BuiltinTypes, 0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } catch (FormatException) { Report.Error (1013, Location, "Invalid number"); - val = new IntLiteral (context.BuiltinTypes, 0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } } - int adjust_real (TypeCode t) + ILiteralConstant adjust_real (TypeCode t, Location loc) { - string s = new String (number_builder, 0, number_pos); + string s = new string (number_builder, 0, number_pos); const string error_details = "Floating-point constant is outside the range of type `{0}'"; switch (t){ case TypeCode.Decimal: try { - val = new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), Location); + return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc); } catch (OverflowException) { - val = new DecimalLiteral (context.BuiltinTypes, 0, Location); Report.Error (594, Location, error_details, "decimal"); + return new DecimalLiteral (context.BuiltinTypes, 0, loc); } - break; case TypeCode.Single: try { - val = new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), Location); + return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc); } catch (OverflowException) { - val = new FloatLiteral (context.BuiltinTypes, 0, Location); Report.Error (594, Location, error_details, "float"); + return new FloatLiteral (context.BuiltinTypes, 0, loc); } - break; default: try { - val = new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), Location); + return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc); } catch (OverflowException) { - val = new DoubleLiteral (context.BuiltinTypes, 0, Location); - Report.Error (594, Location, error_details, "double"); + Report.Error (594, loc, error_details, "double"); + return new DoubleLiteral (context.BuiltinTypes, 0, loc); } - break; } - - return Token.LITERAL; } - int handle_hex () + ILiteralConstant handle_hex (Location loc) { int d; ulong ul; @@ -1449,23 +1491,22 @@ namespace Mono.CSharp } string s = new String (number_builder, 0, number_pos); + try { if (number_pos <= 8) ul = System.UInt32.Parse (s, NumberStyles.HexNumber); else ul = System.UInt64.Parse (s, NumberStyles.HexNumber); + + return integer_type_suffix (ul, peek_char (), loc); } catch (OverflowException){ Error_NumericConstantTooLong (); - val = new IntLiteral (context.BuiltinTypes, 0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } catch (FormatException) { Report.Error (1013, Location, "Invalid number"); - val = new IntLiteral (context.BuiltinTypes, 0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } - - return integer_type_suffix (ul, peek_char ()); } // @@ -1473,16 +1514,26 @@ namespace Mono.CSharp // int is_number (int c) { - bool is_real = false; + ILiteralConstant res; +#if FULL_AST + int read_start = reader.Position - 1; +#endif number_pos = 0; + var loc = Location; if (c >= '0' && c <= '9'){ if (c == '0'){ int peek = peek_char (); - if (peek == 'x' || peek == 'X') - return handle_hex (); + if (peek == 'x' || peek == 'X') { + val = res = handle_hex (loc); +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1); +#endif + + return Token.LITERAL; + } } decimal_digits (c); c = get_char (); @@ -1492,6 +1543,7 @@ namespace Mono.CSharp // We need to handle the case of // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER) // + bool is_real = false; if (c == '.'){ if (decimal_digits ('.')){ is_real = true; @@ -1499,7 +1551,12 @@ namespace Mono.CSharp } else { putback ('.'); number_pos--; - return adjust_int (-1); + val = res = adjust_int (-1, loc); + +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1); +#endif + return Token.LITERAL; } } @@ -1507,7 +1564,7 @@ namespace Mono.CSharp is_real = true; if (number_pos == max_number_size) Error_NumericConstantTooLong (); - number_builder [number_pos++] = 'e'; + number_builder [number_pos++] = (char) c; c = get_char (); if (c == '+'){ @@ -1531,21 +1588,26 @@ namespace Mono.CSharp } var type = real_type_suffix (c); - if (type == TypeCode.Empty && !is_real){ + if (type == TypeCode.Empty && !is_real) { putback (c); - return adjust_int (c); - } + res = adjust_int (c, loc); + } else { + is_real = true; - is_real = true; + if (type == TypeCode.Empty) { + putback (c); + } - if (type == TypeCode.Empty){ - putback (c); + res = adjust_real (type, loc); } - - if (is_real) - return adjust_real (type); - throw new Exception ("Is Number should never reach this point"); + val = res; + +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty ? 1 : 0)); +#endif + + return Token.LITERAL; } // @@ -2646,6 +2708,13 @@ namespace Mono.CSharp int c; int pos = 0; Location start_location = Location; + if (quoted) + start_location = start_location - 1; + +#if FULL_AST + int reader_pos = reader.Position; +#endif + while (true){ c = get_char (); if (c == '"') { @@ -2666,13 +2735,23 @@ namespace Mono.CSharp else s = new string (value_builder, 0, pos); - val = new StringLiteral (context.BuiltinTypes, s, start_location); + ILiteralConstant res = new StringLiteral (context.BuiltinTypes, s, start_location); + val = res; +#if FULL_AST + res.ParsedValue = quoted ? + reader.ReadChars (reader_pos - 2, reader.Position - 1) : + reader.ReadChars (reader_pos - 1, reader.Position); +#endif + return Token.LITERAL; } if (c == '\n') { - if (!quoted) + if (!quoted) { Report.Error (1010, Location, "Newline in constant"); + val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location); + return Token.LITERAL; + } } else if (c == '\\' && !quoted) { int surrogate; c = escape (c, out surrogate); @@ -2716,6 +2795,8 @@ namespace Mono.CSharp int pos = 0; int column = col; + if (quoted) + --column; if (c == '\\') { int surrogate; @@ -2741,9 +2822,10 @@ namespace Mono.CSharp if (c == '\\') { int surrogate; c = escape (c, out surrogate); + if (is_identifier_part_character ((char) c)) + id_builder[pos++] = (char) c; + if (surrogate != 0) { - if (is_identifier_part_character ((char) c)) - id_builder[pos++] = (char) c; c = surrogate; } @@ -2772,7 +2854,7 @@ namespace Mono.CSharp if (id_builder [0] >= '_' && !quoted) { int keyword = GetKeyword (id_builder, pos); if (keyword != -1) { - val = LocatedToken.Create (null, ref_line, column); + val = LocatedToken.Create (keyword == Token.AWAIT ? "await" : null, ref_line, column); return keyword; } } @@ -3087,7 +3169,8 @@ namespace Mono.CSharp WarningMisplacedComment (Location - 3); } } else { - check_incorrect_doc_comment (); + if (xml_comment_buffer.Length > 0) + doc_state = XmlCommentState.NotAllowed; } } @@ -3258,6 +3341,8 @@ namespace Mono.CSharp return Token.EVAL_COMPILATION_UNIT_PARSER; case EvalUsingDeclarationsParserCharacter: return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER; + case DocumentationXref: + return Token.DOC_SEE; } if (is_identifier_start_character (c)) { @@ -3285,16 +3370,20 @@ namespace Mono.CSharp int TokenizeBackslash () { +#if FULL_AST + int read_start = reader.Position; +#endif + Location start_location = Location; int c = get_char (); tokens_seen = true; if (c == '\'') { - val = new CharLiteral (context.BuiltinTypes, (char) c, Location); - Report.Error (1011, Location, "Empty character literal"); + val = new CharLiteral (context.BuiltinTypes, (char) c, start_location); + Report.Error (1011, start_location, "Empty character literal"); return Token.LITERAL; } if (c == '\n') { - Report.Error (1010, Location, "Newline in constant"); + Report.Error (1010, start_location, "Newline in constant"); return Token.ERROR; } @@ -3305,11 +3394,12 @@ namespace Mono.CSharp if (d != 0) throw new NotImplementedException (); - val = new CharLiteral (context.BuiltinTypes, (char) c, Location); + ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location); + val = res; c = get_char (); if (c != '\'') { - Report.Error (1012, Location, "Too many characters in character literal"); + Report.Error (1012, start_location, "Too many characters in character literal"); // Try to recover, read until newline or next "'" while ((c = get_char ()) != -1) { @@ -3318,6 +3408,10 @@ namespace Mono.CSharp } } +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position); +#endif + return Token.LITERAL; } @@ -3337,7 +3431,7 @@ namespace Mono.CSharp // Save current position and parse next token. PushPosition (); if (parse_less_than ()) { - if (parsing_generic_declaration && token () != Token.DOT) { + if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) { d = Token.OP_GENERICS_LT_DECL; } else { d = Token.OP_GENERICS_LT;