X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mcs%2Fmcs%2Fcs-tokenizer.cs;h=2dccc46414da9f88d981f042d186a30fb6433351;hb=8e33e0d088c564d039dd791b7cc3e121ff87b3e4;hp=266956997de43ae367d5137b6cc0e370bbe142f3;hpb=826e589a045a57b407d93063f744dcfa1b9335c1;p=mono.git diff --git a/mcs/mcs/cs-tokenizer.cs b/mcs/mcs/cs-tokenizer.cs index 266956997de..2dccc46414d 100644 --- a/mcs/mcs/cs-tokenizer.cs +++ b/mcs/mcs/cs-tokenizer.cs @@ -3,21 +3,20 @@ // This also implements the preprocessor // // Author: Miguel de Icaza (miguel@gnu.org) -// Marek Safar (marek.safar@seznam.cz) +// Marek Safar (marek.safar@gmail.com) // // Dual licensed under the terms of the MIT X11 or GNU GPL // // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com) // Copyright 2004-2008 Novell, Inc -// +// Copyright 2011 Xamarin, Inc (http://www.xamarin.com) // using System; using System.Text; using System.Collections.Generic; -using System.IO; using System.Globalization; -using System.Reflection; +using System.Diagnostics; namespace Mono.CSharp { @@ -27,13 +26,13 @@ namespace Mono.CSharp public class Tokenizer : yyParser.yyInput { - class KeywordEntry + class KeywordEntry { - public readonly int Token; - public KeywordEntry Next; + public readonly T Token; + public KeywordEntry Next; public readonly char[] Value; - public KeywordEntry (string value, int token) + public KeywordEntry (string value, T token) { this.Value = value.ToCharArray (); this.Token = token; @@ -88,6 +87,11 @@ namespace Mono.CSharp { return Create (null, row, column); } + + public static LocatedToken Create (string value, Location loc) + { + return Create (value, loc.Row, loc.Column); + } public static LocatedToken Create (string value, int row, int column) { @@ -120,12 +124,10 @@ namespace Mono.CSharp // // Used for token not required by expression evaluator // - public static LocatedToken CreateOptional (int row, int col) + [Conditional ("FULL_AST")] + public static void CreateOptional (int row, int col, ref object token) { -#if false - return Create (row, col); -#endif - return null; + token = Create (row, col); } public static void Initialize () @@ -144,9 +146,30 @@ namespace Mono.CSharp } } + public enum PreprocessorDirective + { + Invalid = 0, + + Region = 1, + Endregion = 2, + If = 3 | RequiresArgument, + Endif = 4, + Elif = 5 | RequiresArgument, + Else = 6, + Define = 7 | RequiresArgument, + Undef = 8 | RequiresArgument, + Error = 9, + Warning = 10, + Pragma = 11 | CustomArgumentsParsing, + Line = 12, + + CustomArgumentsParsing = 1 << 10, + RequiresArgument = 1 << 11 + } + SeekableStreamReader reader; SourceFile ref_name; - CompilationUnit file_name; + CompilationSourceFile file_name; CompilerContext context; bool hidden = false; int ref_line = 1; @@ -160,9 +183,9 @@ namespace Mono.CSharp bool handle_where = false; bool handle_typeof = false; bool lambda_arguments_parsing; - Location current_comment_location = Location.Null; List escaped_identifiers; int parsing_generic_less_than; + readonly bool doc_processing; // // Used mainly for parser optimizations. Some expressions for instance @@ -181,6 +204,7 @@ namespace Mono.CSharp // Set when parsing generic declaration (type or method header) // public bool parsing_generic_declaration; + public bool parsing_generic_declaration_doc; // // The value indicates that we have not reach any declaration or @@ -188,19 +212,24 @@ namespace Mono.CSharp // public int parsing_declaration; + public bool parsing_attribute_section; + + public bool parsing_modifiers; + // - // The special character to inject on streams to trigger the EXPRESSION_PARSE - // token to be returned. It just happens to be a Unicode character that - // would never be part of a program (can not be an identifier). + // The special characters to inject on streams to run the unit parser + // in the special expression mode. Using private characters from + // Plane Sixteen (U+100000 to U+10FFFD) // // This character is only tested just before the tokenizer is about to report // an error; So on the regular operation mode, this addition will have no // impact on the tokenizer's performance. // - public const int EvalStatementParserCharacter = 0x2190; // Unicode Left Arrow - public const int EvalCompilationUnitParserCharacter = 0x2191; // Unicode Arrow - public const int EvalUsingDeclarationsParserCharacter = 0x2192; // Unicode Arrow + public const int EvalStatementParserCharacter = 0x100000; + public const int EvalCompilationUnitParserCharacter = 0x100001; + public const int EvalUsingDeclarationsParserCharacter = 0x100002; + public const int DocumentationXref = 0x100003; // // XML documentation buffer. The save point is used to divide @@ -231,7 +260,22 @@ namespace Mono.CSharp // This is needed because `define' is not allowed to be used // after a token has been seen. // - bool any_token_seen = false; + bool any_token_seen; + + // + // Class variables + // + static readonly KeywordEntry[][] keywords; + static readonly KeywordEntry[][] keywords_preprocessor; + static readonly Dictionary keyword_strings; // TODO: HashSet + static readonly NumberStyles styles; + static readonly NumberFormatInfo csharp_format_info; + + // Pragma arguments + static readonly char[] pragma_warning = "warning".ToCharArray (); + static readonly char[] pragma_warning_disable = "disable".ToCharArray (); + static readonly char[] pragma_warning_restore = "restore".ToCharArray (); + static readonly char[] pragma_checksum = "checksum".ToCharArray (); static readonly char[] simple_whitespaces = new char[] { ' ', '\t' }; @@ -283,25 +327,11 @@ namespace Mono.CSharp escaped_identifiers.Add (loc); } - public bool IsEscapedIdentifier (Location loc) + public bool IsEscapedIdentifier (ATypeNameExpression name) { - if (escaped_identifiers != null) { - foreach (Location lt in escaped_identifiers) - if (lt.Equals (loc)) - return true; - } - - return false; + return escaped_identifiers != null && escaped_identifiers.Contains (name.Location); } - // - // Class variables - // - static KeywordEntry[][] keywords; - static Dictionary keyword_strings; // TODO: HashSet - static NumberStyles styles; - static NumberFormatInfo csharp_format_info; - // // Values for the associated token returned // @@ -324,7 +354,7 @@ namespace Mono.CSharp static System.Text.StringBuilder string_builder; const int max_id_size = 512; - static char [] id_builder = new char [max_id_size]; + static readonly char [] id_builder = new char [max_id_size]; public static Dictionary[] identifiers = new Dictionary[max_id_size + 1]; @@ -332,19 +362,8 @@ namespace Mono.CSharp static char [] number_builder = new char [max_number_size]; static int number_pos; - static StringBuilder static_cmd_arg = new System.Text.StringBuilder (); - - // - // Details about the error encoutered by the tokenizer - // - string error_details; - - public string error { - get { - return error_details; - } - } - + static char[] value_builder = new char[256]; + public int Line { get { return ref_line; @@ -393,6 +412,26 @@ namespace Mono.CSharp val = t.val; } } + + public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, CompilerContext ctx) + { + this.ref_name = file; + this.file_name = file; + this.context = ctx; + reader = input; + + putback_char = -1; + + xml_comment_buffer = new StringBuilder (); + doc_processing = ctx.Settings.DocumentationFile != null; + + if (Environment.OSVersion.Platform == PlatformID.Win32NT) + tab_size = 4; + else + tab_size = 8; + + Mono.CSharp.Location.Push (file, file); + } public void PushPosition () { @@ -426,15 +465,25 @@ namespace Mono.CSharp { keyword_strings.Add (kw, null); + AddKeyword (keywords, kw, token); + } + + static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive) + { + AddKeyword (keywords_preprocessor, kw, directive); + } + + static void AddKeyword (KeywordEntry[][] keywords, string kw, T token) + { int length = kw.Length; - if (keywords [length] == null) { - keywords [length] = new KeywordEntry ['z' - '_' + 1]; + if (keywords[length] == null) { + keywords[length] = new KeywordEntry['z' - '_' + 1]; } - int char_index = kw [0] - '_'; - KeywordEntry kwe = keywords [length] [char_index]; + int char_index = kw[0] - '_'; + var kwe = keywords[length][char_index]; if (kwe == null) { - keywords [length] [char_index] = new KeywordEntry (kw, token); + keywords[length][char_index] = new KeywordEntry (kw, token); return; } @@ -442,17 +491,23 @@ namespace Mono.CSharp kwe = kwe.Next; } - kwe.Next = new KeywordEntry (kw, token); + kwe.Next = new KeywordEntry (kw, token); } - static void InitTokens () + // + // Class initializer + // + static Tokenizer () { keyword_strings = new Dictionary (); // 11 is the length of the longest keyword for now - keywords = new KeywordEntry [11] []; + keywords = new KeywordEntry[11][]; AddKeyword ("__arglist", Token.ARGLIST); + AddKeyword ("__makeref", Token.MAKEREF); + AddKeyword ("__reftype", Token.REFTYPE); + AddKeyword ("__refvalue", Token.REFVALUE); AddKeyword ("abstract", Token.ABSTRACT); AddKeyword ("as", Token.AS); AddKeyword ("add", Token.ADD); @@ -550,14 +605,26 @@ namespace Mono.CSharp AddKeyword ("ascending", Token.ASCENDING); AddKeyword ("descending", Token.DESCENDING); AddKeyword ("into", Token.INTO); - } - // - // Class initializer - // - static Tokenizer () - { - InitTokens (); + // Contextual async keywords + AddKeyword ("async", Token.ASYNC); + AddKeyword ("await", Token.AWAIT); + + keywords_preprocessor = new KeywordEntry[10][]; + + AddPreprocessorKeyword ("region", PreprocessorDirective.Region); + AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion); + AddPreprocessorKeyword ("if", PreprocessorDirective.If); + AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif); + AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif); + AddPreprocessorKeyword ("else", PreprocessorDirective.Else); + AddPreprocessorKeyword ("define", PreprocessorDirective.Define); + AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef); + AddPreprocessorKeyword ("error", PreprocessorDirective.Error); + AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning); + AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma); + AddPreprocessorKeyword ("line", PreprocessorDirective.Line); + csharp_format_info = NumberFormatInfo.InvariantInfo; styles = NumberStyles.Float; @@ -577,7 +644,7 @@ namespace Mono.CSharp if (first_index > 'z' - '_') return -1; - KeywordEntry kwe = keywords [id_len] [first_index]; + var kwe = keywords [id_len] [first_index]; if (kwe == null) return -1; @@ -656,8 +723,8 @@ namespace Mono.CSharp res = Token.FROM_FIRST; query_parsing = true; - if (RootContext.Version <= LanguageVersion.ISO_2) - Report.FeatureIsNotAvailable (Location, "query expressions"); + if (context.Settings.Version <= LanguageVersion.ISO_2) + Report.FeatureIsNotAvailable (context, Location, "query expressions"); break; case Token.VOID: Expression.Error_VoidInvalidInTheContext (Location, Report); @@ -712,11 +779,10 @@ namespace Mono.CSharp if (ok) { if (next_token == Token.VOID) { - if (RootContext.Version == LanguageVersion.ISO_1 || - RootContext.Version == LanguageVersion.ISO_2) - Report.FeatureIsNotAvailable (Location, "partial methods"); - } else if (RootContext.Version == LanguageVersion.ISO_1) - Report.FeatureIsNotAvailable (Location, "partial types"); + if (context.Settings.Version <= LanguageVersion.ISO_2) + Report.FeatureIsNotAvailable (context, Location, "partial methods"); + } else if (context.Settings.Version == LanguageVersion.ISO_1) + Report.FeatureIsNotAvailable (context, Location, "partial types"); return res; } @@ -729,38 +795,92 @@ namespace Mono.CSharp res = -1; break; - } - return res; - } + case Token.ASYNC: + if (parsing_modifiers) { + // + // Skip attributes section or constructor called async + // + if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) { + res = -1; + } else { + // async is keyword + } + } else if (parsing_block > 0) { + switch (peek_token ()) { + case Token.DELEGATE: + case Token.OPEN_PARENS_LAMBDA: + // async is keyword + break; + case Token.IDENTIFIER: + PushPosition (); + xtoken (); + if (xtoken () != Token.ARROW) + res = -1; - public Location Location { - get { - return new Location (ref_line, hidden ? -1 : col); + PopPosition (); + break; + default: + res = -1; + break; + } + } else { + res = -1; + } + + if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) { + Report.FeatureIsNotAvailable (context, Location, "asynchronous functions"); + } + + break; + + case Token.AWAIT: + if (parsing_block == 0) + res = -1; + + break; } + + + return res; } - public Tokenizer (SeekableStreamReader input, CompilationUnit file, CompilerContext ctx) + static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len) { - this.ref_name = file; - this.file_name = file; - this.context = ctx; - reader = input; - - putback_char = -1; + // + // Keywords are stored in an array of arrays grouped by their + // length and then by the first character + // + if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null) + return PreprocessorDirective.Invalid; - xml_comment_buffer = new StringBuilder (); + int first_index = id[0] - '_'; + if (first_index > 'z' - '_') + return PreprocessorDirective.Invalid; - if (Environment.OSVersion.Platform == PlatformID.Win32NT) - tab_size = 4; - else - tab_size = 8; + var kwe = keywords_preprocessor[id_len][first_index]; + if (kwe == null) + return PreprocessorDirective.Invalid; - // - // FIXME: This could be `Location.Push' but we have to - // find out why the MS compiler allows this - // - Mono.CSharp.Location.Push (file, file); + PreprocessorDirective res = PreprocessorDirective.Invalid; + do { + res = kwe.Token; + for (int i = 1; i < id_len; ++i) { + if (id[i] != kwe.Value[i]) { + res = 0; + kwe = kwe.Next; + break; + } + } + } while (res == PreprocessorDirective.Invalid && kwe != null); + + return res; + } + + public Location Location { + get { + return new Location (ref_line, hidden ? -1 : col); + } } static bool is_identifier_start_character (int c) @@ -878,6 +998,12 @@ namespace Mono.CSharp case Token.IDENTIFIER: switch (ptoken) { case Token.DOT: + if (bracket_level == 0) { + is_type = false; + can_be_type = true; + } + + continue; case Token.OP_GENERICS_LT: case Token.COMMA: case Token.DOUBLE_COLON: @@ -992,6 +1118,8 @@ namespace Mono.CSharp case Token.VOID: break; case Token.OP_GENERICS_GT: + case Token.IN: + case Token.OUT: return true; default: @@ -1103,6 +1231,7 @@ namespace Mono.CSharp case Token.CLOSE_PARENS: case Token.OPEN_BRACKET: case Token.OP_GENERICS_GT: + case Token.INTERR: next_token = Token.INTERR_NULLABLE; break; @@ -1205,7 +1334,7 @@ namespace Mono.CSharp } } - int integer_type_suffix (ulong ul, int c) + ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc) { bool is_unsigned = false; bool is_long = false; @@ -1248,40 +1377,38 @@ namespace Mono.CSharp } if (is_long && is_unsigned){ - val = new ULongLiteral (ul, Location); - return Token.LITERAL; + return new ULongLiteral (context.BuiltinTypes, ul, loc); } if (is_unsigned){ // uint if possible, or ulong else. if ((ul & 0xffffffff00000000) == 0) - val = new UIntLiteral ((uint) ul, Location); + return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc); else - val = new ULongLiteral (ul, Location); + return new ULongLiteral (context.BuiltinTypes, ul, loc); } else if (is_long){ // long if possible, ulong otherwise if ((ul & 0x8000000000000000) != 0) - val = new ULongLiteral (ul, Location); + return new ULongLiteral (context.BuiltinTypes, ul, loc); else - val = new LongLiteral ((long) ul, Location); + return new LongLiteral (context.BuiltinTypes, (long) ul, loc); } else { // int, uint, long or ulong in that order if ((ul & 0xffffffff00000000) == 0){ uint ui = (uint) ul; if ((ui & 0x80000000) != 0) - val = new UIntLiteral (ui, Location); + return new UIntLiteral (context.BuiltinTypes, ui, loc); else - val = new IntLiteral ((int) ui, Location); + return new IntLiteral (context.BuiltinTypes, (int) ui, loc); } else { if ((ul & 0x8000000000000000) != 0) - val = new ULongLiteral (ul, Location); + return new ULongLiteral (context.BuiltinTypes, ul, loc); else - val = new LongLiteral ((long) ul, Location); + return new LongLiteral (context.BuiltinTypes, (long) ul, loc); } } - return Token.LITERAL; } // @@ -1289,7 +1416,7 @@ namespace Mono.CSharp // we need to convert to a special type, and then choose // the best representation for the integer // - int adjust_int (int c) + ILiteralConstant adjust_int (int c, Location loc) { try { if (number_pos > 9){ @@ -1298,64 +1425,58 @@ namespace Mono.CSharp for (int i = 1; i < number_pos; i++){ ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0'))); } - return integer_type_suffix (ul, c); + + return integer_type_suffix (ul, c, loc); } else { uint ui = (uint) (number_builder [0] - '0'); for (int i = 1; i < number_pos; i++){ ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0'))); } - return integer_type_suffix (ui, c); + + return integer_type_suffix (ui, c, loc); } } catch (OverflowException) { - error_details = "Integral constant is too large"; - Report.Error (1021, Location, error_details); - val = new IntLiteral (0, Location); - return Token.LITERAL; + Error_NumericConstantTooLong (); + return new IntLiteral (context.BuiltinTypes, 0, loc); } catch (FormatException) { Report.Error (1013, Location, "Invalid number"); - val = new IntLiteral (0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } } - int adjust_real (TypeCode t) + ILiteralConstant adjust_real (TypeCode t, Location loc) { - string s = new String (number_builder, 0, number_pos); + string s = new string (number_builder, 0, number_pos); const string error_details = "Floating-point constant is outside the range of type `{0}'"; switch (t){ case TypeCode.Decimal: try { - val = new DecimalLiteral (decimal.Parse (s, styles, csharp_format_info), Location); + return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc); } catch (OverflowException) { - val = new DecimalLiteral (0, Location); Report.Error (594, Location, error_details, "decimal"); + return new DecimalLiteral (context.BuiltinTypes, 0, loc); } - break; case TypeCode.Single: try { - val = new FloatLiteral (float.Parse (s, styles, csharp_format_info), Location); + return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc); } catch (OverflowException) { - val = new FloatLiteral (0, Location); Report.Error (594, Location, error_details, "float"); + return new FloatLiteral (context.BuiltinTypes, 0, loc); } - break; default: try { - val = new DoubleLiteral (double.Parse (s, styles, csharp_format_info), Location); + return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc); } catch (OverflowException) { - val = new DoubleLiteral (0, Location); - Report.Error (594, Location, error_details, "double"); + Report.Error (594, loc, error_details, "double"); + return new DoubleLiteral (context.BuiltinTypes, 0, loc); } - break; } - - return Token.LITERAL; } - int handle_hex () + ILiteralConstant handle_hex (Location loc) { int d; ulong ul; @@ -1370,24 +1491,22 @@ namespace Mono.CSharp } string s = new String (number_builder, 0, number_pos); + try { if (number_pos <= 8) ul = System.UInt32.Parse (s, NumberStyles.HexNumber); else ul = System.UInt64.Parse (s, NumberStyles.HexNumber); + + return integer_type_suffix (ul, peek_char (), loc); } catch (OverflowException){ - error_details = "Integral constant is too large"; - Report.Error (1021, Location, error_details); - val = new IntLiteral (0, Location); - return Token.LITERAL; + Error_NumericConstantTooLong (); + return new IntLiteral (context.BuiltinTypes, 0, loc); } catch (FormatException) { Report.Error (1013, Location, "Invalid number"); - val = new IntLiteral (0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } - - return integer_type_suffix (ul, peek_char ()); } // @@ -1395,16 +1514,26 @@ namespace Mono.CSharp // int is_number (int c) { - bool is_real = false; + ILiteralConstant res; +#if FULL_AST + int read_start = reader.Position - 1; +#endif number_pos = 0; + var loc = Location; if (c >= '0' && c <= '9'){ if (c == '0'){ int peek = peek_char (); - if (peek == 'x' || peek == 'X') - return handle_hex (); + if (peek == 'x' || peek == 'X') { + val = res = handle_hex (loc); +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1); +#endif + + return Token.LITERAL; + } } decimal_digits (c); c = get_char (); @@ -1414,6 +1543,7 @@ namespace Mono.CSharp // We need to handle the case of // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER) // + bool is_real = false; if (c == '.'){ if (decimal_digits ('.')){ is_real = true; @@ -1421,7 +1551,12 @@ namespace Mono.CSharp } else { putback ('.'); number_pos--; - return adjust_int (-1); + val = res = adjust_int (-1, loc); + +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1); +#endif + return Token.LITERAL; } } @@ -1429,7 +1564,7 @@ namespace Mono.CSharp is_real = true; if (number_pos == max_number_size) Error_NumericConstantTooLong (); - number_builder [number_pos++] = 'e'; + number_builder [number_pos++] = (char) c; c = get_char (); if (c == '+'){ @@ -1453,21 +1588,26 @@ namespace Mono.CSharp } var type = real_type_suffix (c); - if (type == TypeCode.Empty && !is_real){ + if (type == TypeCode.Empty && !is_real) { putback (c); - return adjust_int (c); - } + res = adjust_int (c, loc); + } else { + is_real = true; - is_real = true; + if (type == TypeCode.Empty) { + putback (c); + } - if (type == TypeCode.Empty){ - putback (c); + res = adjust_real (type, loc); } - - if (is_real) - return adjust_real (type); - throw new Exception ("Is Number should never reach this point"); + val = res; + +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty ? 1 : 0)); +#endif + + return Token.LITERAL; } // @@ -1597,9 +1737,18 @@ namespace Mono.CSharp if (putback_char != -1) { x = putback_char; putback_char = -1; - } else + } else { x = reader.Read (); - if (x == '\n') { + } + + if (x == '\r') { + if (peek_char () == '\n') { + putback_char = -1; + } + + x = '\n'; + advance_line (); + } else if (x == '\n') { advance_line (); } else { col++; @@ -1671,21 +1820,17 @@ namespace Mono.CSharp return current_token; } - void get_cmd_arg (out string cmd, out string arg) + int TokenizePreprocessorIdentifier (out int c) { - int c; - - tokens_seen = false; - arg = ""; - // skip over white space do { c = get_char (); - } while (c == '\r' || c == ' ' || c == '\t'); + } while (c == ' ' || c == '\t'); - static_cmd_arg.Length = 0; - while (c != -1 && is_identifier_part_character ((char)c)) { - static_cmd_arg.Append ((char)c); + + int pos = 0; + while (c != -1 && c >= 'a' && c <= 'z') { + id_builder[pos++] = (char) c; c = get_char (); if (c == '\\') { int peek = peek_char (); @@ -1693,26 +1838,40 @@ namespace Mono.CSharp int surrogate; c = EscapeUnicode (c, out surrogate); if (surrogate != 0) { - if (is_identifier_part_character ((char) c)) - static_cmd_arg.Append ((char) c); + if (is_identifier_part_character ((char) c)) { + id_builder[pos++] = (char) c; + } c = surrogate; } } } } - cmd = static_cmd_arg.ToString (); + return pos; + } + + PreprocessorDirective get_cmd_arg (out string arg) + { + int c; + + tokens_seen = false; + arg = ""; + + var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorIdentifier (out c)); + + if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0) + return cmd; // skip over white space - while (c == '\r' || c == ' ' || c == '\t') + while (c == ' ' || c == '\t') c = get_char (); - static_cmd_arg.Length = 0; - int has_identifier_argument = 0; + int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument); + int pos = 0; - while (c != -1 && c != '\n' && c != '\r') { + while (c != -1 && c != '\n') { if (c == '\\' && has_identifier_argument >= 0) { - if (has_identifier_argument != 0 || (cmd == "define" || cmd == "if" || cmd == "elif" || cmd == "undef")) { + if (has_identifier_argument != 0) { has_identifier_argument = 1; int peek = peek_char (); @@ -1720,21 +1879,48 @@ namespace Mono.CSharp int surrogate; c = EscapeUnicode (c, out surrogate); if (surrogate != 0) { - if (is_identifier_part_character ((char) c)) - static_cmd_arg.Append ((char) c); + if (is_identifier_part_character ((char) c)) { + if (pos == value_builder.Length) + Array.Resize (ref value_builder, pos * 2); + + value_builder[pos++] = (char) c; + } c = surrogate; } } } else { has_identifier_argument = -1; } + } else if (c == '/' && peek_char () == '/') { + // + // Eat single-line comments + // + get_char (); + do { + c = get_char (); + } while (c != -1 && c != '\n'); + + break; } - static_cmd_arg.Append ((char) c); + + if (pos == value_builder.Length) + Array.Resize (ref value_builder, pos * 2); + + value_builder[pos++] = (char) c; c = get_char (); } - if (static_cmd_arg.Length != 0) - arg = static_cmd_arg.ToString (); + if (pos != 0) { + if (pos > max_id_size) + arg = new string (value_builder, 0, pos); + else + arg = InternIdentifier (value_builder, pos); + + // Eat any trailing whitespaces + arg = arg.Trim (simple_whitespaces); + } + + return cmd; } // @@ -1766,8 +1952,8 @@ namespace Mono.CSharp char [] quotes = { '\"' }; string name = arg.Substring (pos). Trim (quotes); - ref_name = Location.LookupFile (file_name, name); - file_name.AddFile (ref_name); + ref_name = context.LookupFile (file_name, name); + file_name.AddIncludeFile (ref_name); hidden = false; Location.Push (file_name, ref_name); } else { @@ -1813,7 +1999,7 @@ namespace Mono.CSharp // // #define ident // - if (RootContext.IsConditionalDefined (ident)) + if (context.Settings.IsConditionalSymbolDefined (ident)) return; file_name.AddDefine (ident); @@ -1825,12 +2011,10 @@ namespace Mono.CSharp } } - static byte read_hex (string arg, int pos, out bool error) + byte read_hex (out bool error) { - error = false; - int total; - char c = arg [pos]; + int c = get_char (); if ((c >= '0') && (c <= '9')) total = (int) c - (int) '0'; @@ -1844,7 +2028,7 @@ namespace Mono.CSharp } total *= 16; - c = arg [pos+1]; + c = get_char (); if ((c >= '0') && (c <= '9')) total += (int) c - (int) '0'; @@ -1857,164 +2041,262 @@ namespace Mono.CSharp return 0; } + error = false; return (byte) total; } - /// - /// Handles #pragma checksum - /// - bool PreProcessPragmaChecksum (string arg) + // + // Parses #pragma checksum + // + bool ParsePragmaChecksum () { - if ((arg [0] != ' ') && (arg [0] != '\t')) - return false; + // + // The syntax is ` "foo.txt" "{guid}" "hash"' + // + int c = get_char (); - arg = arg.Trim (simple_whitespaces); - if ((arg.Length < 2) || (arg [0] != '"')) + if (c != '"') return false; - StringBuilder file_sb = new StringBuilder (); - - int pos = 1; - char ch; - while ((ch = arg [pos++]) != '"') { - if (pos >= arg.Length) - return false; - - if (ch == '\\') { - if (pos+1 >= arg.Length) - return false; - ch = arg [pos++]; + string_builder.Length = 0; + while (c != -1 && c != '\n') { + c = get_char (); + if (c == '"') { + c = get_char (); + break; } - file_sb.Append (ch); + string_builder.Append ((char) c); + } + + if (string_builder.Length == 0) { + Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty"); } - if ((pos+2 >= arg.Length) || ((arg [pos] != ' ') && (arg [pos] != '\t'))) + // TODO: Any white-spaces count + if (c != ' ') return false; - arg = arg.Substring (pos).Trim (simple_whitespaces); - if ((arg.Length < 42) || (arg [0] != '"') || (arg [1] != '{') || - (arg [10] != '-') || (arg [15] != '-') || (arg [20] != '-') || - (arg [25] != '-') || (arg [38] != '}') || (arg [39] != '"')) + SourceFile file = context.LookupFile (file_name, string_builder.ToString ()); + + if (get_char () != '"' || get_char () != '{') return false; bool error; byte[] guid_bytes = new byte [16]; + int i = 0; - for (int i = 0; i < 4; i++) { - guid_bytes [i] = read_hex (arg, 2+2*i, out error); + for (; i < 4; i++) { + guid_bytes [i] = read_hex (out error); if (error) return false; } - for (int i = 0; i < 2; i++) { - guid_bytes [i+4] = read_hex (arg, 11+2*i, out error); + + if (get_char () != '-') + return false; + + for (; i < 10; i++) { + guid_bytes [i] = read_hex (out error); if (error) return false; - guid_bytes [i+6] = read_hex (arg, 16+2*i, out error); + + guid_bytes [i++] = read_hex (out error); if (error) return false; - guid_bytes [i+8] = read_hex (arg, 21+2*i, out error); - if (error) + + if (get_char () != '-') return false; } - for (int i = 0; i < 6; i++) { - guid_bytes [i+10] = read_hex (arg, 26+2*i, out error); + for (; i < 16; i++) { + guid_bytes [i] = read_hex (out error); if (error) return false; } - arg = arg.Substring (40).Trim (simple_whitespaces); - if ((arg.Length < 34) || (arg [0] != '"') || (arg [33] != '"')) + if (get_char () != '}' || get_char () != '"') + return false; + + // TODO: Any white-spaces count + c = get_char (); + if (c != ' ') + return false; + + if (get_char () != '"') return false; - byte[] checksum_bytes = new byte [16]; - for (int i = 0; i < 16; i++) { - checksum_bytes [i] = read_hex (arg, 1+2*i, out error); + // Any length of checksum + List checksum_bytes = new List (16); + + c = peek_char (); + while (c != '"' && c != -1) { + checksum_bytes.Add (read_hex (out error)); if (error) return false; + + c = peek_char (); } - arg = arg.Substring (34).Trim (simple_whitespaces); - if (arg.Length > 0) + if (c == '/') { + ReadSingleLineComment (); + } else if (get_char () != '"') { return false; + } - SourceFile file = Location.LookupFile (file_name, file_sb.ToString ()); - file.SetChecksum (guid_bytes, checksum_bytes); + file.SetChecksum (guid_bytes, checksum_bytes.ToArray ()); ref_name.AutoGenerated = true; return true; } + bool IsTokenIdentifierEqual (char[] identifier) + { + for (int i = 0; i < identifier.Length; ++i) { + if (identifier[i] != id_builder[i]) + return false; + } + + return true; + } + + int TokenizePragmaNumber (ref int c) + { + number_pos = 0; + + int number; + + if (c >= '0' && c <= '9') { + decimal_digits (c); + uint ui = (uint) (number_builder[0] - '0'); + + try { + for (int i = 1; i < number_pos; i++) { + ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0'))); + } + + number = (int) ui; + } catch (OverflowException) { + Error_NumericConstantTooLong (); + number = -1; + } + + + c = get_char (); + + // skip over white space + while (c == ' ' || c == '\t') + c = get_char (); + + if (c == ',') { + c = get_char (); + } + + // skip over white space + while (c == ' ' || c == '\t') + c = get_char (); + } else { + number = -1; + if (c == '/') { + ReadSingleLineComment (); + } else { + Report.Warning (1692, 1, Location, "Invalid number"); + + // Read everything till the end of the line or file + do { + c = get_char (); + } while (c != -1 && c != '\n'); + } + } + + return number; + } + + void ReadSingleLineComment () + { + if (peek_char () != '/') + Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected"); + + // Read everything till the end of the line or file + int c; + do { + c = get_char (); + } while (c != -1 && c != '\n'); + } + /// /// Handles #pragma directive /// - void PreProcessPragma (string arg) + void ParsePragmaDirective (string arg) { - const string warning = "warning"; - const string w_disable = "warning disable"; - const string w_restore = "warning restore"; - const string checksum = "checksum"; + int c; + int length = TokenizePreprocessorIdentifier (out c); + if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) { + length = TokenizePreprocessorIdentifier (out c); - if (arg == w_disable) { - Report.RegisterWarningRegion (Location).WarningDisable (Location.Row); - return; - } + // + // #pragma warning disable + // #pragma warning restore + // + if (length == pragma_warning_disable.Length) { + bool disable = IsTokenIdentifierEqual (pragma_warning_disable); + if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) { + // skip over white space + while (c == ' ' || c == '\t') + c = get_char (); - if (arg == w_restore) { - Report.RegisterWarningRegion (Location).WarningEnable (Location.Row); - return; - } + var loc = Location; - if (arg.StartsWith (w_disable)) { - int[] codes = ParseNumbers (arg.Substring (w_disable.Length)); - foreach (int code in codes) { - if (code != 0) - Report.RegisterWarningRegion (Location).WarningDisable (Location, code, Report); - } - return; - } + if (c == '\n' || c == '/') { + if (c == '/') + ReadSingleLineComment (); - if (arg.StartsWith (w_restore)) { - int[] codes = ParseNumbers (arg.Substring (w_restore.Length)); - var w_table = Report.warning_ignore_table; - foreach (int code in codes) { - if (w_table != null && w_table.ContainsKey (code)) - Report.Warning (1635, 1, Location, "Cannot restore warning `CS{0:0000}' because it was disabled globally", code); - Report.RegisterWarningRegion (Location).WarningEnable (Location, code, Report); + // + // Disable/Restore all warnings + // + if (disable) { + Report.RegisterWarningRegion (loc).WarningDisable (loc.Row); + } else { + Report.RegisterWarningRegion (loc).WarningEnable (loc.Row); + } + } else { + // + // Disable/Restore a warning or group of warnings + // + int code; + do { + code = TokenizePragmaNumber (ref c); + if (code > 0) { + if (disable) { + Report.RegisterWarningRegion (loc).WarningDisable (loc, code, Report); + } else { + Report.RegisterWarningRegion (loc).WarningEnable (loc, code, Report); + } + } + } while (code >= 0 && c != '\n' && c != -1); + } + + return; + } } - return; - } - if (arg.StartsWith (warning)) { Report.Warning (1634, 1, Location, "Expected disable or restore"); return; } - if (arg.StartsWith (checksum)) { - if (!PreProcessPragmaChecksum (arg.Substring (checksum.Length))) - Warning_InvalidPragmaChecksum (); + // + // #pragma checksum + // + if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) { + if (c != ' ' || !ParsePragmaChecksum ()) { + Report.Warning (1695, 1, Location, + "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\""); + } + return; } Report.Warning (1633, 1, Location, "Unrecognized #pragma directive"); } - int[] ParseNumbers (string text) - { - string[] string_array = text.Split (','); - int[] values = new int [string_array.Length]; - int index = 0; - foreach (string string_code in string_array) { - try { - values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture); - } - catch (FormatException) { - Report.Warning (1692, 1, Location, "Invalid number"); - } - } - return values; - } - bool eval_val (string s) { if (s == "true") @@ -2022,7 +2304,7 @@ namespace Mono.CSharp if (s == "false") return false; - return file_name.IsConditionalDefined (s); + return file_name.IsConditionalDefined (context, s); } bool pp_primary (ref string s) @@ -2177,7 +2459,7 @@ namespace Mono.CSharp void Error_NumericConstantTooLong () { - Report.Error (1021, Location, "Numeric constant too long"); + Report.Error (1021, Location, "Integral constant is too large"); } void Error_InvalidDirective () @@ -2208,42 +2490,42 @@ namespace Mono.CSharp { Report.Error (1025, Location, "Single-line comment or end-of-line expected"); } - - void Warning_InvalidPragmaChecksum () + + // + // Raises a warning when tokenizer found documentation comment + // on unexpected place + // + void WarningMisplacedComment (Location loc) { - Report.Warning (1695, 1, Location, - "Invalid #pragma checksum syntax; should be " + - "#pragma checksum \"filename\" " + - "\"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\""); + if (doc_state != XmlCommentState.Error) { + doc_state = XmlCommentState.Error; + Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element"); + } } + // // if true, then the code continues processing the code // if false, the code stays in a loop until another directive is // reached. // When caller_is_taking is false we ignore all directives except the ones // which can help us to identify where the #if block ends - bool handle_preprocessing_directive (bool caller_is_taking) + bool ParsePreprocessingDirective (bool caller_is_taking) { - string cmd, arg; + string arg; bool region_directive = false; - get_cmd_arg (out cmd, out arg); - - // Eat any trailing whitespaces and single-line comments - if (arg.IndexOf ("//") != -1) - arg = arg.Substring (0, arg.IndexOf ("//")); - arg = arg.Trim (simple_whitespaces); + var directive = get_cmd_arg (out arg); // // The first group of pre-processing instructions is always processed // - switch (cmd){ - case "region": + switch (directive) { + case PreprocessorDirective.Region: region_directive = true; arg = "true"; - goto case "if"; + goto case PreprocessorDirective.If; - case "endregion": + case PreprocessorDirective.Endregion: if (ifstack == null || ifstack.Count == 0){ Error_UnexpectedDirective ("no #region for this #endregion"); return true; @@ -2255,7 +2537,7 @@ namespace Mono.CSharp return caller_is_taking; - case "if": + case PreprocessorDirective.If: if (ifstack == null) ifstack = new Stack (2); @@ -2269,14 +2551,14 @@ namespace Mono.CSharp } } - if (caller_is_taking && eval (arg)) { + if (eval (arg) && caller_is_taking) { ifstack.Push (flags | TAKING); return true; } ifstack.Push (flags); return false; - - case "endif": + + case PreprocessorDirective.Endif: if (ifstack == null || ifstack.Count == 0){ Error_UnexpectedDirective ("no #if for this #endif"); return true; @@ -2297,7 +2579,7 @@ namespace Mono.CSharp return (state & TAKING) != 0; } - case "elif": + case PreprocessorDirective.Elif: if (ifstack == null || ifstack.Count == 0){ Error_UnexpectedDirective ("no #if for this #elif"); return true; @@ -2328,7 +2610,7 @@ namespace Mono.CSharp return false; } - case "else": + case PreprocessorDirective.Else: if (ifstack == null || ifstack.Count == 0){ Error_UnexpectedDirective ("no #if for this #else"); return true; @@ -2366,7 +2648,7 @@ namespace Mono.CSharp return ret; } - case "define": + case PreprocessorDirective.Define: if (any_token_seen){ Error_TokensSeen (); return caller_is_taking; @@ -2374,13 +2656,17 @@ namespace Mono.CSharp PreProcessDefinition (true, arg, caller_is_taking); return caller_is_taking; - case "undef": + case PreprocessorDirective.Undef: if (any_token_seen){ Error_TokensSeen (); return caller_is_taking; } PreProcessDefinition (false, arg, caller_is_taking); return caller_is_taking; + + case PreprocessorDirective.Invalid: + Report.Error (1024, Location, "Wrong preprocessor directive"); + return true; } // @@ -2389,25 +2675,24 @@ namespace Mono.CSharp if (!caller_is_taking) return false; - switch (cmd){ - case "error": + switch (directive){ + case PreprocessorDirective.Error: Report.Error (1029, Location, "#error: '{0}'", arg); return true; - case "warning": + case PreprocessorDirective.Warning: Report.Warning (1030, 1, Location, "#warning: `{0}'", arg); return true; - case "pragma": - if (RootContext.Version == LanguageVersion.ISO_1) { - Report.FeatureIsNotAvailable (Location, "#pragma"); - return true; + case PreprocessorDirective.Pragma: + if (context.Settings.Version == LanguageVersion.ISO_1) { + Report.FeatureIsNotAvailable (context, Location, "#pragma"); } - PreProcessPragma (arg); + ParsePragmaDirective (arg); return true; - case "line": + case PreprocessorDirective.Line: if (!PreProcessLine (arg)) Report.Error ( 1576, Location, @@ -2415,48 +2700,80 @@ namespace Mono.CSharp return caller_is_taking; } - Report.Error (1024, Location, "Wrong preprocessor directive"); - return true; - + throw new NotImplementedException (directive.ToString ()); } private int consume_string (bool quoted) { int c; - string_builder.Length = 0; + int pos = 0; + Location start_location = Location; + if (quoted) + start_location = start_location - 1; + +#if FULL_AST + int reader_pos = reader.Position; +#endif + + while (true){ + c = get_char (); + if (c == '"') { + if (quoted && peek_char () == '"') { + if (pos == value_builder.Length) + Array.Resize (ref value_builder, pos * 2); - while ((c = get_char ()) != -1){ - if (c == '"'){ - if (quoted && peek_char () == '"'){ - string_builder.Append ((char) c); + value_builder[pos++] = (char) c; get_char (); continue; - } else { - val = new StringLiteral (string_builder.ToString (), Location); - return Token.LITERAL; } - } - if (c == '\n'){ - if (!quoted) - Report.Error (1010, Location, "Newline in constant"); + string s; + if (pos == 0) + s = string.Empty; + else if (pos <= 4) + s = InternIdentifier (value_builder, pos); + else + s = new string (value_builder, 0, pos); + + ILiteralConstant res = new StringLiteral (context.BuiltinTypes, s, start_location); + val = res; +#if FULL_AST + res.ParsedValue = quoted ? + reader.ReadChars (reader_pos - 2, reader.Position - 1) : + reader.ReadChars (reader_pos - 1, reader.Position); +#endif + + return Token.LITERAL; } - if (!quoted){ + if (c == '\n') { + if (!quoted) { + Report.Error (1010, Location, "Newline in constant"); + val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location); + return Token.LITERAL; + } + } else if (c == '\\' && !quoted) { int surrogate; c = escape (c, out surrogate); if (c == -1) return Token.ERROR; if (surrogate != 0) { - string_builder.Append ((char) c); + if (pos == value_builder.Length) + Array.Resize (ref value_builder, pos * 2); + + value_builder[pos++] = (char) c; c = surrogate; } + } else if (c == -1) { + Report.Error (1039, Location, "Unterminated string literal"); + return Token.EOF; } - string_builder.Append ((char) c); - } - Report.Error (1039, Location, "Unterminated string literal"); - return Token.EOF; + if (pos == value_builder.Length) + Array.Resize (ref value_builder, pos * 2); + + value_builder[pos++] = (char) c; + } } private int consume_identifier (int s) @@ -2478,6 +2795,8 @@ namespace Mono.CSharp int pos = 0; int column = col; + if (quoted) + --column; if (c == '\\') { int surrogate; @@ -2503,9 +2822,10 @@ namespace Mono.CSharp if (c == '\\') { int surrogate; c = escape (c, out surrogate); + if (is_identifier_part_character ((char) c)) + id_builder[pos++] = (char) c; + if (surrogate != 0) { - if (is_identifier_part_character ((char) c)) - id_builder[pos++] = (char) c; c = surrogate; } @@ -2521,8 +2841,8 @@ namespace Mono.CSharp } } catch (IndexOutOfRangeException) { Report.Error (645, Location, "Identifier too long (limit is 512 chars)"); - col += pos - 1; - return Token.ERROR; + --pos; + col += pos; } col += pos - 1; @@ -2534,43 +2854,45 @@ namespace Mono.CSharp if (id_builder [0] >= '_' && !quoted) { int keyword = GetKeyword (id_builder, pos); if (keyword != -1) { - val = LocatedToken.Create (null, ref_line, column); + val = LocatedToken.Create (keyword == Token.AWAIT ? "await" : null, ref_line, column); return keyword; } } + string s = InternIdentifier (id_builder, pos); + val = LocatedToken.Create (s, ref_line, column); + if (quoted && parsing_attribute_section) + AddEscapedIdentifier (((LocatedToken) val).Location); + + return Token.IDENTIFIER; + } + + static string InternIdentifier (char[] charBuffer, int length) + { // // Keep identifiers in an array of hashtables to avoid needless // allocations // - var identifiers_group = identifiers [pos]; + var identifiers_group = identifiers[length]; string s; if (identifiers_group != null) { - if (identifiers_group.TryGetValue (id_builder, out s)) { - val = LocatedToken.Create (s, ref_line, column); - if (quoted) - AddEscapedIdentifier (((LocatedToken) val).Location); - return Token.IDENTIFIER; + if (identifiers_group.TryGetValue (charBuffer, out s)) { + return s; } } else { // TODO: this should be number of files dependant // corlib compilation peaks at 1000 and System.Core at 150 - int capacity = pos > 20 ? 10 : 100; - identifiers_group = new Dictionary (capacity, new IdentifiersComparer (pos)); - identifiers [pos] = identifiers_group; + int capacity = length > 20 ? 10 : 100; + identifiers_group = new Dictionary (capacity, new IdentifiersComparer (length)); + identifiers[length] = identifiers_group; } - char [] chars = new char [pos]; - Array.Copy (id_builder, chars, pos); + char[] chars = new char[length]; + Array.Copy (charBuffer, chars, length); - s = new string (id_builder, 0, pos); + s = new string (charBuffer, 0, length); identifiers_group.Add (chars, s); - - val = LocatedToken.Create (s, ref_line, column); - if (quoted) - AddEscapedIdentifier (((LocatedToken) val).Location); - - return Token.IDENTIFIER; + return s; } public int xtoken () @@ -2582,7 +2904,7 @@ namespace Mono.CSharp while ((c = get_char ()) != -1) { switch (c) { case '\t': - col = ((col + tab_size) / tab_size) * tab_size; + col = ((col - 1 + tab_size) / tab_size) * tab_size; continue; case ' ': @@ -2604,17 +2926,6 @@ namespace Mono.CSharp } break; */ - case '\r': - if (peek_char () != '\n') - advance_line (); - else - get_char (); - - any_token_seen |= tokens_seen; - tokens_seen = false; - comments_seen = false; - continue; - case '\\': tokens_seen = true; return consume_identifier (c); @@ -2656,7 +2967,7 @@ namespace Mono.CSharp return Token.OPEN_BRACKET_EXPR; } case ']': - val = LocatedToken.CreateOptional (ref_line, col); + LocatedToken.CreateOptional (ref_line, col, ref val); return Token.CLOSE_BRACKET; case '(': val = LocatedToken.Create (ref_line, col); @@ -2704,13 +3015,13 @@ namespace Mono.CSharp return Token.OPEN_PARENS; case ')': - val = LocatedToken.CreateOptional (ref_line, col); + LocatedToken.CreateOptional (ref_line, col, ref val); return Token.CLOSE_PARENS; case ',': - val = LocatedToken.CreateOptional (ref_line, col); + LocatedToken.CreateOptional (ref_line, col, ref val); return Token.COMMA; case ';': - val = LocatedToken.CreateOptional (ref_line, col); + LocatedToken.CreateOptional (ref_line, col, ref val); return Token.SEMICOLON; case '~': val = LocatedToken.Create (ref_line, col); @@ -2847,18 +3158,23 @@ namespace Mono.CSharp // Handle double-slash comments. if (d == '/'){ get_char (); - if (RootContext.Documentation != null && peek_char () == '/') { - get_char (); - // Don't allow ////. - if ((d = peek_char ()) != '/') { - update_comment_location (); - if (doc_state == XmlCommentState.Allowed) - handle_one_line_xml_comment (); - else if (doc_state == XmlCommentState.NotAllowed) - warn_incorrect_doc_comment (); + if (doc_processing) { + if (peek_char () == '/') { + get_char (); + // Don't allow ////. + if ((d = peek_char ()) != '/') { + if (doc_state == XmlCommentState.Allowed) + handle_one_line_xml_comment (); + else if (doc_state == XmlCommentState.NotAllowed) + WarningMisplacedComment (Location - 3); + } + } else { + if (xml_comment_buffer.Length > 0) + doc_state = XmlCommentState.NotAllowed; } } - while ((d = get_char ()) != -1 && (d != '\n') && d != '\r'); + + while ((d = get_char ()) != -1 && d != '\n'); any_token_seen |= tokens_seen; tokens_seen = false; @@ -2867,9 +3183,8 @@ namespace Mono.CSharp } else if (d == '*'){ get_char (); bool docAppend = false; - if (RootContext.Documentation != null && peek_char () == '*') { + if (doc_processing && peek_char () == '*') { get_char (); - update_comment_location (); // But when it is /**/, just do nothing. if (peek_char () == '/') { get_char (); @@ -2877,8 +3192,9 @@ namespace Mono.CSharp } if (doc_state == XmlCommentState.Allowed) docAppend = true; - else if (doc_state == XmlCommentState.NotAllowed) - warn_incorrect_doc_comment (); + else if (doc_state == XmlCommentState.NotAllowed) { + WarningMisplacedComment (Location - 2); + } } int current_comment_start = 0; @@ -2913,6 +3229,7 @@ namespace Mono.CSharp update_formatted_doc_comment (current_comment_start); continue; } + val = LocatedToken.Create (ref_line, col); return Token.DIV; case '%': @@ -2956,7 +3273,7 @@ namespace Mono.CSharp if (d >= '0' && d <= '9') return is_number (c); - val = LocatedToken.CreateOptional (ref_line, col); + LocatedToken.CreateOptional (ref_line, col, ref val); return Token.DOT; case '#': @@ -2965,7 +3282,7 @@ namespace Mono.CSharp return Token.ERROR; } - if (handle_preprocessing_directive (true)) + if (ParsePreprocessingDirective (true)) continue; bool directive_expected = false; @@ -2981,11 +3298,11 @@ namespace Mono.CSharp continue; } - if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' ) + if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' ) continue; if (c == '#') { - if (handle_preprocessing_directive (false)) + if (ParsePreprocessingDirective (false)) break; } directive_expected = false; @@ -3024,6 +3341,8 @@ namespace Mono.CSharp return Token.EVAL_COMPILATION_UNIT_PARSER; case EvalUsingDeclarationsParserCharacter: return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER; + case DocumentationXref: + return Token.DOC_SEE; } if (is_identifier_start_character (c)) { @@ -3031,8 +3350,10 @@ namespace Mono.CSharp return consume_identifier (c); } - error_details = ((char)c).ToString (); - return Token.ERROR; + if (char.IsWhiteSpace ((char) c)) + continue; + + Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ()); } if (CompleteOnEOF){ @@ -3049,15 +3370,20 @@ namespace Mono.CSharp int TokenizeBackslash () { +#if FULL_AST + int read_start = reader.Position; +#endif + Location start_location = Location; int c = get_char (); tokens_seen = true; if (c == '\'') { - error_details = "Empty character literal"; - Report.Error (1011, Location, error_details); - return Token.ERROR; + val = new CharLiteral (context.BuiltinTypes, (char) c, start_location); + Report.Error (1011, start_location, "Empty character literal"); + return Token.LITERAL; } - if (c == '\r' || c == '\n') { - Report.Error (1010, Location, "Newline in constant"); + + if (c == '\n') { + Report.Error (1010, start_location, "Newline in constant"); return Token.ERROR; } @@ -3068,20 +3394,24 @@ namespace Mono.CSharp if (d != 0) throw new NotImplementedException (); - val = new CharLiteral ((char) c, Location); + ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location); + val = res; c = get_char (); if (c != '\'') { - Report.Error (1012, Location, "Too many characters in character literal"); + Report.Error (1012, start_location, "Too many characters in character literal"); // Try to recover, read until newline or next "'" while ((c = get_char ()) != -1) { if (c == '\n' || c == '\'') break; } - return Token.ERROR; } +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position); +#endif + return Token.LITERAL; } @@ -3101,7 +3431,7 @@ namespace Mono.CSharp // Save current position and parse next token. PushPosition (); if (parse_less_than ()) { - if (parsing_generic_declaration && token () != Token.DOT) { + if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) { d = Token.OP_GENERICS_LT_DECL; } else { d = Token.OP_GENERICS_LT; @@ -3178,18 +3508,6 @@ namespace Mono.CSharp xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines)); } - // - // Updates current comment location. - // - private void update_comment_location () - { - if (current_comment_location.IsNull) { - // "-2" is for heading "//" or "/*" - current_comment_location = - new Location (ref_line, hidden ? -1 : col - 2); - } - } - // // Checks if there was incorrect doc comments and raise // warnings. @@ -3197,22 +3515,7 @@ namespace Mono.CSharp public void check_incorrect_doc_comment () { if (xml_comment_buffer.Length > 0) - warn_incorrect_doc_comment (); - } - - // - // Raises a warning when tokenizer found incorrect doccomment - // markup. - // - private void warn_incorrect_doc_comment () - { - if (doc_state != XmlCommentState.Error) { - doc_state = XmlCommentState.Error; - // in csc, it is 'XML comment is not placed on - // a valid language element'. But that does not - // make sense. - Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element"); - } + WarningMisplacedComment (Location); } // @@ -3236,7 +3539,6 @@ namespace Mono.CSharp void reset_doc_comment () { xml_comment_buffer.Length = 0; - current_comment_location = Location.Null; } public void cleanup ()