X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mcs%2Fmcs%2Fcs-tokenizer.cs;h=2dccc46414da9f88d981f042d186a30fb6433351;hb=8e33e0d088c564d039dd791b7cc3e121ff87b3e4;hp=f4d4ad5b43cb43b5a8efb61844e4a7d0e4b7288e;hpb=7ef6f8144f0ff1d19ca1c824af2bac3f591ba089;p=mono.git diff --git a/mcs/mcs/cs-tokenizer.cs b/mcs/mcs/cs-tokenizer.cs index f4d4ad5b43c..2dccc46414d 100644 --- a/mcs/mcs/cs-tokenizer.cs +++ b/mcs/mcs/cs-tokenizer.cs @@ -3,13 +3,13 @@ // This also implements the preprocessor // // Author: Miguel de Icaza (miguel@gnu.org) -// Marek Safar (marek.safar@seznam.cz) +// Marek Safar (marek.safar@gmail.com) // // Dual licensed under the terms of the MIT X11 or GNU GPL // // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com) // Copyright 2004-2008 Novell, Inc -// +// Copyright 2011 Xamarin, Inc (http://www.xamarin.com) // using System; @@ -87,6 +87,11 @@ namespace Mono.CSharp { return Create (null, row, column); } + + public static LocatedToken Create (string value, Location loc) + { + return Create (value, loc.Row, loc.Column); + } public static LocatedToken Create (string value, int row, int column) { @@ -141,7 +146,7 @@ namespace Mono.CSharp } } - enum PreprocessorDirective + public enum PreprocessorDirective { Invalid = 0, @@ -209,6 +214,8 @@ namespace Mono.CSharp public bool parsing_attribute_section; + public bool parsing_modifiers; + // // The special characters to inject on streams to run the unit parser // in the special expression mode. Using private characters from @@ -320,7 +327,7 @@ namespace Mono.CSharp escaped_identifiers.Add (loc); } - public bool IsEscapedIdentifier (MemberName name) + public bool IsEscapedIdentifier (ATypeNameExpression name) { return escaped_identifiers != null && escaped_identifiers.Contains (name.Location); } @@ -423,10 +430,6 @@ namespace Mono.CSharp else tab_size = 8; - // - // FIXME: This could be `Location.Push' but we have to - // find out why the MS compiler allows this - // Mono.CSharp.Location.Push (file, file); } @@ -793,18 +796,47 @@ namespace Mono.CSharp res = -1; break; - // TODO: async, it's modifiers context only case Token.ASYNC: - if (parsing_block > 0 || context.Settings.Version != LanguageVersion.Future) { + if (parsing_modifiers) { + // + // Skip attributes section or constructor called async + // + if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) { + res = -1; + } else { + // async is keyword + } + } else if (parsing_block > 0) { + switch (peek_token ()) { + case Token.DELEGATE: + case Token.OPEN_PARENS_LAMBDA: + // async is keyword + break; + case Token.IDENTIFIER: + PushPosition (); + xtoken (); + if (xtoken () != Token.ARROW) + res = -1; + + PopPosition (); + break; + default: + res = -1; + break; + } + } else { res = -1; } + + if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) { + Report.FeatureIsNotAvailable (context, Location, "asynchronous functions"); + } + break; - // TODO: async, it's async block context only case Token.AWAIT: - if (context.Settings.Version != LanguageVersion.Future) { + if (parsing_block == 0) res = -1; - } break; } @@ -1086,6 +1118,8 @@ namespace Mono.CSharp case Token.VOID: break; case Token.OP_GENERICS_GT: + case Token.IN: + case Token.OUT: return true; default: @@ -1300,7 +1334,7 @@ namespace Mono.CSharp } } - int integer_type_suffix (ulong ul, int c) + ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc) { bool is_unsigned = false; bool is_long = false; @@ -1343,40 +1377,38 @@ namespace Mono.CSharp } if (is_long && is_unsigned){ - val = new ULongLiteral (context.BuiltinTypes, ul, Location); - return Token.LITERAL; + return new ULongLiteral (context.BuiltinTypes, ul, loc); } if (is_unsigned){ // uint if possible, or ulong else. if ((ul & 0xffffffff00000000) == 0) - val = new UIntLiteral (context.BuiltinTypes, (uint) ul, Location); + return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc); else - val = new ULongLiteral (context.BuiltinTypes, ul, Location); + return new ULongLiteral (context.BuiltinTypes, ul, loc); } else if (is_long){ // long if possible, ulong otherwise if ((ul & 0x8000000000000000) != 0) - val = new ULongLiteral (context.BuiltinTypes, ul, Location); + return new ULongLiteral (context.BuiltinTypes, ul, loc); else - val = new LongLiteral (context.BuiltinTypes, (long) ul, Location); + return new LongLiteral (context.BuiltinTypes, (long) ul, loc); } else { // int, uint, long or ulong in that order if ((ul & 0xffffffff00000000) == 0){ uint ui = (uint) ul; if ((ui & 0x80000000) != 0) - val = new UIntLiteral (context.BuiltinTypes, ui, Location); + return new UIntLiteral (context.BuiltinTypes, ui, loc); else - val = new IntLiteral (context.BuiltinTypes, (int) ui, Location); + return new IntLiteral (context.BuiltinTypes, (int) ui, loc); } else { if ((ul & 0x8000000000000000) != 0) - val = new ULongLiteral (context.BuiltinTypes, ul, Location); + return new ULongLiteral (context.BuiltinTypes, ul, loc); else - val = new LongLiteral (context.BuiltinTypes, (long) ul, Location); + return new LongLiteral (context.BuiltinTypes, (long) ul, loc); } } - return Token.LITERAL; } // @@ -1384,7 +1416,7 @@ namespace Mono.CSharp // we need to convert to a special type, and then choose // the best representation for the integer // - int adjust_int (int c) + ILiteralConstant adjust_int (int c, Location loc) { try { if (number_pos > 9){ @@ -1393,63 +1425,58 @@ namespace Mono.CSharp for (int i = 1; i < number_pos; i++){ ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0'))); } - return integer_type_suffix (ul, c); + + return integer_type_suffix (ul, c, loc); } else { uint ui = (uint) (number_builder [0] - '0'); for (int i = 1; i < number_pos; i++){ ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0'))); } - return integer_type_suffix (ui, c); + + return integer_type_suffix (ui, c, loc); } } catch (OverflowException) { Error_NumericConstantTooLong (); - val = new IntLiteral (context.BuiltinTypes, 0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } catch (FormatException) { Report.Error (1013, Location, "Invalid number"); - val = new IntLiteral (context.BuiltinTypes, 0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } } - int adjust_real (TypeCode t) + ILiteralConstant adjust_real (TypeCode t, Location loc) { - string s = new String (number_builder, 0, number_pos); + string s = new string (number_builder, 0, number_pos); const string error_details = "Floating-point constant is outside the range of type `{0}'"; switch (t){ case TypeCode.Decimal: try { - val = new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), Location); + return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc); } catch (OverflowException) { - val = new DecimalLiteral (context.BuiltinTypes, 0, Location); Report.Error (594, Location, error_details, "decimal"); + return new DecimalLiteral (context.BuiltinTypes, 0, loc); } - break; case TypeCode.Single: try { - val = new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), Location); + return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc); } catch (OverflowException) { - val = new FloatLiteral (context.BuiltinTypes, 0, Location); Report.Error (594, Location, error_details, "float"); + return new FloatLiteral (context.BuiltinTypes, 0, loc); } - break; default: try { - val = new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), Location); + return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc); } catch (OverflowException) { - val = new DoubleLiteral (context.BuiltinTypes, 0, Location); - Report.Error (594, Location, error_details, "double"); + Report.Error (594, loc, error_details, "double"); + return new DoubleLiteral (context.BuiltinTypes, 0, loc); } - break; } - - return Token.LITERAL; } - int handle_hex () + ILiteralConstant handle_hex (Location loc) { int d; ulong ul; @@ -1464,23 +1491,22 @@ namespace Mono.CSharp } string s = new String (number_builder, 0, number_pos); + try { if (number_pos <= 8) ul = System.UInt32.Parse (s, NumberStyles.HexNumber); else ul = System.UInt64.Parse (s, NumberStyles.HexNumber); + + return integer_type_suffix (ul, peek_char (), loc); } catch (OverflowException){ Error_NumericConstantTooLong (); - val = new IntLiteral (context.BuiltinTypes, 0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } catch (FormatException) { Report.Error (1013, Location, "Invalid number"); - val = new IntLiteral (context.BuiltinTypes, 0, Location); - return Token.LITERAL; + return new IntLiteral (context.BuiltinTypes, 0, loc); } - - return integer_type_suffix (ul, peek_char ()); } // @@ -1488,16 +1514,26 @@ namespace Mono.CSharp // int is_number (int c) { - bool is_real = false; + ILiteralConstant res; +#if FULL_AST + int read_start = reader.Position - 1; +#endif number_pos = 0; + var loc = Location; if (c >= '0' && c <= '9'){ if (c == '0'){ int peek = peek_char (); - if (peek == 'x' || peek == 'X') - return handle_hex (); + if (peek == 'x' || peek == 'X') { + val = res = handle_hex (loc); +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1); +#endif + + return Token.LITERAL; + } } decimal_digits (c); c = get_char (); @@ -1507,6 +1543,7 @@ namespace Mono.CSharp // We need to handle the case of // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER) // + bool is_real = false; if (c == '.'){ if (decimal_digits ('.')){ is_real = true; @@ -1514,7 +1551,12 @@ namespace Mono.CSharp } else { putback ('.'); number_pos--; - return adjust_int (-1); + val = res = adjust_int (-1, loc); + +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1); +#endif + return Token.LITERAL; } } @@ -1522,7 +1564,7 @@ namespace Mono.CSharp is_real = true; if (number_pos == max_number_size) Error_NumericConstantTooLong (); - number_builder [number_pos++] = 'e'; + number_builder [number_pos++] = (char) c; c = get_char (); if (c == '+'){ @@ -1546,21 +1588,26 @@ namespace Mono.CSharp } var type = real_type_suffix (c); - if (type == TypeCode.Empty && !is_real){ + if (type == TypeCode.Empty && !is_real) { putback (c); - return adjust_int (c); - } + res = adjust_int (c, loc); + } else { + is_real = true; - is_real = true; + if (type == TypeCode.Empty) { + putback (c); + } - if (type == TypeCode.Empty){ - putback (c); + res = adjust_real (type, loc); } - - if (is_real) - return adjust_real (type); - throw new Exception ("Is Number should never reach this point"); + val = res; + +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty ? 1 : 0)); +#endif + + return Token.LITERAL; } // @@ -2664,6 +2711,10 @@ namespace Mono.CSharp if (quoted) start_location = start_location - 1; +#if FULL_AST + int reader_pos = reader.Position; +#endif + while (true){ c = get_char (); if (c == '"') { @@ -2684,13 +2735,23 @@ namespace Mono.CSharp else s = new string (value_builder, 0, pos); - val = new StringLiteral (context.BuiltinTypes, s, start_location); + ILiteralConstant res = new StringLiteral (context.BuiltinTypes, s, start_location); + val = res; +#if FULL_AST + res.ParsedValue = quoted ? + reader.ReadChars (reader_pos - 2, reader.Position - 1) : + reader.ReadChars (reader_pos - 1, reader.Position); +#endif + return Token.LITERAL; } if (c == '\n') { - if (!quoted) + if (!quoted) { Report.Error (1010, Location, "Newline in constant"); + val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location); + return Token.LITERAL; + } } else if (c == '\\' && !quoted) { int surrogate; c = escape (c, out surrogate); @@ -2793,7 +2854,7 @@ namespace Mono.CSharp if (id_builder [0] >= '_' && !quoted) { int keyword = GetKeyword (id_builder, pos); if (keyword != -1) { - val = LocatedToken.Create (null, ref_line, column); + val = LocatedToken.Create (keyword == Token.AWAIT ? "await" : null, ref_line, column); return keyword; } } @@ -3309,16 +3370,20 @@ namespace Mono.CSharp int TokenizeBackslash () { +#if FULL_AST + int read_start = reader.Position; +#endif + Location start_location = Location; int c = get_char (); tokens_seen = true; if (c == '\'') { - val = new CharLiteral (context.BuiltinTypes, (char) c, Location); - Report.Error (1011, Location, "Empty character literal"); + val = new CharLiteral (context.BuiltinTypes, (char) c, start_location); + Report.Error (1011, start_location, "Empty character literal"); return Token.LITERAL; } if (c == '\n') { - Report.Error (1010, Location, "Newline in constant"); + Report.Error (1010, start_location, "Newline in constant"); return Token.ERROR; } @@ -3329,11 +3394,12 @@ namespace Mono.CSharp if (d != 0) throw new NotImplementedException (); - val = new CharLiteral (context.BuiltinTypes, (char) c, Location); + ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location); + val = res; c = get_char (); if (c != '\'') { - Report.Error (1012, Location, "Too many characters in character literal"); + Report.Error (1012, start_location, "Too many characters in character literal"); // Try to recover, read until newline or next "'" while ((c = get_char ()) != -1) { @@ -3342,6 +3408,10 @@ namespace Mono.CSharp } } +#if FULL_AST + res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position); +#endif + return Token.LITERAL; }