X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mcs%2Fgmcs%2Fcs-tokenizer.cs;h=4eb5bcbafd2ec413d122996754c69fd987f3c382;hb=5697a97b9870b44d6dab7bec66577fa3ec815ee0;hp=bf4bb4478e8ed556d0cc61365bcf3b9909ad2cd1;hpb=edfea48c02b4ab9bbe9651a45b54fa4acc8dea1a;p=mono.git

diff --git a/mcs/gmcs/cs-tokenizer.cs b/mcs/gmcs/cs-tokenizer.cs
index bf4bb4478e8..4eb5bcbafd2 100755
--- a/mcs/gmcs/cs-tokenizer.cs
+++ b/mcs/gmcs/cs-tokenizer.cs
@@ -41,6 +41,7 @@ namespace Mono.CSharp
 		bool handle_remove_add = false;
 		bool handle_assembly = false;
 		bool handle_constraints = false;
+		bool handle_typeof = false;
 
 		//
 		// Whether tokens have been seen on this line
@@ -143,6 +144,16 @@ namespace Mono.CSharp
 				handle_constraints = value;
 			}
 		}
+
+		public bool TypeOfParsing {
+			get {
+				return handle_typeof;
+			}
+
+			set {
+				handle_typeof = value;
+			}
+		}
 		
 		//
 		// Class variables
@@ -178,6 +189,8 @@ namespace Mono.CSharp
 		const int max_id_size = 512;
 		static char [] id_builder = new char [max_id_size];
 
+		static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
+
 		const int max_number_size = 128;
 		static char [] number_builder = new char [max_number_size];
 		static int number_pos;
@@ -216,6 +229,7 @@ namespace Mono.CSharp
 		{
 			keywords = new CharArrayHashtable [64];
 
+			AddKeyword ("__arglist", Token.ARGLIST);
 			AddKeyword ("abstract", Token.ABSTRACT);
 			AddKeyword ("as", Token.AS);
 			AddKeyword ("add", Token.ADD);
@@ -299,11 +313,7 @@ namespace Mono.CSharp
 			AddKeyword ("volatile", Token.VOLATILE);
 			AddKeyword ("where", Token.WHERE);
 			AddKeyword ("while", Token.WHILE);
-
-			if (RootContext.V2){
-				AddKeyword ("__yield", Token.YIELD);
-				AddKeyword ("yield", Token.YIELD);
-			}
+			AddKeyword ("partial", Token.PARTIAL);
 		}
 
 		//
@@ -342,6 +352,7 @@ namespace Mono.CSharp
 				return -1;
 			if (handle_constraints == false && res == Token.WHERE)
 				return -1;
+
 			return res;
 			
 		}
@@ -383,15 +394,50 @@ namespace Mono.CSharp
 			Mono.CSharp.Location.Push (file);
 		}
 
-		bool is_identifier_start_character (char c)
+		public static void Cleanup () {
+			identifiers = null;
+		}
+
+		static bool is_identifier_start_character (char c)
 		{
 			return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c);
 		}
 
-		bool is_identifier_part_character (char c)
+		static bool is_identifier_part_character (char c)
 		{
 			return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
 		}
+		
+		public static bool IsValidIdentifier (string s)
+		{
+			if (s == null || s.Length == 0)
+				return false;
+			
+			if (!is_identifier_start_character (s [0]))
+				return false;
+			
+			for (int i = 1; i < s.Length; i ++)
+				if (! is_identifier_part_character (s [i]))
+					return false;
+			
+			return true;
+		}
+
+		bool parse_generic_dimension (out int dimension)
+		{
+			dimension = 1;
+
+		again:
+			int the_token = token ();
+			if (the_token == Token.OP_GENERICS_GT)
+				return true;
+			else if (the_token == Token.COMMA) {
+				dimension++;
+				goto again;
+			}
+
+			return false;
+		}
 
 		bool parse_less_than ()
 		{
@@ -425,12 +471,20 @@ namespace Mono.CSharp
 
 			if (the_token == Token.OP_GENERICS_GT)
 				return true;
-			else if (the_token == Token.COMMA)
+			else if ((the_token == Token.COMMA) || (the_token == Token.DOT))
 				goto start;
 			else if (the_token == Token.OP_GENERICS_LT) {
 				if (!parse_less_than ())
 					return false;
 				goto again;
+			} else if (the_token == Token.OPEN_BRACKET) {
+			rank_specifiers:
+				the_token = token ();
+				if (the_token == Token.CLOSE_BRACKET)
+					goto again;
+				else if (the_token == Token.COMMA)
+					goto rank_specifiers;
+				return false;
 			}
 
 			return false;
@@ -495,8 +549,19 @@ namespace Mono.CSharp
 				if (parsing_generic_less_than++ > 0)
 					return Token.OP_GENERICS_LT;
 
-				// Save current position and parse next token.
 				int old = reader.Position;
+				if (handle_typeof) {
+					int dimension;
+					if (parse_generic_dimension (out dimension)) {
+						val = dimension;
+						return Token.GENERIC_DIMENSION;
+					}
+					reader.Position = old;
+					putback_char = -1;
+				}
+
+				// Save current position and parse next token.
+				old = reader.Position;
 				bool is_generic_lt = parse_less_than ();
 				reader.Position = old;
 				putback_char = -1;
@@ -640,37 +705,6 @@ namespace Mono.CSharp
 				return Token.CARRET;
 			}
 
-#if FIXME
-			if (c == '>'){
-				if (deambiguate_greater_than == 0)
-					return Token.OP_GT;
-
-				--deambiguate_greater_than;
-
-				// Save current position and parse next token.
-				int old = reader.Position;
-				int new_token = token ();
-				reader.Position = old;
-				putback_char = -1;
-
-				switch (new_token) {
-				case Token.OPEN_PARENS:
-				case Token.CLOSE_PARENS:
-				case Token.CLOSE_BRACKET:
-				case Token.OP_GT:
-				case Token.COLON:
-				case Token.SEMICOLON:
-				case Token.COMMA:
-				case Token.DOT:
-				case Token.INTERR:
-					return Token.OP_GENERICS_GT;
-
-				default:
-					return Token.OP_GT;
-				}
-			}
-#endif
-
 			return Token.ERROR;
 		}
 
@@ -765,15 +799,12 @@ namespace Mono.CSharp
 						break;
 
 					case 'l':
-						if (!is_unsigned){
+						if (!is_unsigned && (RootContext.WarningLevel >= 4)){
 							//
 							// if we have not seen anything in between
 							// report this error
 							//
-							Report.Warning (
-								78, Location,
-							"the 'l' suffix is easily confused with digit `1'," +
-							" use 'L' for clarity");
+							Report.Warning (78, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
 						}
 						goto case 'L';
 						
@@ -856,6 +887,11 @@ namespace Mono.CSharp
 				val = 0ul;
 				return Token.LITERAL_INTEGER;
 			}
+			catch (FormatException) {
+				Report.Error (1013, Location, "Invalid number");
+				val = 0ul;
+				return Token.LITERAL_INTEGER;
+			}
 		}
 		
 		int adjust_real (int t)
@@ -1181,6 +1217,19 @@ namespace Mono.CSharp
 			case Token.TYPEOF:
 			case Token.UNCHECKED:
 			case Token.UNSAFE:
+
+				//
+				// These can be part of a member access
+				//
+			case Token.INT:
+			case Token.UINT:
+			case Token.SHORT:
+			case Token.USHORT:
+			case Token.LONG:
+			case Token.ULONG:
+			case Token.DOUBLE:
+			case Token.FLOAT:
+			case Token.CHAR:
 				return true;
 
 			default:
@@ -1191,7 +1240,19 @@ namespace Mono.CSharp
 		public int token ()
                 {
 			current_token = xtoken ();
-                        return current_token;
+
+			if (current_token != Token.DEFAULT)
+				return current_token;
+
+			int c = consume_whitespace ();
+			if (c == -1)
+				current_token = Token.ERROR;
+			else if (c == '(')
+				current_token = Token.DEFAULT_OPEN_PARENS;
+			else
+				putback (c);
+
+			return current_token;
                 }
 
 		static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
@@ -1203,9 +1264,19 @@ namespace Mono.CSharp
 			tokens_seen = false;
 			arg = "";
 			static_cmd_arg.Length = 0;
+
+			// skip over white space
+			while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
+				;
 				
-			while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
-				static_cmd_arg.Append ((char) c);
+			while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
+ 				if (is_identifier_part_character ((char) c)){
+					static_cmd_arg.Append ((char) c);
+					c = getChar ();
+ 				} else {
+ 					putback (c);
+ 					break;
+ 				}
 			}
 
 			cmd = static_cmd_arg.ToString ();
@@ -1258,6 +1329,11 @@ namespace Mono.CSharp
 				ref_name = file_name;
 				Location.Push (ref_name);
 				return true;
+			} else if (arg == "hidden"){
+				//
+				// We ignore #line hidden
+				//
+				return true;
 			}
 			
 			try {
@@ -1271,6 +1347,8 @@ namespace Mono.CSharp
 					
 					string name = arg.Substring (pos). Trim (quotes);
 					ref_name = Location.LookupFile (name);
+					file_name.HasLineDirective = true;
+					ref_name.HasLineDirective = true;
 					Location.Push (ref_name);
 				} else {
 					ref_line = System.Int32.Parse (arg);
@@ -1320,6 +1398,60 @@ namespace Mono.CSharp
 			}
 		}
 
+		/// <summary>
+		/// Handles #pragma directive
+		/// </summary>
+		void PreProcessPragma (string arg)
+		{
+			const string disable = "warning disable";
+			const string restore = "warning restore";
+
+			if (arg == disable) {
+				Report.RegisterWarningRegion (Location).WarningDisable (line);
+				return;
+			}
+
+			if (arg == restore) {
+				Report.RegisterWarningRegion (Location).WarningEnable (line);
+				return;
+			}
+
+			if (arg.StartsWith (disable)) {
+				int[] codes = ParseNumbers (arg.Substring (disable.Length));
+				foreach (int code in codes) {
+					if (code != 0)
+						Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
+				}
+				return;
+			}
+
+			if (arg.StartsWith (restore)) {
+				int[] codes = ParseNumbers (arg.Substring (restore.Length));
+				foreach (int code in codes) {
+					Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
+				}
+				return;
+			}
+
+			return;
+		}
+
+		int[] ParseNumbers (string text)
+		{
+			string[] string_array = text.Split (',');
+			int[] values = new int [string_array.Length];
+			int index = 0;
+			foreach (string string_code in string_array) {
+				try {
+					values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
+				}
+				catch (FormatException) {
+					Report.Warning (1692, Location, "Invalid number");
+				}
+			}
+			return values;
+		}
+
 		bool eval_val (string s)
 		{
 			if (s == "true")
@@ -1523,16 +1655,20 @@ namespace Mono.CSharp
 			//
 			switch (cmd){
 			case "pragma":
-				if (RootContext.V2)
-					return true;
-				break;
+				if (RootContext.Version == LanguageVersion.ISO_1) {
+					Report.FeatureIsNotStandardized (Location, "#pragma");
+					return caller_is_taking;
+				}
+
+				PreProcessPragma (arg);
+				return caller_is_taking;
 				
 			case "line":
 				if (!PreProcessLine (arg))
 					Report.Error (
 						1576, Location,
 						"Argument to #line directive is missing or invalid");
-				return true;
+				return caller_is_taking;
 
 			case "region":
 				region_directive = true;
@@ -1692,7 +1828,7 @@ namespace Mono.CSharp
 				return true;
 
 			case "warning":
-				Report.Warning (1030, Location, "#warning: '" + arg + "'");
+				Report.Warning (1030, Location, "#warning: '{0}'", arg);
 				return true;
 			}
 
@@ -1739,6 +1875,36 @@ namespace Mono.CSharp
 			return Token.EOF;
 		}
 
+		private int consume_identifier (int s)
+		{
+			int res = consume_identifier (s, false);
+
+			if (res == Token.PARTIAL) {
+				// Save current position and parse next token.
+				int old = reader.Position;
+				int old_putback = putback_char;
+
+				putback_char = -1;
+
+				int next_token = token ();
+				bool ok = (next_token == Token.CLASS) ||
+					(next_token == Token.STRUCT) ||
+					(next_token == Token.INTERFACE);
+
+				reader.Position = old;
+				putback_char = old_putback;
+
+				if (ok)
+					return res;
+				else {
+					val = "partial";
+					return Token.IDENTIFIER;
+				}
+			}
+
+			return res;
+		}
+
 		private int consume_identifier (int s, bool quoted) 
 		{
 			int pos = 1;
@@ -1766,18 +1932,37 @@ namespace Mono.CSharp
 			// Optimization: avoids doing the keyword lookup
 			// on uppercase letters and _
 			//
-			if (s >= 'a'){
+			if (!quoted && (s >= 'a' || s == '_')){
 				int keyword = GetKeyword (id_builder, pos);
-				if (keyword != -1 && !quoted)
+				if (keyword != -1)
 				return keyword;
 			}
 
+			//
+			// Keep identifiers in an array of hashtables to avoid needless
+			// allocations
+			//
+
+			if (identifiers [pos] != null) {
+				val = identifiers [pos][id_builder];
+				if (val != null) {
+					return Token.IDENTIFIER;
+				}
+			}
+			else
+				identifiers [pos] = new CharArrayHashtable (pos);
+
 			val = new String (id_builder, 0, pos);
 
+			char [] chars = new char [pos];
+			Array.Copy (id_builder, chars, pos);
+
+			identifiers [pos] [chars] = val;
+
 			return Token.IDENTIFIER;
 		}
-		
-		public int xtoken ()
+
+		int consume_whitespace ()
 		{
 			int t;
 			bool doread = false;
@@ -1786,10 +1971,26 @@ namespace Mono.CSharp
 			val = null;
 			// optimization: eliminate col and implement #directive semantic correctly.
 			for (;(c = getChar ()) != -1; col++) {
-				if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r' || c == 0xa0){
-					
-					if (c == '\t')
-						col = (((col + 8) / 8) * 8) - 1;
+				if (c == ' ')
+					continue;
+				
+				if (c == '\t') {
+					col = (((col + 8) / 8) * 8) - 1;
+					continue;
+				}
+				
+				if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0)
+					continue;
+
+				if (c == '\r') {
+					if (peekChar () == '\n')
+						getChar ();
+
+					line++;
+					ref_line++;
+					col = 0;
+					any_token_seen |= tokens_seen;
+					tokens_seen = false;
 					continue;
 				}
 
@@ -1831,22 +2032,7 @@ namespace Mono.CSharp
 					goto is_punct_label;
 				}
 
-				
-				if (is_identifier_start_character ((char)c)){
-					tokens_seen = true;
-					return consume_identifier (c, false);
-				}
-
 			is_punct_label:
-				if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
-					tokens_seen = true;
-					if (doread){
-						getChar ();
-						col++;
-					}
-					return t;
-				}
-
 				// white space
 				if (c == '\n'){
 					line++;
@@ -1857,19 +2043,6 @@ namespace Mono.CSharp
 					continue;
 				}
 
-				if (c >= '0' && c <= '9'){
-					tokens_seen = true;
-					return is_number (c);
-				}
-
-				if (c == '.'){
-					tokens_seen = true;
-					int peek = peekChar ();
-					if (peek >= '0' && peek <= '9')
-						return is_number (c);
-					return Token.DOT;
-				}
-				
 				/* For now, ignore pre-processor commands */
 				// FIXME: In C# the '#' is not limited to appear
 				// on the first column.
@@ -1906,72 +2079,115 @@ namespace Mono.CSharp
 						Report.Error (1027, Location, "#endif/#endregion expected");
 					continue;
 				}
-				
-				if (c == '"') 
-					return consume_string (false);
 
-				if (c == '\''){
-					c = getChar ();
-					tokens_seen = true;
-					if (c == '\''){
-						error_details = "Empty character literal";
-						Report.Error (1011, Location, error_details);
-						return Token.ERROR;
-					}
-					c = escape (c);
-					if (c == -1)
-						return Token.ERROR;
-					val = new System.Char ();
-					val = (char) c;
-					c = getChar ();
+				return c;
+			}
 
-					if (c != '\''){
-						error_details = "Too many characters in character literal";
-						Report.Error (1012, Location, error_details);
+			return -1;
+		}
+		
+		public int xtoken ()
+		{
+			int t;
+			bool doread = false;
+			int c;
 
-						// Try to recover, read until newline or next "'"
-						while ((c = getChar ()) != -1){
-							if (c == '\n' || c == '\''){
-								line++;
-								ref_line++;
-								col = 0;
-								break;
-							} else
-								col++;
-							
-						}
-						return Token.ERROR;
-					}
-					return Token.LITERAL_CHARACTER;
-				}
-				
-				if (c == '@') {
-					c = getChar ();
-					if (c == '"') {
-						tokens_seen = true;
-						return consume_string (true);
-					} else if (is_identifier_start_character ((char) c)){
-						return consume_identifier (c, true);
-					} else {
-						Report.Error (1033, Location, "'@' must be followed by string constant or identifier");
-					}
+			val = null;
+			// optimization: eliminate col and implement #directive semantic correctly.
+
+			c = consume_whitespace ();
+			if (c == -1)
+				return Token.EOF;
+
+			if (is_identifier_start_character ((char)c)){
+				tokens_seen = true;
+					return consume_identifier (c);
+			}
+
+		is_punct_label:
+			if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
+				tokens_seen = true;
+				if (doread){
+					getChar ();
+					col++;
 				}
+				return t;
+			}
+
+			if (c >= '0' && c <= '9'){
+				tokens_seen = true;
+				return is_number (c);
+			}
 
-				if (c == '#') {
-					error_details = "Preprocessor directives must appear as the first non-whitespace " +
-						"character on a line.";
+			if (c == '.'){
+				tokens_seen = true;
+				int peek = peekChar ();
+				if (peek >= '0' && peek <= '9')
+					return is_number (c);
+				return Token.DOT;
+			}
 
-					Report.Error (1040, Location, error_details);
+			if (c == '"') 
+				return consume_string (false);
 
+			if (c == '\''){
+				c = getChar ();
+				tokens_seen = true;
+				if (c == '\''){
+					error_details = "Empty character literal";
+					Report.Error (1011, Location, error_details);
 					return Token.ERROR;
 				}
+				c = escape (c);
+				if (c == -1)
+					return Token.ERROR;
+				val = new System.Char ();
+				val = (char) c;
+				c = getChar ();
+
+				if (c != '\''){
+					error_details = "Too many characters in character literal";
+					Report.Error (1012, Location, error_details);
 
-				error_details = ((char)c).ToString ();
+					// Try to recover, read until newline or next "'"
+					while ((c = getChar ()) != -1){
+						if (c == '\n' || c == '\''){
+							line++;
+							ref_line++;
+							col = 0;
+							break;
+						} else
+							col++;
+					}
+					return Token.ERROR;
+				}
+				return Token.LITERAL_CHARACTER;
+			}
 				
+			if (c == '@') {
+				c = getChar ();
+				if (c == '"') {
+					tokens_seen = true;
+					return consume_string (true);
+				} else if (is_identifier_start_character ((char) c)){
+					return consume_identifier (c, true);
+				} else {
+					Report.Error (1033, Location, "'@' must be followed by string constant or identifier");
+				}
+			}
+
+			if (c == '#') {
+				error_details = "Preprocessor directives must appear as the first non-whitespace " +
+					"character on a line.";
+
+				Report.Error (1040, Location, error_details);
+
 				return Token.ERROR;
 			}
 
-			return Token.EOF;
+			error_details = ((char)c).ToString ();
+
+			return Token.ERROR;
 		}
 
 		public void cleanup ()
@@ -1985,7 +2201,6 @@ namespace Mono.CSharp
 			}
 				
 		}
-
 	}
 }