2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
\r
3 // This also implements the preprocessor
\r
5 // Author: Miguel de Icaza (miguel@gnu.org)
\r
7 // Licensed under the terms of the GNU GPL
\r
9 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
\r
14 * Make sure we accept the proper Unicode ranges, per the spec.
\r
20 using System.Collections;
\r
22 using System.Globalization;
\r
23 using System.Reflection;
\r
25 namespace Mono.CSharp
\r
28 /// Tokenizer for C# source code.
\r
31 public class Tokenizer : yyParser.yyInput
\r
33 StreamReader reader;
\r
34 public string ref_name;
\r
35 public int ref_line = 1;
\r
36 public int line = 1;
\r
38 public int current_token;
\r
39 bool handle_get_set = false;
\r
40 bool handle_remove_add = false;
\r
41 bool handle_assembly = false;
\r
44 // Whether tokens have been seen on this line
\r
46 bool tokens_seen = false;
\r
49 // Whether a token has been seen on the file
\r
50 // This is needed because `define' is not allowed to be used
\r
51 // after a token has been seen.
\r
53 bool any_token_seen = false;
\r
54 static Hashtable tokenValues;
\r
56 private static Hashtable TokenValueName
\r
59 if (tokenValues == null)
\r
60 tokenValues = GetTokenValueNameHash ();
\r
66 private static Hashtable GetTokenValueNameHash ()
\r
68 Type t = typeof (Token);
\r
69 FieldInfo [] fields = t.GetFields ();
\r
70 Hashtable hash = new Hashtable ();
\r
71 foreach (FieldInfo field in fields) {
\r
72 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))
\r
73 hash.Add (field.GetValue (null), field.Name);
\r
79 // Returns a verbose representation of the current location
\r
81 public string location {
\r
85 if (current_token == Token.ERROR)
\r
86 det = "detail: " + error_details;
\r
90 // return "Line: "+line+" Col: "+col + "\n" +
\r
91 // "VirtLine: "+ref_line +
\r
92 // " Token: "+current_token + " " + det;
\r
93 string current_token_name = TokenValueName [current_token] as string;
\r
94 if (current_token_name == null)
\r
95 current_token_name = current_token.ToString ();
\r
97 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name,
\r
100 current_token_name,
\r
105 public bool PropertyParsing {
\r
107 return handle_get_set;
\r
111 handle_get_set = value;
\r
115 public bool AssemblyTargetParsing {
\r
117 return handle_assembly;
\r
121 handle_assembly = value;
\r
125 public bool EventParsing {
\r
127 return handle_remove_add;
\r
131 handle_remove_add = value;
\r
138 static Hashtable keywords;
\r
139 static NumberStyles styles;
\r
140 static NumberFormatInfo csharp_format_info;
\r
143 // Values for the associated token returned
\r
153 const int TAKING = 1;
\r
154 const int TAKEN_BEFORE = 2;
\r
155 const int ELSE_SEEN = 4;
\r
156 const int PARENT_TAKING = 8;
\r
159 // pre-processor if stack state:
\r
163 static System.Text.StringBuilder id_builder;
\r
164 static System.Text.StringBuilder string_builder;
\r
165 static System.Text.StringBuilder number_builder;
\r
168 // Details about the error encoutered by the tokenizer
\r
170 string error_details;
\r
172 public string error {
\r
174 return error_details;
\r
190 static void InitTokens ()
\r
192 keywords = new Hashtable ();
\r
194 keywords.Add ("abstract", Token.ABSTRACT);
\r
195 keywords.Add ("as", Token.AS);
\r
196 keywords.Add ("add", Token.ADD);
\r
197 keywords.Add ("assembly", Token.ASSEMBLY);
\r
198 keywords.Add ("base", Token.BASE);
\r
199 keywords.Add ("bool", Token.BOOL);
\r
200 keywords.Add ("break", Token.BREAK);
\r
201 keywords.Add ("byte", Token.BYTE);
\r
202 keywords.Add ("case", Token.CASE);
\r
203 keywords.Add ("catch", Token.CATCH);
\r
204 keywords.Add ("char", Token.CHAR);
\r
205 keywords.Add ("checked", Token.CHECKED);
\r
206 keywords.Add ("class", Token.CLASS);
\r
207 keywords.Add ("const", Token.CONST);
\r
208 keywords.Add ("continue", Token.CONTINUE);
\r
209 keywords.Add ("decimal", Token.DECIMAL);
\r
210 keywords.Add ("default", Token.DEFAULT);
\r
211 keywords.Add ("delegate", Token.DELEGATE);
\r
212 keywords.Add ("do", Token.DO);
\r
213 keywords.Add ("double", Token.DOUBLE);
\r
214 keywords.Add ("else", Token.ELSE);
\r
215 keywords.Add ("enum", Token.ENUM);
\r
216 keywords.Add ("event", Token.EVENT);
\r
217 keywords.Add ("explicit", Token.EXPLICIT);
\r
218 keywords.Add ("extern", Token.EXTERN);
\r
219 keywords.Add ("false", Token.FALSE);
\r
220 keywords.Add ("finally", Token.FINALLY);
\r
221 keywords.Add ("fixed", Token.FIXED);
\r
222 keywords.Add ("float", Token.FLOAT);
\r
223 keywords.Add ("for", Token.FOR);
\r
224 keywords.Add ("foreach", Token.FOREACH);
\r
225 keywords.Add ("goto", Token.GOTO);
\r
226 keywords.Add ("get", Token.GET);
\r
227 keywords.Add ("if", Token.IF);
\r
228 keywords.Add ("implicit", Token.IMPLICIT);
\r
229 keywords.Add ("in", Token.IN);
\r
230 keywords.Add ("int", Token.INT);
\r
231 keywords.Add ("interface", Token.INTERFACE);
\r
232 keywords.Add ("internal", Token.INTERNAL);
\r
233 keywords.Add ("is", Token.IS);
\r
234 keywords.Add ("lock", Token.LOCK);
\r
235 keywords.Add ("long", Token.LONG);
\r
236 keywords.Add ("namespace", Token.NAMESPACE);
\r
237 keywords.Add ("new", Token.NEW);
\r
238 keywords.Add ("null", Token.NULL);
\r
239 keywords.Add ("object", Token.OBJECT);
\r
240 keywords.Add ("operator", Token.OPERATOR);
\r
241 keywords.Add ("out", Token.OUT);
\r
242 keywords.Add ("override", Token.OVERRIDE);
\r
243 keywords.Add ("params", Token.PARAMS);
\r
244 keywords.Add ("private", Token.PRIVATE);
\r
245 keywords.Add ("protected", Token.PROTECTED);
\r
246 keywords.Add ("public", Token.PUBLIC);
\r
247 keywords.Add ("readonly", Token.READONLY);
\r
248 keywords.Add ("ref", Token.REF);
\r
249 keywords.Add ("remove", Token.REMOVE);
\r
250 keywords.Add ("return", Token.RETURN);
\r
251 keywords.Add ("sbyte", Token.SBYTE);
\r
252 keywords.Add ("sealed", Token.SEALED);
\r
253 keywords.Add ("set", Token.SET);
\r
254 keywords.Add ("short", Token.SHORT);
\r
255 keywords.Add ("sizeof", Token.SIZEOF);
\r
256 keywords.Add ("stackalloc", Token.STACKALLOC);
\r
257 keywords.Add ("static", Token.STATIC);
\r
258 keywords.Add ("string", Token.STRING);
\r
259 keywords.Add ("struct", Token.STRUCT);
\r
260 keywords.Add ("switch", Token.SWITCH);
\r
261 keywords.Add ("this", Token.THIS);
\r
262 keywords.Add ("throw", Token.THROW);
\r
263 keywords.Add ("true", Token.TRUE);
\r
264 keywords.Add ("try", Token.TRY);
\r
265 keywords.Add ("typeof", Token.TYPEOF);
\r
266 keywords.Add ("uint", Token.UINT);
\r
267 keywords.Add ("ulong", Token.ULONG);
\r
268 keywords.Add ("unchecked", Token.UNCHECKED);
\r
269 keywords.Add ("unsafe", Token.UNSAFE);
\r
270 keywords.Add ("ushort", Token.USHORT);
\r
271 keywords.Add ("using", Token.USING);
\r
272 keywords.Add ("virtual", Token.VIRTUAL);
\r
273 keywords.Add ("void", Token.VOID);
\r
274 keywords.Add ("volatile", Token.VOLATILE);
\r
275 keywords.Add ("while", Token.WHILE);
\r
279 // Class initializer
\r
281 static Tokenizer ()
\r
284 csharp_format_info = NumberFormatInfo.InvariantInfo;
\r
285 styles = NumberStyles.Float;
\r
287 id_builder = new System.Text.StringBuilder ();
\r
288 string_builder = new System.Text.StringBuilder ();
\r
289 number_builder = new System.Text.StringBuilder ();
\r
292 int GetKeyword (string name)
\r
294 object o = keywords [name];
\r
301 if (handle_get_set == false && (res == Token.GET || res == Token.SET))
\r
303 if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
\r
305 if (handle_assembly == false && res == Token.ASSEMBLY)
\r
311 public Location Location {
\r
313 return new Location (ref_line);
\r
317 void define (string def)
\r
319 if (!RootContext.AllDefines.Contains (def)){
\r
320 RootContext.AllDefines [def] = true;
\r
322 if (defines.Contains (def))
\r
324 defines [def] = true;
\r
327 public Tokenizer (StreamReader input, string fname, ArrayList defs)
\r
329 this.ref_name = fname;
\r
335 defines = new Hashtable ();
\r
336 foreach (string def in defs)
\r
341 // FIXME: This could be `Location.Push' but we have to
\r
342 // find out why the MS compiler allows this
\r
344 Mono.CSharp.Location.Push (fname);
\r
347 bool is_identifier_start_character (char c)
\r
349 return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c);
\r
352 bool is_identifier_part_character (char c)
\r
354 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
\r
357 int is_punct (char c, ref bool doread)
\r
366 return Token.OPEN_BRACE;
\r
368 return Token.CLOSE_BRACE;
\r
370 return Token.OPEN_BRACKET;
\r
372 return Token.CLOSE_BRACKET;
\r
374 return Token.OPEN_PARENS;
\r
376 return Token.CLOSE_PARENS;
\r
378 return Token.COMMA;
\r
380 return Token.COLON;
\r
382 return Token.SEMICOLON;
\r
384 return Token.TILDE;
\r
386 return Token.INTERR;
\r
395 t = Token.OP_ADD_ASSIGN;
\r
405 t = Token.OP_SUB_ASSIGN;
\r
409 return Token.MINUS;
\r
417 return Token.OP_NE;
\r
425 return Token.OP_EQ;
\r
427 return Token.ASSIGN;
\r
433 return Token.OP_AND;
\r
434 } else if (d == '='){
\r
436 return Token.OP_AND_ASSIGN;
\r
438 return Token.BITWISE_AND;
\r
444 return Token.OP_OR;
\r
445 } else if (d == '='){
\r
447 return Token.OP_OR_ASSIGN;
\r
449 return Token.BITWISE_OR;
\r
455 return Token.OP_MULT_ASSIGN;
\r
463 return Token.OP_DIV_ASSIGN;
\r
471 return Token.OP_MOD_ASSIGN;
\r
473 return Token.PERCENT;
\r
479 return Token.OP_XOR_ASSIGN;
\r
481 return Token.CARRET;
\r
491 return Token.OP_SHIFT_LEFT_ASSIGN;
\r
493 return Token.OP_SHIFT_LEFT;
\r
494 } else if (d == '='){
\r
496 return Token.OP_LE;
\r
498 return Token.OP_LT;
\r
508 return Token.OP_SHIFT_RIGHT_ASSIGN;
\r
510 return Token.OP_SHIFT_RIGHT;
\r
511 } else if (d == '='){
\r
513 return Token.OP_GE;
\r
515 return Token.OP_GT;
\r
517 return Token.ERROR;
\r
520 bool decimal_digits (int c)
\r
523 bool seen_digits = false;
\r
526 number_builder.Append ((char) c);
\r
529 // We use peekChar2, because decimal_digits needs to do a
\r
530 // 2-character look-ahead (5.ToString for example).
\r
532 while ((d = peekChar2 ()) != -1){
\r
533 if (d >= '0' && d <= '9'){
\r
534 number_builder.Append ((char) d);
\r
536 seen_digits = true;
\r
541 return seen_digits;
\r
544 bool is_hex (int e)
\r
546 return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
\r
549 void hex_digits (int c)
\r
554 number_builder.Append ((char) c);
\r
555 while ((d = peekChar ()) != -1){
\r
557 number_builder.Append ((char) d);
\r
564 int real_type_suffix (int c)
\r
569 case 'F': case 'f':
\r
570 t = Token.LITERAL_FLOAT;
\r
572 case 'D': case 'd':
\r
573 t = Token.LITERAL_DOUBLE;
\r
575 case 'M': case 'm':
\r
576 t= Token.LITERAL_DECIMAL;
\r
584 int integer_type_suffix (ulong ul, int c)
\r
586 bool is_unsigned = false;
\r
587 bool is_long = false;
\r
590 bool scanning = true;
\r
593 case 'U': case 'u':
\r
596 is_unsigned = true;
\r
603 // if we have not seen anything in between
\r
604 // report this error
\r
608 "the 'l' suffix is easily confused with digit `1'," +
\r
609 " use 'L' for clarity");
\r
625 } while (scanning);
\r
628 if (is_long && is_unsigned){
\r
630 return Token.LITERAL_INTEGER;
\r
631 } else if (is_unsigned){
\r
632 // uint if possible, or ulong else.
\r
634 if ((ul & 0xffffffff00000000) == 0)
\r
638 } else if (is_long){
\r
639 // long if possible, ulong otherwise
\r
640 if ((ul & 0x8000000000000000) != 0)
\r
645 // int, uint, long or ulong in that order
\r
646 if ((ul & 0xffffffff00000000) == 0){
\r
647 uint ui = (uint) ul;
\r
649 if ((ui & 0x80000000) != 0)
\r
654 if ((ul & 0x8000000000000000) != 0)
\r
660 return Token.LITERAL_INTEGER;
\r
664 // given `c' as the next char in the input decide whether
\r
665 // we need to convert to a special type, and then choose
\r
666 // the best representation for the integer
\r
668 int adjust_int (int c)
\r
671 ulong ul = System.UInt64.Parse (number_builder.ToString ());
\r
672 return integer_type_suffix (ul, c);
\r
673 } catch (OverflowException) {
\r
674 error_details = "Integral constant is too large";
\r
675 Report.Error (1021, Location, error_details);
\r
677 return Token.LITERAL_INTEGER;
\r
681 int adjust_real (int t)
\r
683 string s = number_builder.ToString ();
\r
686 case Token.LITERAL_DECIMAL:
\r
688 val = System.Decimal.Parse (s, styles, csharp_format_info);
\r
689 } catch (OverflowException) {
\r
691 error_details = "Floating-point constant is outside the range of the type 'decimal'";
\r
692 Report.Error (594, Location, error_details);
\r
695 case Token.LITERAL_FLOAT:
\r
697 val = (float) System.Double.Parse (s, styles, csharp_format_info);
\r
698 } catch (OverflowException) {
\r
700 error_details = "Floating-point constant is outside the range of the type 'float'";
\r
701 Report.Error (594, Location, error_details);
\r
705 case Token.LITERAL_DOUBLE:
\r
707 t = Token.LITERAL_DOUBLE;
\r
709 val = System.Double.Parse (s, styles, csharp_format_info);
\r
710 } catch (OverflowException) {
\r
712 error_details = "Floating-point constant is outside the range of the type 'double'";
\r
713 Report.Error (594, Location, error_details);
\r
721 // Invoked if we know we have .digits or digits
\r
723 int is_number (int c)
\r
725 bool is_real = false;
\r
728 number_builder.Length = 0;
\r
730 if (c >= '0' && c <= '9'){
\r
731 if (c == '0' && peekChar () == 'x' || peekChar () == 'X'){
\r
736 string s = number_builder.ToString ();
\r
738 ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
\r
739 return integer_type_suffix (ul, peekChar ());
\r
741 decimal_digits (c);
\r
746 // We need to handle the case of
\r
747 // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
\r
750 if (decimal_digits ('.')){
\r
755 number_builder.Length -= 1;
\r
756 return adjust_int (-1);
\r
760 if (c == 'e' || c == 'E'){
\r
762 number_builder.Append ("e");
\r
766 number_builder.Append ((char) c);
\r
768 } else if (c == '-') {
\r
769 number_builder.Append ((char) c);
\r
772 number_builder.Append ('+');
\r
774 decimal_digits (c);
\r
778 type = real_type_suffix (c);
\r
779 if (type == Token.NONE && !is_real){
\r
781 return adjust_int (c);
\r
785 if (type == Token.NONE){
\r
790 return adjust_real (type);
\r
792 Console.WriteLine ("This should not be reached");
\r
793 throw new Exception ("Is Number should never reach this point");
\r
797 // Accepts exactly count (4 or 8) hex, no more no less
\r
799 int getHex (int count, out bool error)
\r
801 int [] buffer = new int [8];
\r
805 int top = count != -1 ? count : 4;
\r
809 for (i = 0; i < top; i++){
\r
812 if (c >= '0' && c <= '9')
\r
813 c = (int) c - (int) '0';
\r
814 else if (c >= 'A' && c <= 'F')
\r
815 c = (int) c - (int) 'A' + 10;
\r
816 else if (c >= 'a' && c <= 'f')
\r
817 c = (int) c - (int) 'a' + 10;
\r
823 total = (total * 16) + c;
\r
825 int p = peekChar ();
\r
828 if (!is_hex ((char)p))
\r
869 v = getHex (-1, out error);
\r
874 v = getHex (4, out error);
\r
879 v = getHex (8, out error);
\r
884 Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);
\r
893 if (putback_char != -1){
\r
894 int x = putback_char;
\r
899 return reader.Read ();
\r
904 if (putback_char != -1)
\r
905 return putback_char;
\r
906 putback_char = reader.Read ();
\r
907 return putback_char;
\r
912 if (putback_char != -1)
\r
913 return putback_char;
\r
914 return reader.Peek ();
\r
917 void putback (int c)
\r
919 if (putback_char != -1){
\r
920 Console.WriteLine ("Col: " + col);
\r
921 Console.WriteLine ("Row: " + line);
\r
922 Console.WriteLine ("Name: " + ref_name);
\r
923 Console.WriteLine ("Current [{0}] putting back [{1}] ", putback_char, c);
\r
924 throw new Exception ("This should not happen putback on putback");
\r
929 public bool advance ()
\r
931 return peekChar () != -1;
\r
934 public Object Value {
\r
940 public Object value ()
\r
945 public int token ()
\r
947 current_token = xtoken ();
\r
948 return current_token;
\r
951 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
\r
953 void get_cmd_arg (out string cmd, out string arg)
\r
957 tokens_seen = false;
\r
959 static_cmd_arg.Length = 0;
\r
961 while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t')){
\r
964 static_cmd_arg.Append ((char) c);
\r
967 cmd = static_cmd_arg.ToString ();
\r
975 // skip over white space
\r
976 while ((c = getChar ()) != -1 && (c != '\n') && ((c == ' ') || (c == '\t')))
\r
985 static_cmd_arg.Length = 0;
\r
986 static_cmd_arg.Append ((char) c);
\r
988 while ((c = getChar ()) != -1 && (c != '\n')){
\r
991 static_cmd_arg.Append ((char) c);
\r
998 arg = static_cmd_arg.ToString ().Trim ();
\r
1002 // Handles the #line directive
\r
1004 bool PreProcessLine (string arg)
\r
1009 if (arg == "default"){
\r
1010 line = ref_line = line;
\r
1017 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
\r
1018 ref_line = System.Int32.Parse (arg.Substring (0, pos));
\r
1021 char [] quotes = { '\"' };
\r
1023 ref_name = arg.Substring (pos). Trim (quotes);
\r
1025 ref_line = System.Int32.Parse (arg);
\r
1035 // Handles #define and #undef
\r
1037 void PreProcessDefinition (bool is_define, string arg)
\r
1039 if (arg == "" || arg == "true" || arg == "false"){
\r
1040 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
\r
1044 char[] whitespace = { ' ', '\t' };
\r
1045 if (arg.IndexOfAny (whitespace) != -1){
\r
1046 Report.Error (1025, Location, "Single-line comment or end-of-line expected");
\r
1050 foreach (char c in arg){
\r
1051 if (!Char.IsLetter (c) && (c != '_')){
\r
1052 Report.Error (1001, Location, "Identifier expected");
\r
1058 if (defines == null)
\r
1059 defines = new Hashtable ();
\r
1062 if (defines == null)
\r
1064 if (defines.Contains (arg))
\r
1065 defines.Remove (arg);
\r
1069 bool eval_val (string s)
\r
1076 if (defines == null)
\r
1078 if (defines.Contains (s))
\r
1084 bool pp_primary (ref string s)
\r
1087 int len = s.Length;
\r
1093 s = s.Substring (1);
\r
1094 bool val = pp_expr (ref s);
\r
1095 if (s.Length > 0 && s [0] == ')'){
\r
1096 s = s.Substring (1);
\r
1099 Error_InvalidDirective ();
\r
1103 if (is_identifier_start_character (c)){
\r
1109 if (is_identifier_part_character (c)){
\r
1113 bool v = eval_val (s.Substring (0, j));
\r
1114 s = s.Substring (j);
\r
1117 bool vv = eval_val (s);
\r
1122 Error_InvalidDirective ();
\r
1126 bool pp_unary (ref string s)
\r
1129 int len = s.Length;
\r
1132 if (s [0] == '!'){
\r
1133 if (len > 1 && s [1] == '='){
\r
1134 Error_InvalidDirective ();
\r
1137 s = s.Substring (1);
\r
1138 return ! pp_primary (ref s);
\r
1140 return pp_primary (ref s);
\r
1142 Error_InvalidDirective ();
\r
1147 bool pp_eq (ref string s)
\r
1149 bool va = pp_unary (ref s);
\r
1152 int len = s.Length;
\r
1154 if (s [0] == '='){
\r
1155 if (len > 2 && s [1] == '='){
\r
1156 s = s.Substring (2);
\r
1157 return va == pp_unary (ref s);
\r
1159 Error_InvalidDirective ();
\r
1162 } else if (s [0] == '!' && len > 1 && s [1] == '='){
\r
1163 s = s.Substring (2);
\r
1165 return va != pp_unary (ref s);
\r
1174 bool pp_and (ref string s)
\r
1176 bool va = pp_eq (ref s);
\r
1179 int len = s.Length;
\r
1181 if (s [0] == '&'){
\r
1182 if (len > 2 && s [1] == '&'){
\r
1183 s = s.Substring (2);
\r
1184 return (va & pp_eq (ref s));
\r
1186 Error_InvalidDirective ();
\r
1195 // Evaluates an expression for `#if' or `#elif'
\r
1197 bool pp_expr (ref string s)
\r
1199 bool va = pp_and (ref s);
\r
1201 int len = s.Length;
\r
1206 if (len > 2 && s [1] == '|'){
\r
1207 s = s.Substring (2);
\r
1208 return va | pp_expr (ref s);
\r
1210 Error_InvalidDirective ();
\r
1219 bool eval (string s)
\r
1221 bool v = pp_expr (ref s);
\r
1223 if (s.Length != 0){
\r
1224 Error_InvalidDirective ();
\r
1231 void Error_InvalidDirective ()
\r
1233 Report.Error (1517, Location, "Invalid pre-processor directive");
\r
1236 void Error_UnexpectedDirective (string extra)
\r
1240 "Unexpected processor directive (" + extra + ")");
\r
1243 void Error_TokensSeen ()
\r
1247 "Cannot define or undefine pre-processor symbols after a token in the file");
\r
1251 // if true, then the code continues processing the code
\r
1252 // if false, the code stays in a loop until another directive is
\r
1255 bool handle_preprocessing_directive (bool caller_is_taking)
\r
1257 char [] blank = { ' ', '\t' };
\r
1260 get_cmd_arg (out cmd, out arg);
\r
1262 // Eat any trailing whitespaces and single-line comments
\r
1263 if (arg.IndexOf ("//") != -1)
\r
1264 arg = arg.Substring (0, arg.IndexOf ("//"));
\r
1265 arg = arg.TrimEnd (' ', '\t');
\r
1268 // The first group of pre-processing instructions is always processed
\r
1272 if (!PreProcessLine (arg))
\r
1275 "Argument to #line directive is missing or invalid");
\r
1283 goto case "endif";
\r
1287 Error_InvalidDirective ();
\r
1290 bool taking = false;
\r
1291 if (ifstack == null)
\r
1292 ifstack = new Stack ();
\r
1294 if (ifstack.Count == 0){
\r
1297 int state = (int) ifstack.Peek ();
\r
1298 if ((state & TAKING) != 0)
\r
1302 if (eval (arg) && taking){
\r
1303 ifstack.Push (TAKING | TAKEN_BEFORE | PARENT_TAKING);
\r
1306 ifstack.Push (taking ? PARENT_TAKING : 0);
\r
1311 if (ifstack == null || ifstack.Count == 0){
\r
1312 Error_UnexpectedDirective ("no #if for this #endif");
\r
1316 if (ifstack.Count == 0)
\r
1319 int state = (int) ifstack.Peek ();
\r
1321 if ((state & TAKING) != 0)
\r
1329 if (ifstack == null || ifstack.Count == 0){
\r
1330 Error_UnexpectedDirective ("no #if for this #elif");
\r
1333 int state = (int) ifstack.Peek ();
\r
1335 if ((state & ELSE_SEEN) != 0){
\r
1336 Error_UnexpectedDirective ("#elif not valid after #else");
\r
1340 if ((state & (TAKEN_BEFORE | TAKING)) != 0)
\r
1343 if (eval (arg) && ((state & PARENT_TAKING) != 0)){
\r
1344 state = (int) ifstack.Pop ();
\r
1345 ifstack.Push (state | TAKING | TAKEN_BEFORE);
\r
1352 if (ifstack == null || ifstack.Count == 0){
\r
1355 "Unexpected processor directive (no #if for this #else)");
\r
1358 int state = (int) ifstack.Peek ();
\r
1360 if ((state & ELSE_SEEN) != 0){
\r
1361 Error_UnexpectedDirective ("#else within #else");
\r
1368 if ((state & TAKEN_BEFORE) == 0){
\r
1369 ret = ((state & PARENT_TAKING) != 0);
\r
1378 ifstack.Push (state | ELSE_SEEN);
\r
1385 // These are only processed if we are in a `taking' block
\r
1387 if (!caller_is_taking)
\r
1392 if (any_token_seen){
\r
1393 Error_TokensSeen ();
\r
1396 PreProcessDefinition (true, arg);
\r
1400 if (any_token_seen){
\r
1401 Error_TokensSeen ();
\r
1404 PreProcessDefinition (false, arg);
\r
1408 Report.Error (1029, Location, "#error: '" + arg + "'");
\r
1412 Report.Warning (1030, Location, "#warning: '" + arg + "'");
\r
1416 Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");
\r
1421 private int consume_string (bool quoted)
\r
1424 string_builder.Length = 0;
\r
1426 while ((c = getChar ()) != -1){
\r
1428 if (quoted && peekChar () == '"'){
\r
1429 string_builder.Append ((char) c);
\r
1433 val = string_builder.ToString ();
\r
1434 return Token.LITERAL_STRING;
\r
1440 Report.Error (1010, Location, "Newline in constant");
\r
1450 return Token.ERROR;
\r
1452 string_builder.Append ((char) c);
\r
1455 Report.Error (1039, Location, "Unterminated string literal");
\r
1459 private int consume_identifier (int c, bool quoted)
\r
1461 id_builder.Length = 0;
\r
1463 id_builder.Append ((char) c);
\r
1465 while ((c = reader.Read ()) != -1) {
\r
1466 if (is_identifier_part_character ((char) c)){
\r
1467 id_builder.Append ((char)c);
\r
1468 putback_char = -1;
\r
1476 string ids = id_builder.ToString ();
\r
1477 int keyword = GetKeyword (ids);
\r
1479 if (keyword == -1 || quoted){
\r
1481 if (ids.Length > 512){
\r
1484 "Identifier too long (limit is 512 chars)");
\r
1486 return Token.IDENTIFIER;
\r
1492 public int xtoken ()
\r
1495 bool doread = false;
\r
1499 // optimization: eliminate col and implement #directive semantic correctly.
\r
1500 for (;(c = getChar ()) != -1; col++) {
\r
1501 if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r' || c == 0xa0){
\r
1504 col = (((col + 8) / 8) * 8) - 1;
\r
1508 // Handle double-slash comments.
\r
1510 int d = peekChar ();
\r
1514 while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
\r
1521 any_token_seen |= tokens_seen;
\r
1522 tokens_seen = false;
\r
1524 } else if (d == '*'){
\r
1527 while ((d = getChar ()) != -1){
\r
1528 if (d == '*' && peekChar () == '/'){
\r
1537 any_token_seen |= tokens_seen;
\r
1538 tokens_seen = false;
\r
1543 goto is_punct_label;
\r
1547 if (is_identifier_start_character ((char)c)){
\r
1548 tokens_seen = true;
\r
1549 return consume_identifier (c, false);
\r
1553 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
\r
1554 tokens_seen = true;
\r
1567 any_token_seen |= tokens_seen;
\r
1568 tokens_seen = false;
\r
1572 if (c >= '0' && c <= '9'){
\r
1573 tokens_seen = true;
\r
1574 return is_number (c);
\r
1578 tokens_seen = true;
\r
1579 int peek = peekChar ();
\r
1580 if (peek >= '0' && peek <= '9')
\r
1581 return is_number (c);
\r
1585 /* For now, ignore pre-processor commands */
\r
1586 // FIXME: In C# the '#' is not limited to appear
\r
1587 // on the first column.
\r
1588 if (c == '#' && !tokens_seen){
\r
1593 cont = handle_preprocessing_directive (cont);
\r
1601 bool skipping = false;
\r
1602 for (;(c = getChar ()) != -1; col++){
\r
1608 } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
\r
1610 else if (c != '#')
\r
1612 if (c == '#' && !skipping)
\r
1615 any_token_seen |= tokens_seen;
\r
1616 tokens_seen = false;
\r
1618 Report.Error (1027, Location, "#endif/#endregion expected");
\r
1623 return consume_string (false);
\r
1627 tokens_seen = true;
\r
1629 error_details = "Empty character literal";
\r
1630 Report.Error (1011, Location, error_details);
\r
1631 return Token.ERROR;
\r
1635 return Token.ERROR;
\r
1636 val = new System.Char ();
\r
1641 error_details = "Too many characters in character literal";
\r
1642 Report.Error (1012, Location, error_details);
\r
1644 // Try to recover, read until newline or next "'"
\r
1645 while ((c = getChar ()) != -1){
\r
1646 if (c == '\n' || c == '\''){
\r
1655 return Token.ERROR;
\r
1657 return Token.LITERAL_CHARACTER;
\r
1663 tokens_seen = true;
\r
1664 return consume_string (true);
\r
1665 } else if (is_identifier_start_character ((char) c)){
\r
1666 return consume_identifier (c, true);
\r
1668 Report.Error (1033, Location, "'@' must be followed by string constant or identifier");
\r
1672 error_details = ((char)c).ToString ();
\r
1674 return Token.ERROR;
\r
1677 if (ifstack != null && ifstack.Count >= 1)
\r
1678 Report.Error (1027, Location, "#endif/#endregion expected");
\r