2004-01-21 Rafael Teixeira <rafaelteixeirabr@hotmail.com>
[mono.git] / mcs / gmcs / cs-tokenizer.cs
1 //\r
2 // cs-tokenizer.cs: The Tokenizer for the C# compiler\r
3 //                  This also implements the preprocessor\r
4 //\r
5 // Author: Miguel de Icaza (miguel@gnu.org)\r
6 //\r
7 // Licensed under the terms of the GNU GPL\r
8 //\r
9 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)\r
10 //\r
11 \r
12 /*\r
13  * TODO:\r
14  *   Make sure we accept the proper Unicode ranges, per the spec.\r
15  *   Report error 1032\r
16 */\r
17 \r
18 using System;\r
19 using System.Text;\r
20 using System.Collections;\r
21 using System.IO;\r
22 using System.Globalization;\r
23 using System.Reflection;\r
24 \r
25 namespace Mono.CSharp\r
26 {\r
27         /// <summary>\r
28         ///    Tokenizer for C# source code. \r
29         /// </summary>\r
30 \r
31         public class Tokenizer : yyParser.yyInput\r
32         {\r
33                 SeekableStreamReader reader;\r
34                 public SourceFile ref_name;\r
35                 public SourceFile file_name;\r
36                 public int ref_line = 1;\r
37                 public int line = 1;\r
38                 public int col = 1;\r
39                 public int current_token;\r
40                 bool handle_get_set = false;\r
41                 bool handle_remove_add = false;\r
42                 bool handle_assembly = false;\r
43                 bool handle_constraints = false;\r
44 \r
45                 //\r
46                 // Whether tokens have been seen on this line\r
47                 //\r
48                 bool tokens_seen = false;\r
49 \r
50                 //\r
51                 // Whether a token has been seen on the file\r
52                 // This is needed because `define' is not allowed to be used\r
53                 // after a token has been seen.\r
54                 //\r
55                 bool any_token_seen = false;\r
56                 static Hashtable tokenValues;\r
57                 \r
58                 private static Hashtable TokenValueName\r
59                 {\r
60                         get {\r
61                                 if (tokenValues == null)\r
62                                         tokenValues = GetTokenValueNameHash ();\r
63 \r
64                                 return tokenValues;\r
65                         }\r
66                 }\r
67 \r
68                 private static Hashtable GetTokenValueNameHash ()\r
69                 {\r
70                         Type t = typeof (Token);\r
71                         FieldInfo [] fields = t.GetFields ();\r
72                         Hashtable hash = new Hashtable ();\r
73                         foreach (FieldInfo field in fields) {\r
74                                 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))\r
75                                         hash.Add (field.GetValue (null), field.Name);\r
76                         }\r
77                         return hash;\r
78                 }\r
79                 \r
80                 //\r
81                 // Returns a verbose representation of the current location\r
82                 //\r
83                 public string location {\r
84                         get {\r
85                                 string det;\r
86 \r
87                                 if (current_token == Token.ERROR)\r
88                                         det = "detail: " + error_details;\r
89                                 else\r
90                                         det = "";\r
91                                 \r
92                                 // return "Line:     "+line+" Col: "+col + "\n" +\r
93                                 //       "VirtLine: "+ref_line +\r
94                                 //       " Token: "+current_token + " " + det;\r
95                                 string current_token_name = TokenValueName [current_token] as string;\r
96                                 if (current_token_name == null)\r
97                                         current_token_name = current_token.ToString ();\r
98 \r
99                                 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,\r
100                                                                                        ref_line,\r
101                                                                                        col,\r
102                                                                                        current_token_name,\r
103                                                                                        det);\r
104                         }\r
105                 }\r
106 \r
107                 public bool PropertyParsing {\r
108                         get {\r
109                                 return handle_get_set;\r
110                         }\r
111 \r
112                         set {\r
113                                 handle_get_set = value;\r
114                         }\r
115                 }\r
116 \r
117                 public bool AssemblyTargetParsing {\r
118                         get {\r
119                                 return handle_assembly;\r
120                         }\r
121 \r
122                         set {\r
123                                 handle_assembly = value;\r
124                         }\r
125                 }\r
126 \r
127                 public bool EventParsing {\r
128                         get {\r
129                                 return handle_remove_add;\r
130                         }\r
131 \r
132                         set {\r
133                                 handle_remove_add = value;\r
134                         }\r
135                 }\r
136 \r
137                 public bool ConstraintsParsing {\r
138                         get {\r
139                                 return handle_constraints;\r
140                         }\r
141 \r
142                         set {\r
143                                 handle_constraints = value;\r
144                         }\r
145                 }\r
146                 \r
147                 //\r
148                 // Class variables\r
149                 // \r
150                 static CharArrayHashtable[] keywords;\r
151                 static NumberStyles styles;\r
152                 static NumberFormatInfo csharp_format_info;\r
153                 \r
154                 //\r
155                 // Values for the associated token returned\r
156                 //\r
157                 int putback_char;\r
158                 Object val;\r
159 \r
160                 //\r
161                 // Pre-processor\r
162                 //\r
163                 Hashtable defines;\r
164 \r
165                 const int TAKING        = 1;\r
166                 const int TAKEN_BEFORE  = 2;\r
167                 const int ELSE_SEEN     = 4;\r
168                 const int PARENT_TAKING = 8;\r
169                 const int REGION        = 16;           \r
170 \r
171                 //\r
172                 // pre-processor if stack state:\r
173                 //\r
174                 Stack ifstack;\r
175 \r
176                 static System.Text.StringBuilder string_builder;\r
177 \r
178                 const int max_id_size = 512;\r
179                 static char [] id_builder = new char [max_id_size];\r
180 \r
181                 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];\r
182 \r
183                 const int max_number_size = 128;\r
184                 static char [] number_builder = new char [max_number_size];\r
185                 static int number_pos;\r
186                 \r
187                 //\r
188                 // Details about the error encoutered by the tokenizer\r
189                 //\r
190                 string error_details;\r
191                 \r
192                 public string error {\r
193                         get {\r
194                                 return error_details;\r
195                         }\r
196                 }\r
197                 \r
198                 public int Line {\r
199                         get {\r
200                                 return ref_line;\r
201                         }\r
202                 }\r
203 \r
204                 public int Col {\r
205                         get {\r
206                                 return col;\r
207                         }\r
208                 }\r
209 \r
210                 static void AddKeyword (string kw, int token) {\r
211                         if (keywords [kw.Length] == null) {\r
212                                 keywords [kw.Length] = new CharArrayHashtable (kw.Length);\r
213                         }\r
214                         keywords [kw.Length] [kw.ToCharArray ()] = token;\r
215                 }\r
216 \r
217                 static void InitTokens ()\r
218                 {\r
219                         keywords = new CharArrayHashtable [64];\r
220 \r
221                         AddKeyword ("abstract", Token.ABSTRACT);\r
222                         AddKeyword ("as", Token.AS);\r
223                         AddKeyword ("add", Token.ADD);\r
224                         AddKeyword ("assembly", Token.ASSEMBLY);\r
225                         AddKeyword ("base", Token.BASE);\r
226                         AddKeyword ("bool", Token.BOOL);\r
227                         AddKeyword ("break", Token.BREAK);\r
228                         AddKeyword ("byte", Token.BYTE);\r
229                         AddKeyword ("case", Token.CASE);\r
230                         AddKeyword ("catch", Token.CATCH);\r
231                         AddKeyword ("char", Token.CHAR);\r
232                         AddKeyword ("checked", Token.CHECKED);\r
233                         AddKeyword ("class", Token.CLASS);\r
234                         AddKeyword ("const", Token.CONST);\r
235                         AddKeyword ("continue", Token.CONTINUE);\r
236                         AddKeyword ("decimal", Token.DECIMAL);\r
237                         AddKeyword ("default", Token.DEFAULT);\r
238                         AddKeyword ("delegate", Token.DELEGATE);\r
239                         AddKeyword ("do", Token.DO);\r
240                         AddKeyword ("double", Token.DOUBLE);\r
241                         AddKeyword ("else", Token.ELSE);\r
242                         AddKeyword ("enum", Token.ENUM);\r
243                         AddKeyword ("event", Token.EVENT);\r
244                         AddKeyword ("explicit", Token.EXPLICIT);\r
245                         AddKeyword ("extern", Token.EXTERN);\r
246                         AddKeyword ("false", Token.FALSE);\r
247                         AddKeyword ("finally", Token.FINALLY);\r
248                         AddKeyword ("fixed", Token.FIXED);\r
249                         AddKeyword ("float", Token.FLOAT);\r
250                         AddKeyword ("for", Token.FOR);\r
251                         AddKeyword ("foreach", Token.FOREACH);\r
252                         AddKeyword ("goto", Token.GOTO);\r
253                         AddKeyword ("get", Token.GET);\r
254                         AddKeyword ("if", Token.IF);\r
255                         AddKeyword ("implicit", Token.IMPLICIT);\r
256                         AddKeyword ("in", Token.IN);\r
257                         AddKeyword ("int", Token.INT);\r
258                         AddKeyword ("interface", Token.INTERFACE);\r
259                         AddKeyword ("internal", Token.INTERNAL);\r
260                         AddKeyword ("is", Token.IS);\r
261                         AddKeyword ("lock", Token.LOCK);\r
262                         AddKeyword ("long", Token.LONG);\r
263                         AddKeyword ("namespace", Token.NAMESPACE);\r
264                         AddKeyword ("new", Token.NEW);\r
265                         AddKeyword ("null", Token.NULL);\r
266                         AddKeyword ("object", Token.OBJECT);\r
267                         AddKeyword ("operator", Token.OPERATOR);\r
268                         AddKeyword ("out", Token.OUT);\r
269                         AddKeyword ("override", Token.OVERRIDE);\r
270                         AddKeyword ("params", Token.PARAMS);\r
271                         AddKeyword ("private", Token.PRIVATE);\r
272                         AddKeyword ("protected", Token.PROTECTED);\r
273                         AddKeyword ("public", Token.PUBLIC);\r
274                         AddKeyword ("readonly", Token.READONLY);\r
275                         AddKeyword ("ref", Token.REF);\r
276                         AddKeyword ("remove", Token.REMOVE);\r
277                         AddKeyword ("return", Token.RETURN);\r
278                         AddKeyword ("sbyte", Token.SBYTE);\r
279                         AddKeyword ("sealed", Token.SEALED);\r
280                         AddKeyword ("set", Token.SET);\r
281                         AddKeyword ("short", Token.SHORT);\r
282                         AddKeyword ("sizeof", Token.SIZEOF);\r
283                         AddKeyword ("stackalloc", Token.STACKALLOC);\r
284                         AddKeyword ("static", Token.STATIC);\r
285                         AddKeyword ("string", Token.STRING);\r
286                         AddKeyword ("struct", Token.STRUCT);\r
287                         AddKeyword ("switch", Token.SWITCH);\r
288                         AddKeyword ("this", Token.THIS);\r
289                         AddKeyword ("throw", Token.THROW);\r
290                         AddKeyword ("true", Token.TRUE);\r
291                         AddKeyword ("try", Token.TRY);\r
292                         AddKeyword ("typeof", Token.TYPEOF);\r
293                         AddKeyword ("uint", Token.UINT);\r
294                         AddKeyword ("ulong", Token.ULONG);\r
295                         AddKeyword ("unchecked", Token.UNCHECKED);\r
296                         AddKeyword ("unsafe", Token.UNSAFE);\r
297                         AddKeyword ("ushort", Token.USHORT);\r
298                         AddKeyword ("using", Token.USING);\r
299                         AddKeyword ("virtual", Token.VIRTUAL);\r
300                         AddKeyword ("void", Token.VOID);\r
301                         AddKeyword ("volatile", Token.VOLATILE);\r
302                         AddKeyword ("where", Token.WHERE);\r
303                         AddKeyword ("while", Token.WHILE);\r
304 \r
305                         if (RootContext.V2){\r
306                                 AddKeyword ("__yield", Token.YIELD);\r
307                                 AddKeyword ("yield", Token.YIELD);\r
308                         }\r
309                 }\r
310 \r
311                 //\r
312                 // Class initializer\r
313                 // \r
314                 static Tokenizer ()\r
315                 {\r
316                         InitTokens ();\r
317                         csharp_format_info = NumberFormatInfo.InvariantInfo;\r
318                         styles = NumberStyles.Float;\r
319                         \r
320                         string_builder = new System.Text.StringBuilder ();\r
321                 }\r
322 \r
323                 int GetKeyword (char[] id, int id_len)\r
324                 {\r
325                         /*\r
326                          * Keywords are stored in an array of hashtables grouped by their\r
327                          * length.\r
328                          */\r
329 \r
330                         if ((id_len >= keywords.Length) || (keywords [id_len] == null))\r
331                                 return -1;\r
332                         object o = keywords [id_len] [id];\r
333 \r
334                         if (o == null)\r
335                                 return -1;\r
336                         \r
337                         int res = (int) o;\r
338 \r
339                         if (handle_get_set == false && (res == Token.GET || res == Token.SET))\r
340                                 return -1;\r
341                         if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))\r
342                                 return -1;\r
343                         if (handle_assembly == false && res == Token.ASSEMBLY)\r
344                                 return -1;\r
345                         if (handle_constraints == false && res == Token.WHERE)\r
346                                 return -1;\r
347                         return res;\r
348                         \r
349                 }\r
350 \r
351                 public Location Location {\r
352                         get {\r
353                                 return new Location (ref_line);\r
354                         }\r
355                 }\r
356 \r
357                 void define (string def)\r
358                 {\r
359                         if (!RootContext.AllDefines.Contains (def)){\r
360                                 RootContext.AllDefines [def] = true;\r
361                         }\r
362                         if (defines.Contains (def))\r
363                                 return;\r
364                         defines [def] = true;\r
365                 }\r
366                 \r
367                 public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)\r
368                 {\r
369                         this.ref_name = file;\r
370                         this.file_name = file;\r
371                         reader = input;\r
372                         \r
373                         putback_char = -1;\r
374 \r
375                         if (defs != null){\r
376                                 defines = new Hashtable ();\r
377                                 foreach (string def in defs)\r
378                                         define (def);\r
379                         }\r
380 \r
381                         //\r
382                         // FIXME: This could be `Location.Push' but we have to\r
383                         // find out why the MS compiler allows this\r
384                         //\r
385                         Mono.CSharp.Location.Push (file);\r
386                 }\r
387 \r
388                 public static void Cleanup () {\r
389                         identifiers = null;\r
390                 }\r
391 \r
392                 bool is_identifier_start_character (char c)\r
393                 {\r
394                         return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c);\r
395                 }\r
396 \r
397                 bool is_identifier_part_character (char c)\r
398                 {\r
399                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);\r
400                 }\r
401 \r
402                 bool parse_less_than ()\r
403                 {\r
404                 start:\r
405                         int the_token = token ();\r
406                         switch (the_token) {\r
407                         case Token.IDENTIFIER:\r
408                         case Token.OBJECT:\r
409                         case Token.STRING:\r
410                         case Token.BOOL:\r
411                         case Token.DECIMAL:\r
412                         case Token.FLOAT:\r
413                         case Token.DOUBLE:\r
414                         case Token.SBYTE:\r
415                         case Token.BYTE:\r
416                         case Token.SHORT:\r
417                         case Token.USHORT:\r
418                         case Token.INT:\r
419                         case Token.UINT:\r
420                         case Token.LONG:\r
421                         case Token.ULONG:\r
422                         case Token.CHAR:\r
423                         case Token.VOID:\r
424                                 break;\r
425 \r
426                         default:\r
427                                 return false;\r
428                         }\r
429                 again:\r
430                         the_token = token ();\r
431 \r
432                         if (the_token == Token.OP_GENERICS_GT)\r
433                                 return true;\r
434                         else if (the_token == Token.COMMA)\r
435                                 goto start;\r
436                         else if (the_token == Token.OP_GENERICS_LT) {\r
437                                 if (!parse_less_than ())\r
438                                         return false;\r
439                                 goto again;\r
440                         }\r
441 \r
442                         return false;\r
443                 }\r
444 \r
445                 bool parsing_less_than = false;\r
446                 int parsing_generic_less_than = 0;\r
447 \r
448                 int is_punct (char c, ref bool doread)\r
449                 {\r
450                         int d;\r
451                         int t;\r
452 \r
453                         doread = false;\r
454 \r
455                         switch (c){\r
456                         case '{':\r
457                                 return Token.OPEN_BRACE;\r
458                         case '}':\r
459                                 return Token.CLOSE_BRACE;\r
460                         case '[':\r
461                                 return Token.OPEN_BRACKET;\r
462                         case ']':\r
463                                 return Token.CLOSE_BRACKET;\r
464                         case '(':\r
465                                 return Token.OPEN_PARENS;\r
466                         case ')': {\r
467                                 if (deambiguate_close_parens == 0)\r
468                                         return Token.CLOSE_PARENS;\r
469 \r
470                                 --deambiguate_close_parens;\r
471 \r
472                                 // Save current position and parse next token.\r
473                                 int old = reader.Position;\r
474                                 int new_token = token ();\r
475                                 reader.Position = old;\r
476                                 putback_char = -1;\r
477 \r
478                                 if (new_token == Token.OPEN_PARENS)\r
479                                         return Token.CLOSE_PARENS_OPEN_PARENS;\r
480                                 else if (new_token == Token.MINUS)\r
481                                         return Token.CLOSE_PARENS_MINUS;\r
482                                 else if (IsCastToken (new_token))\r
483                                         return Token.CLOSE_PARENS_CAST;\r
484                                 else\r
485                                         return Token.CLOSE_PARENS_NO_CAST;\r
486                         }\r
487 \r
488                         case ',':\r
489                                 return Token.COMMA;\r
490                         case ':':\r
491                                 return Token.COLON;\r
492                         case ';':\r
493                                 return Token.SEMICOLON;\r
494                         case '~':\r
495                                 return Token.TILDE;\r
496                         case '?':\r
497                                 return Token.INTERR;\r
498                         }\r
499 \r
500                         if (c == '<') {\r
501                                 if (parsing_generic_less_than++ > 0)\r
502                                         return Token.OP_GENERICS_LT;\r
503 \r
504                                 // Save current position and parse next token.\r
505                                 int old = reader.Position;\r
506                                 bool is_generic_lt = parse_less_than ();\r
507                                 reader.Position = old;\r
508                                 putback_char = -1;\r
509 \r
510                                 if (is_generic_lt) {\r
511                                         parsing_generic_less_than++;\r
512                                         return Token.OP_GENERICS_LT;\r
513                                 } else\r
514                                         parsing_generic_less_than = 0;\r
515 \r
516                                 d = peekChar ();\r
517                                 if (d == '<'){\r
518                                         getChar ();\r
519                                         d = peekChar ();\r
520 \r
521                                         if (d == '='){\r
522                                                 doread = true;\r
523                                                 return Token.OP_SHIFT_LEFT_ASSIGN;\r
524                                         }\r
525                                         return Token.OP_SHIFT_LEFT;\r
526                                 } else if (d == '='){\r
527                                         doread = true;\r
528                                         return Token.OP_LE;\r
529                                 }\r
530                                 return Token.OP_LT;\r
531                         } else if (c == '>') {\r
532                                 if (parsing_generic_less_than > 0) {\r
533                                         parsing_generic_less_than--;\r
534                                         return Token.OP_GENERICS_GT;\r
535                                 }\r
536 \r
537                                 d = peekChar ();\r
538                                 if (d == '>'){\r
539                                         getChar ();\r
540                                         d = peekChar ();\r
541 \r
542                                         if (d == '='){\r
543                                                 doread = true;\r
544                                                 return Token.OP_SHIFT_RIGHT_ASSIGN;\r
545                                         }\r
546                                         return Token.OP_SHIFT_RIGHT;\r
547                                 } else if (d == '='){\r
548                                         doread = true;\r
549                                         return Token.OP_GE;\r
550                                 }\r
551                                 return Token.OP_GT;\r
552                         }\r
553 \r
554                         d = peekChar ();\r
555                         if (c == '+'){\r
556                                 \r
557                                 if (d == '+')\r
558                                         t = Token.OP_INC;\r
559                                 else if (d == '=')\r
560                                         t = Token.OP_ADD_ASSIGN;\r
561                                 else\r
562                                         return Token.PLUS;\r
563                                 doread = true;\r
564                                 return t;\r
565                         }\r
566                         if (c == '-'){\r
567                                 if (d == '-')\r
568                                         t = Token.OP_DEC;\r
569                                 else if (d == '=')\r
570                                         t = Token.OP_SUB_ASSIGN;\r
571                                 else if (d == '>')\r
572                                         t = Token.OP_PTR;\r
573                                 else\r
574                                         return Token.MINUS;\r
575                                 doread = true;\r
576                                 return t;\r
577                         }\r
578 \r
579                         if (c == '!'){\r
580                                 if (d == '='){\r
581                                         doread = true;\r
582                                         return Token.OP_NE;\r
583                                 }\r
584                                 return Token.BANG;\r
585                         }\r
586 \r
587                         if (c == '='){\r
588                                 if (d == '='){\r
589                                         doread = true;\r
590                                         return Token.OP_EQ;\r
591                                 }\r
592                                 return Token.ASSIGN;\r
593                         }\r
594 \r
595                         if (c == '&'){\r
596                                 if (d == '&'){\r
597                                         doread = true;\r
598                                         return Token.OP_AND;\r
599                                 } else if (d == '='){\r
600                                         doread = true;\r
601                                         return Token.OP_AND_ASSIGN;\r
602                                 }\r
603                                 return Token.BITWISE_AND;\r
604                         }\r
605 \r
606                         if (c == '|'){\r
607                                 if (d == '|'){\r
608                                         doread = true;\r
609                                         return Token.OP_OR;\r
610                                 } else if (d == '='){\r
611                                         doread = true;\r
612                                         return Token.OP_OR_ASSIGN;\r
613                                 }\r
614                                 return Token.BITWISE_OR;\r
615                         }\r
616 \r
617                         if (c == '*'){\r
618                                 if (d == '='){\r
619                                         doread = true;\r
620                                         return Token.OP_MULT_ASSIGN;\r
621                                 }\r
622                                 return Token.STAR;\r
623                         }\r
624 \r
625                         if (c == '/'){\r
626                                 if (d == '='){\r
627                                         doread = true;\r
628                                         return Token.OP_DIV_ASSIGN;\r
629                                 }\r
630                                 return Token.DIV;\r
631                         }\r
632 \r
633                         if (c == '%'){\r
634                                 if (d == '='){\r
635                                         doread = true;\r
636                                         return Token.OP_MOD_ASSIGN;\r
637                                 }\r
638                                 return Token.PERCENT;\r
639                         }\r
640 \r
641                         if (c == '^'){\r
642                                 if (d == '='){\r
643                                         doread = true;\r
644                                         return Token.OP_XOR_ASSIGN;\r
645                                 }\r
646                                 return Token.CARRET;\r
647                         }\r
648 \r
649 #if FIXME\r
650                         if (c == '>'){\r
651                                 if (deambiguate_greater_than == 0)\r
652                                         return Token.OP_GT;\r
653 \r
654                                 --deambiguate_greater_than;\r
655 \r
656                                 // Save current position and parse next token.\r
657                                 int old = reader.Position;\r
658                                 int new_token = token ();\r
659                                 reader.Position = old;\r
660                                 putback_char = -1;\r
661 \r
662                                 switch (new_token) {\r
663                                 case Token.OPEN_PARENS:\r
664                                 case Token.CLOSE_PARENS:\r
665                                 case Token.CLOSE_BRACKET:\r
666                                 case Token.OP_GT:\r
667                                 case Token.COLON:\r
668                                 case Token.SEMICOLON:\r
669                                 case Token.COMMA:\r
670                                 case Token.DOT:\r
671                                 case Token.INTERR:\r
672                                         return Token.OP_GENERICS_GT;\r
673 \r
674                                 default:\r
675                                         return Token.OP_GT;\r
676                                 }\r
677                         }\r
678 #endif\r
679 \r
680                         return Token.ERROR;\r
681                 }\r
682 \r
683                 int deambiguate_close_parens = 0;\r
684 \r
685                 public void Deambiguate_CloseParens ()\r
686                 {\r
687                         putback (')');\r
688                         deambiguate_close_parens++;\r
689                 }\r
690 \r
691                 void Error_NumericConstantTooLong ()\r
692                 {\r
693                         Report.Error (1021, Location, "Numeric constant too long");                     \r
694                 }\r
695                 \r
696                 bool decimal_digits (int c)\r
697                 {\r
698                         int d;\r
699                         bool seen_digits = false;\r
700                         \r
701                         if (c != -1){\r
702                                 if (number_pos == max_number_size)\r
703                                         Error_NumericConstantTooLong ();\r
704                                 number_builder [number_pos++] = (char) c;\r
705                         }\r
706                         \r
707                         //\r
708                         // We use peekChar2, because decimal_digits needs to do a \r
709                         // 2-character look-ahead (5.ToString for example).\r
710                         //\r
711                         while ((d = peekChar2 ()) != -1){\r
712                                 if (d >= '0' && d <= '9'){\r
713                                         if (number_pos == max_number_size)\r
714                                                 Error_NumericConstantTooLong ();\r
715                                         number_builder [number_pos++] = (char) d;\r
716                                         getChar ();\r
717                                         seen_digits = true;\r
718                                 } else\r
719                                         break;\r
720                         }\r
721                         \r
722                         return seen_digits;\r
723                 }\r
724 \r
725                 bool is_hex (int e)\r
726                 {\r
727                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');\r
728                 }\r
729                 \r
730                 void hex_digits (int c)\r
731                 {\r
732                         if (c != -1)\r
733                                 number_builder [number_pos++] = (char) c;\r
734                         \r
735                 }\r
736                 \r
737                 int real_type_suffix (int c)\r
738                 {\r
739                         int t;\r
740 \r
741                         switch (c){\r
742                         case 'F': case 'f':\r
743                                 t =  Token.LITERAL_FLOAT;\r
744                                 break;\r
745                         case 'D': case 'd':\r
746                                 t = Token.LITERAL_DOUBLE;\r
747                                 break;\r
748                         case 'M': case 'm':\r
749                                  t= Token.LITERAL_DECIMAL;\r
750                                 break;\r
751                         default:\r
752                                 return Token.NONE;\r
753                         }\r
754                         return t;\r
755                 }\r
756 \r
757                 int integer_type_suffix (ulong ul, int c)\r
758                 {\r
759                         bool is_unsigned = false;\r
760                         bool is_long = false;\r
761 \r
762                         if (c != -1){\r
763                                 bool scanning = true;\r
764                                 do {\r
765                                         switch (c){\r
766                                         case 'U': case 'u':\r
767                                                 if (is_unsigned)\r
768                                                         scanning = false;\r
769                                                 is_unsigned = true;\r
770                                                 getChar ();\r
771                                                 break;\r
772 \r
773                                         case 'l':\r
774                                                 if (!is_unsigned){\r
775                                                         //\r
776                                                         // if we have not seen anything in between\r
777                                                         // report this error\r
778                                                         //\r
779                                                         Report.Warning (\r
780                                                                 78, Location,\r
781                                                         "the 'l' suffix is easily confused with digit `1'," +\r
782                                                         " use 'L' for clarity");\r
783                                                 }\r
784                                                 goto case 'L';\r
785                                                 \r
786                                         case 'L': \r
787                                                 if (is_long)\r
788                                                         scanning = false;\r
789                                                 is_long = true;\r
790                                                 getChar ();\r
791                                                 break;\r
792                                                 \r
793                                         default:\r
794                                                 scanning = false;\r
795                                                 break;\r
796                                         }\r
797                                         c = peekChar ();\r
798                                 } while (scanning);\r
799                         }\r
800 \r
801                         if (is_long && is_unsigned){\r
802                                 val = ul;\r
803                                 return Token.LITERAL_INTEGER;\r
804                         } else if (is_unsigned){\r
805                                 // uint if possible, or ulong else.\r
806 \r
807                                 if ((ul & 0xffffffff00000000) == 0)\r
808                                         val = (uint) ul;\r
809                                 else\r
810                                         val = ul;\r
811                         } else if (is_long){\r
812                                 // long if possible, ulong otherwise\r
813                                 if ((ul & 0x8000000000000000) != 0)\r
814                                         val = ul;\r
815                                 else\r
816                                         val = (long) ul;\r
817                         } else {\r
818                                 // int, uint, long or ulong in that order\r
819                                 if ((ul & 0xffffffff00000000) == 0){\r
820                                         uint ui = (uint) ul;\r
821                                         \r
822                                         if ((ui & 0x80000000) != 0)\r
823                                                 val = ui;\r
824                                         else\r
825                                                 val = (int) ui;\r
826                                 } else {\r
827                                         if ((ul & 0x8000000000000000) != 0)\r
828                                                 val = ul;\r
829                                         else\r
830                                                 val = (long) ul;\r
831                                 }\r
832                         }\r
833                         return Token.LITERAL_INTEGER;\r
834                 }\r
835                                 \r
836                 //\r
837                 // given `c' as the next char in the input decide whether\r
838                 // we need to convert to a special type, and then choose\r
839                 // the best representation for the integer\r
840                 //\r
841                 int adjust_int (int c)\r
842                 {\r
843                         try {\r
844                                 if (number_pos > 9){\r
845                                         ulong ul = (uint) (number_builder [0] - '0');\r
846 \r
847                                         for (int i = 1; i < number_pos; i++){\r
848                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));\r
849                                         }\r
850                                         return integer_type_suffix (ul, c);\r
851                                 } else {\r
852                                         uint ui = (uint) (number_builder [0] - '0');\r
853 \r
854                                         for (int i = 1; i < number_pos; i++){\r
855                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));\r
856                                         }\r
857                                         return integer_type_suffix (ui, c);\r
858                                 }\r
859                         } catch (OverflowException) {\r
860                                 error_details = "Integral constant is too large";\r
861                                 Report.Error (1021, Location, error_details);\r
862                                 val = 0ul;\r
863                                 return Token.LITERAL_INTEGER;\r
864                         }\r
865                 }\r
866                 \r
867                 int adjust_real (int t)\r
868                 {\r
869                         string s = new String (number_builder, 0, number_pos);\r
870 \r
871                         switch (t){\r
872                         case Token.LITERAL_DECIMAL:\r
873                                 try {\r
874                                         val = System.Decimal.Parse (s, styles, csharp_format_info);\r
875                                 } catch (OverflowException) {\r
876                                         val = 0m;     \r
877                                         error_details = "Floating-point constant is outside the range of the type 'decimal'";\r
878                                         Report.Error (594, Location, error_details);\r
879                                 }\r
880                                 break;\r
881                         case Token.LITERAL_FLOAT:\r
882                                 try {\r
883                                         val = (float) System.Double.Parse (s, styles, csharp_format_info);\r
884                                 } catch (OverflowException) {\r
885                                         val = 0.0f;     \r
886                                         error_details = "Floating-point constant is outside the range of the type 'float'";\r
887                                         Report.Error (594, Location, error_details);\r
888                                 }\r
889                                 break;\r
890                                 \r
891                         case Token.LITERAL_DOUBLE:\r
892                         case Token.NONE:\r
893                                 t = Token.LITERAL_DOUBLE;\r
894                                 try {\r
895                                         val = System.Double.Parse (s, styles, csharp_format_info);\r
896                                 } catch (OverflowException) {\r
897                                         val = 0.0;     \r
898                                         error_details = "Floating-point constant is outside the range of the type 'double'";\r
899                                         Report.Error (594, Location, error_details);\r
900                                 }\r
901                                 break;\r
902                         }\r
903                         return t;\r
904                 }\r
905 \r
906                 int handle_hex ()\r
907                 {\r
908                         int d;\r
909                         ulong ul;\r
910                         \r
911                         getChar ();\r
912                         while ((d = peekChar ()) != -1){\r
913                                 if (is_hex (d)){\r
914                                         number_builder [number_pos++] = (char) d;\r
915                                         getChar ();\r
916                                 } else\r
917                                         break;\r
918                         }\r
919                         \r
920                         string s = new String (number_builder, 0, number_pos);\r
921                         try {\r
922                                 if (number_pos <= 8)\r
923                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);\r
924                                 else\r
925                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);\r
926                         } catch (OverflowException){\r
927                                 error_details = "Integral constant is too large";\r
928                                 Report.Error (1021, Location, error_details);\r
929                                 val = 0ul;\r
930                                 return Token.LITERAL_INTEGER;\r
931                         }\r
932                         \r
933                         return integer_type_suffix (ul, peekChar ());\r
934                 }\r
935 \r
936                 //\r
937                 // Invoked if we know we have .digits or digits\r
938                 //\r
939                 int is_number (int c)\r
940                 {\r
941                         bool is_real = false;\r
942                         int type;\r
943 \r
944                         number_pos = 0;\r
945 \r
946                         if (c >= '0' && c <= '9'){\r
947                                 if (c == '0'){\r
948                                         int peek = peekChar ();\r
949 \r
950                                         if (peek == 'x' || peek == 'X')\r
951                                                 return handle_hex ();\r
952                                 }\r
953                                 decimal_digits (c);\r
954                                 c = getChar ();\r
955                         }\r
956 \r
957                         //\r
958                         // We need to handle the case of\r
959                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)\r
960                         //\r
961                         if (c == '.'){\r
962                                 if (decimal_digits ('.')){\r
963                                         is_real = true;\r
964                                         c = getChar ();\r
965                                 } else {\r
966                                         putback ('.');\r
967                                         number_pos--;\r
968                                         return adjust_int (-1);\r
969                                 }\r
970                         }\r
971                         \r
972                         if (c == 'e' || c == 'E'){\r
973                                 is_real = true;\r
974                                 if (number_pos == max_number_size)\r
975                                         Error_NumericConstantTooLong ();\r
976                                 number_builder [number_pos++] = 'e';\r
977                                 c = getChar ();\r
978                                 \r
979                                 if (c == '+'){\r
980                                         if (number_pos == max_number_size)\r
981                                                 Error_NumericConstantTooLong ();\r
982                                         number_builder [number_pos++] = '+';\r
983                                         c = -1;\r
984                                 } else if (c == '-') {\r
985                                         if (number_pos == max_number_size)\r
986                                                 Error_NumericConstantTooLong ();\r
987                                         number_builder [number_pos++] = '-';\r
988                                         c = -1;\r
989                                 } else {\r
990                                         if (number_pos == max_number_size)\r
991                                                 Error_NumericConstantTooLong ();\r
992                                         number_builder [number_pos++] = '+';\r
993                                 }\r
994                                         \r
995                                 decimal_digits (c);\r
996                                 c = getChar ();\r
997                         }\r
998 \r
999                         type = real_type_suffix (c);\r
1000                         if (type == Token.NONE && !is_real){\r
1001                                 putback (c);\r
1002                                 return adjust_int (c);\r
1003                         } else \r
1004                                 is_real = true;\r
1005 \r
1006                         if (type == Token.NONE){\r
1007                                 putback (c);\r
1008                         }\r
1009                         \r
1010                         if (is_real)\r
1011                                 return adjust_real (type);\r
1012 \r
1013                         Console.WriteLine ("This should not be reached");\r
1014                         throw new Exception ("Is Number should never reach this point");\r
1015                 }\r
1016 \r
1017                 //\r
1018                 // Accepts exactly count (4 or 8) hex, no more no less\r
1019                 //\r
1020                 int getHex (int count, out bool error)\r
1021                 {\r
1022                         int i;\r
1023                         int total = 0;\r
1024                         int c;\r
1025                         int top = count != -1 ? count : 4;\r
1026                         \r
1027                         getChar ();\r
1028                         error = false;\r
1029                         for (i = 0; i < top; i++){\r
1030                                 c = getChar ();\r
1031                                 \r
1032                                 if (c >= '0' && c <= '9')\r
1033                                         c = (int) c - (int) '0';\r
1034                                 else if (c >= 'A' && c <= 'F')\r
1035                                         c = (int) c - (int) 'A' + 10;\r
1036                                 else if (c >= 'a' && c <= 'f')\r
1037                                         c = (int) c - (int) 'a' + 10;\r
1038                                 else {\r
1039                                         error = true;\r
1040                                         return 0;\r
1041                                 }\r
1042                                 \r
1043                                 total = (total * 16) + c;\r
1044                                 if (count == -1){\r
1045                                         int p = peekChar ();\r
1046                                         if (p == -1)\r
1047                                                 break;\r
1048                                         if (!is_hex ((char)p))\r
1049                                                 break;\r
1050                                 }\r
1051                         }\r
1052                         return total;\r
1053                 }\r
1054 \r
1055                 int escape (int c)\r
1056                 {\r
1057                         bool error;\r
1058                         int d;\r
1059                         int v;\r
1060 \r
1061                         d = peekChar ();\r
1062                         if (c != '\\')\r
1063                                 return c;\r
1064                         \r
1065                         switch (d){\r
1066                         case 'a':\r
1067                                 v = '\a'; break;\r
1068                         case 'b':\r
1069                                 v = '\b'; break;\r
1070                         case 'n':\r
1071                                 v = '\n'; break;\r
1072                         case 't':\r
1073                                 v = '\t'; break;\r
1074                         case 'v':\r
1075                                 v = '\v'; break;\r
1076                         case 'r':\r
1077                                 v = '\r'; break;\r
1078                         case '\\':\r
1079                                 v = '\\'; break;\r
1080                         case 'f':\r
1081                                 v = '\f'; break;\r
1082                         case '0':\r
1083                                 v = 0; break;\r
1084                         case '"':\r
1085                                 v = '"'; break;\r
1086                         case '\'':\r
1087                                 v = '\''; break;\r
1088                         case 'x':\r
1089                                 v = getHex (-1, out error);\r
1090                                 if (error)\r
1091                                         goto default;\r
1092                                 return v;\r
1093                         case 'u':\r
1094                                 v = getHex (4, out error);\r
1095                                 if (error)\r
1096                                         goto default;\r
1097                                 return v;\r
1098                         case 'U':\r
1099                                 v = getHex (8, out error);\r
1100                                 if (error)\r
1101                                         goto default;\r
1102                                 return v;\r
1103                         default:\r
1104                                 Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);\r
1105                                 return d;\r
1106                         }\r
1107                         getChar ();\r
1108                         return v;\r
1109                 }\r
1110 \r
1111                 int getChar ()\r
1112                 {\r
1113                         if (putback_char != -1){\r
1114                                 int x = putback_char;\r
1115                                 putback_char = -1;\r
1116 \r
1117                                 return x;\r
1118                         }\r
1119                         return reader.Read ();\r
1120                 }\r
1121 \r
1122                 int peekChar ()\r
1123                 {\r
1124                         if (putback_char != -1)\r
1125                                 return putback_char;\r
1126                         putback_char = reader.Read ();\r
1127                         return putback_char;\r
1128                 }\r
1129 \r
1130                 int peekChar2 ()\r
1131                 {\r
1132                         if (putback_char != -1)\r
1133                                 return putback_char;\r
1134                         return reader.Peek ();\r
1135                 }\r
1136                 \r
1137                 void putback (int c)\r
1138                 {\r
1139                         if (putback_char != -1){\r
1140                                 Console.WriteLine ("Col: " + col);\r
1141                                 Console.WriteLine ("Row: " + line);\r
1142                                 Console.WriteLine ("Name: " + ref_name.Name);\r
1143                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);\r
1144                                 throw new Exception ("This should not happen putback on putback");\r
1145                         }\r
1146                         putback_char = c;\r
1147                 }\r
1148 \r
1149                 public bool advance ()\r
1150                 {\r
1151                         return peekChar () != -1;\r
1152                 }\r
1153 \r
1154                 public Object Value {\r
1155                         get {\r
1156                                 return val;\r
1157                         }\r
1158                 }\r
1159 \r
1160                 public Object value ()\r
1161                 {\r
1162                         return val;\r
1163                 }\r
1164 \r
1165                 bool IsCastToken (int token)\r
1166                 {\r
1167                         switch (token) {\r
1168                         case Token.BANG:\r
1169                         case Token.TILDE:\r
1170                         case Token.IDENTIFIER:\r
1171                         case Token.LITERAL_INTEGER:\r
1172                         case Token.LITERAL_FLOAT:\r
1173                         case Token.LITERAL_DOUBLE:\r
1174                         case Token.LITERAL_DECIMAL:\r
1175                         case Token.LITERAL_CHARACTER:\r
1176                         case Token.LITERAL_STRING:\r
1177                         case Token.BASE:\r
1178                         case Token.CHECKED:\r
1179                         case Token.FALSE:\r
1180                         case Token.FIXED:\r
1181                         case Token.NEW:\r
1182                         case Token.NULL:\r
1183                         case Token.SIZEOF:\r
1184                         case Token.THIS:\r
1185                         case Token.THROW:\r
1186                         case Token.TRUE:\r
1187                         case Token.TYPEOF:\r
1188                         case Token.UNCHECKED:\r
1189                         case Token.UNSAFE:\r
1190 \r
1191                                 //\r
1192                                 // These can be part of a member access\r
1193                                 //\r
1194                         case Token.INT:\r
1195                         case Token.UINT:\r
1196                         case Token.SHORT:\r
1197                         case Token.USHORT:\r
1198                         case Token.LONG:\r
1199                         case Token.ULONG:\r
1200                         case Token.DOUBLE:\r
1201                         case Token.FLOAT:\r
1202                         case Token.CHAR:\r
1203                                 return true;\r
1204 \r
1205                         default:\r
1206                                 return false;\r
1207                         }\r
1208                 }\r
1209 \r
1210                 public int token ()\r
1211                 {\r
1212                         current_token = xtoken ();\r
1213                         return current_token;\r
1214                 }\r
1215 \r
1216                 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();\r
1217                 \r
1218                 void get_cmd_arg (out string cmd, out string arg)\r
1219                 {\r
1220                         int c;\r
1221                         \r
1222                         tokens_seen = false;\r
1223                         arg = "";\r
1224                         static_cmd_arg.Length = 0;\r
1225                                 \r
1226                         while ((c = getChar ()) != -1 && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){\r
1227                                 static_cmd_arg.Append ((char) c);\r
1228                         }\r
1229 \r
1230                         cmd = static_cmd_arg.ToString ();\r
1231 \r
1232                         if (c == '\n'){\r
1233                                 line++;\r
1234                                 ref_line++;\r
1235                                 return;\r
1236                         } else if (c == '\r')\r
1237                                 col = 0;\r
1238 \r
1239                         // skip over white space\r
1240                         while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))\r
1241                                 ;\r
1242 \r
1243                         if (c == '\n'){\r
1244                                 line++;\r
1245                                 ref_line++;\r
1246                                 return;\r
1247                         } else if (c == '\r'){\r
1248                                 col = 0;\r
1249                                 return;\r
1250                         }\r
1251                         \r
1252                         static_cmd_arg.Length = 0;\r
1253                         static_cmd_arg.Append ((char) c);\r
1254                         \r
1255                         while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){\r
1256                                 static_cmd_arg.Append ((char) c);\r
1257                         }\r
1258 \r
1259                         if (c == '\n'){\r
1260                                 line++;\r
1261                                 ref_line++;\r
1262                         } else if (c == '\r')\r
1263                                 col = 0;\r
1264                         arg = static_cmd_arg.ToString ().Trim ();\r
1265                 }\r
1266 \r
1267                 //\r
1268                 // Handles the #line directive\r
1269                 //\r
1270                 bool PreProcessLine (string arg)\r
1271                 {\r
1272                         if (arg == "")\r
1273                                 return false;\r
1274 \r
1275                         if (arg == "default"){\r
1276                                 ref_line = line;\r
1277                                 ref_name = file_name;\r
1278                                 Location.Push (ref_name);\r
1279                                 return true;\r
1280                         }\r
1281                         \r
1282                         try {\r
1283                                 int pos;\r
1284 \r
1285                                 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){\r
1286                                         ref_line = System.Int32.Parse (arg.Substring (0, pos));\r
1287                                         pos++;\r
1288                                         \r
1289                                         char [] quotes = { '\"' };\r
1290                                         \r
1291                                         string name = arg.Substring (pos). Trim (quotes);\r
1292                                         ref_name = Location.LookupFile (name);\r
1293                                         Location.Push (ref_name);\r
1294                                 } else {\r
1295                                         ref_line = System.Int32.Parse (arg);\r
1296                                 }\r
1297                         } catch {\r
1298                                 return false;\r
1299                         }\r
1300                         \r
1301                         return true;\r
1302                 }\r
1303 \r
1304                 //\r
1305                 // Handles #define and #undef\r
1306                 //\r
1307                 void PreProcessDefinition (bool is_define, string arg)\r
1308                 {\r
1309                         if (arg == "" || arg == "true" || arg == "false"){\r
1310                                 Report.Error (1001, Location, "Missing identifer to pre-processor directive");\r
1311                                 return;\r
1312                         }\r
1313 \r
1314                         char[] whitespace = { ' ', '\t' };\r
1315                         if (arg.IndexOfAny (whitespace) != -1){\r
1316                                 Report.Error (1025, Location, "Single-line comment or end-of-line expected");\r
1317                                 return;\r
1318                         }\r
1319 \r
1320                         if (!is_identifier_start_character (arg [0]))\r
1321                                 Report.Error (1001, Location, "Identifier expected: " + arg);\r
1322                         \r
1323                         foreach (char c in arg.Substring (1)){\r
1324                                 if (!is_identifier_part_character (c)){\r
1325                                         Report.Error (1001, Location, "Identifier expected: " + arg);\r
1326                                         return;\r
1327                                 }\r
1328                         }\r
1329 \r
1330                         if (is_define){\r
1331                                 if (defines == null)\r
1332                                         defines = new Hashtable ();\r
1333                                 define (arg);\r
1334                         } else {\r
1335                                 if (defines == null)\r
1336                                         return;\r
1337                                 if (defines.Contains (arg))\r
1338                                         defines.Remove (arg);\r
1339                         }\r
1340                 }\r
1341 \r
1342                 bool eval_val (string s)\r
1343                 {\r
1344                         if (s == "true")\r
1345                                 return true;\r
1346                         if (s == "false")\r
1347                                 return false;\r
1348                         \r
1349                         if (defines == null)\r
1350                                 return false;\r
1351                         if (defines.Contains (s))\r
1352                                 return true;\r
1353 \r
1354                         return false;\r
1355                 }\r
1356 \r
1357                 bool pp_primary (ref string s)\r
1358                 {\r
1359                         s = s.Trim ();\r
1360                         int len = s.Length;\r
1361 \r
1362                         if (len > 0){\r
1363                                 char c = s [0];\r
1364                                 \r
1365                                 if (c == '('){\r
1366                                         s = s.Substring (1);\r
1367                                         bool val = pp_expr (ref s);\r
1368                                         if (s.Length > 0 && s [0] == ')'){\r
1369                                                 s = s.Substring (1);\r
1370                                                 return val;\r
1371                                         }\r
1372                                         Error_InvalidDirective ();\r
1373                                         return false;\r
1374                                 }\r
1375                                 \r
1376                                 if (is_identifier_start_character (c)){\r
1377                                         int j = 1;\r
1378 \r
1379                                         while (j < len){\r
1380                                                 c = s [j];\r
1381                                                 \r
1382                                                 if (is_identifier_part_character (c)){\r
1383                                                         j++;\r
1384                                                         continue;\r
1385                                                 }\r
1386                                                 bool v = eval_val (s.Substring (0, j));\r
1387                                                 s = s.Substring (j);\r
1388                                                 return v;\r
1389                                         }\r
1390                                         bool vv = eval_val (s);\r
1391                                         s = "";\r
1392                                         return vv;\r
1393                                 }\r
1394                         }\r
1395                         Error_InvalidDirective ();\r
1396                         return false;\r
1397                 }\r
1398                 \r
1399                 bool pp_unary (ref string s)\r
1400                 {\r
1401                         s = s.Trim ();\r
1402                         int len = s.Length;\r
1403 \r
1404                         if (len > 0){\r
1405                                 if (s [0] == '!'){\r
1406                                         if (len > 1 && s [1] == '='){\r
1407                                                 Error_InvalidDirective ();\r
1408                                                 return false;\r
1409                                         }\r
1410                                         s = s.Substring (1);\r
1411                                         return ! pp_primary (ref s);\r
1412                                 } else\r
1413                                         return pp_primary (ref s);\r
1414                         } else {\r
1415                                 Error_InvalidDirective ();\r
1416                                 return false;\r
1417                         }\r
1418                 }\r
1419                 \r
1420                 bool pp_eq (ref string s)\r
1421                 {\r
1422                         bool va = pp_unary (ref s);\r
1423 \r
1424                         s = s.Trim ();\r
1425                         int len = s.Length;\r
1426                         if (len > 0){\r
1427                                 if (s [0] == '='){\r
1428                                         if (len > 2 && s [1] == '='){\r
1429                                                 s = s.Substring (2);\r
1430                                                 return va == pp_unary (ref s);\r
1431                                         } else {\r
1432                                                 Error_InvalidDirective ();\r
1433                                                 return false;\r
1434                                         }\r
1435                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){\r
1436                                         s = s.Substring (2);\r
1437 \r
1438                                         return va != pp_unary (ref s);\r
1439 \r
1440                                 } \r
1441                         }\r
1442 \r
1443                         return va;\r
1444                                 \r
1445                 }\r
1446                 \r
1447                 bool pp_and (ref string s)\r
1448                 {\r
1449                         bool va = pp_eq (ref s);\r
1450 \r
1451                         s = s.Trim ();\r
1452                         int len = s.Length;\r
1453                         if (len > 0){\r
1454                                 if (s [0] == '&'){\r
1455                                         if (len > 2 && s [1] == '&'){\r
1456                                                 s = s.Substring (2);\r
1457                                                 return (va & pp_eq (ref s));\r
1458                                         } else {\r
1459                                                 Error_InvalidDirective ();\r
1460                                                 return false;\r
1461                                         }\r
1462                                 } \r
1463                         }\r
1464                         return va;\r
1465                 }\r
1466                 \r
1467                 //\r
1468                 // Evaluates an expression for `#if' or `#elif'\r
1469                 //\r
1470                 bool pp_expr (ref string s)\r
1471                 {\r
1472                         bool va = pp_and (ref s);\r
1473                         s = s.Trim ();\r
1474                         int len = s.Length;\r
1475                         if (len > 0){\r
1476                                 char c = s [0];\r
1477                                 \r
1478                                 if (c == '|'){\r
1479                                         if (len > 2 && s [1] == '|'){\r
1480                                                 s = s.Substring (2);\r
1481                                                 return va | pp_expr (ref s);\r
1482                                         } else {\r
1483                                                 Error_InvalidDirective ();\r
1484                                                 return false;\r
1485                                         }\r
1486                                 } \r
1487                         }\r
1488                         \r
1489                         return va;\r
1490                 }\r
1491 \r
1492                 bool eval (string s)\r
1493                 {\r
1494                         bool v = pp_expr (ref s);\r
1495                         s = s.Trim ();\r
1496                         if (s.Length != 0){\r
1497                                 Error_InvalidDirective ();\r
1498                                 return false;\r
1499                         }\r
1500 \r
1501                         return v;\r
1502                 }\r
1503                 \r
1504                 void Error_InvalidDirective ()\r
1505                 {\r
1506                         Report.Error (1517, Location, "Invalid pre-processor directive");\r
1507                 }\r
1508 \r
1509                 void Error_UnexpectedDirective (string extra)\r
1510                 {\r
1511                         Report.Error (\r
1512                                 1028, Location,\r
1513                                 "Unexpected processor directive (" + extra + ")");\r
1514                 }\r
1515 \r
1516                 void Error_TokensSeen ()\r
1517                 {\r
1518                         Report.Error (\r
1519                                 1032, Location,\r
1520                                 "Cannot define or undefine pre-processor symbols after a token in the file");\r
1521                 }\r
1522                 \r
1523                 //\r
1524                 // if true, then the code continues processing the code\r
1525                 // if false, the code stays in a loop until another directive is\r
1526                 // reached.\r
1527                 //\r
1528                 bool handle_preprocessing_directive (bool caller_is_taking)\r
1529                 {\r
1530                         string cmd, arg;\r
1531                         bool region_directive = false;\r
1532 \r
1533                         get_cmd_arg (out cmd, out arg);\r
1534 \r
1535                         // Eat any trailing whitespaces and single-line comments\r
1536                         if (arg.IndexOf ("//") != -1)\r
1537                                 arg = arg.Substring (0, arg.IndexOf ("//"));\r
1538                         arg = arg.TrimEnd (' ', '\t');\r
1539 \r
1540                         //\r
1541                         // The first group of pre-processing instructions is always processed\r
1542                         //\r
1543                         switch (cmd){\r
1544                         case "pragma":\r
1545                                 if (RootContext.V2)\r
1546                                         return true;\r
1547                                 break;\r
1548                                 \r
1549                         case "line":\r
1550                                 if (!PreProcessLine (arg))\r
1551                                         Report.Error (\r
1552                                                 1576, Location,\r
1553                                                 "Argument to #line directive is missing or invalid");\r
1554                                 return true;\r
1555 \r
1556                         case "region":\r
1557                                 region_directive = true;\r
1558                                 arg = "true";\r
1559                                 goto case "if";\r
1560 \r
1561                         case "endregion":\r
1562                                 region_directive = true;\r
1563                                 goto case "endif";\r
1564                                 \r
1565                         case "if":\r
1566                                 if (arg == ""){\r
1567                                         Error_InvalidDirective ();\r
1568                                         return true;\r
1569                                 }\r
1570                                 bool taking = false;\r
1571                                 if (ifstack == null)\r
1572                                         ifstack = new Stack ();\r
1573 \r
1574                                 if (ifstack.Count == 0){\r
1575                                         taking = true;\r
1576                                 } else {\r
1577                                         int state = (int) ifstack.Peek ();\r
1578                                         if ((state & TAKING) != 0)\r
1579                                                 taking = true;\r
1580                                 }\r
1581 \r
1582                                 if (eval (arg) && taking){\r
1583                                         int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;\r
1584                                         if (region_directive)\r
1585                                                 push |= REGION;\r
1586                                         ifstack.Push (push);\r
1587                                         return true;\r
1588                                 } else {\r
1589                                         int push = (taking ? PARENT_TAKING : 0);\r
1590                                         if (region_directive)\r
1591                                                 push |= REGION;\r
1592                                         ifstack.Push (push);\r
1593                                         return false;\r
1594                                 }\r
1595                                 \r
1596                         case "endif":\r
1597                                 if (ifstack == null || ifstack.Count == 0){\r
1598                                         Error_UnexpectedDirective ("no #if for this #endif");\r
1599                                         return true;\r
1600                                 } else {\r
1601                                         int pop = (int) ifstack.Pop ();\r
1602                                         \r
1603                                         if (region_directive && ((pop & REGION) == 0))\r
1604                                                 Report.Error (1027, Location, "#endif directive expected");\r
1605                                         else if (!region_directive && ((pop & REGION) != 0))\r
1606                                                 Report.Error (1038, Location, "#endregion directive expected");\r
1607                                         \r
1608                                         if (ifstack.Count == 0)\r
1609                                                 return true;\r
1610                                         else {\r
1611                                                 int state = (int) ifstack.Peek ();\r
1612 \r
1613                                                 if ((state & TAKING) != 0)\r
1614                                                         return true;\r
1615                                                 else\r
1616                                                         return false;\r
1617                                         }\r
1618                                 }\r
1619 \r
1620                         case "elif":\r
1621                                 if (ifstack == null || ifstack.Count == 0){\r
1622                                         Error_UnexpectedDirective ("no #if for this #elif");\r
1623                                         return true;\r
1624                                 } else {\r
1625                                         int state = (int) ifstack.Peek ();\r
1626 \r
1627                                         if ((state & REGION) != 0) {\r
1628                                                 Report.Error (1038, Location, "#endregion directive expected");\r
1629                                                 return true;\r
1630                                         }\r
1631 \r
1632                                         if ((state & ELSE_SEEN) != 0){\r
1633                                                 Error_UnexpectedDirective ("#elif not valid after #else");\r
1634                                                 return true;\r
1635                                         }\r
1636 \r
1637                                         if ((state & (TAKEN_BEFORE | TAKING)) != 0)\r
1638                                                 return false;\r
1639 \r
1640                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){\r
1641                                                 state = (int) ifstack.Pop ();\r
1642                                                 ifstack.Push (state | TAKING | TAKEN_BEFORE);\r
1643                                                 return true;\r
1644                                         } else \r
1645                                                 return false;\r
1646                                 }\r
1647 \r
1648                         case "else":\r
1649                                 if (ifstack == null || ifstack.Count == 0){\r
1650                                         Report.Error (\r
1651                                                 1028, Location,\r
1652                                                 "Unexpected processor directive (no #if for this #else)");\r
1653                                         return true;\r
1654                                 } else {\r
1655                                         int state = (int) ifstack.Peek ();\r
1656 \r
1657                                         if ((state & REGION) != 0) {\r
1658                                                 Report.Error (1038, Location, "#endregion directive expected");\r
1659                                                 return true;\r
1660                                         }\r
1661 \r
1662                                         if ((state & ELSE_SEEN) != 0){\r
1663                                                 Error_UnexpectedDirective ("#else within #else");\r
1664                                                 return true;\r
1665                                         }\r
1666 \r
1667                                         ifstack.Pop ();\r
1668 \r
1669                                         bool ret;\r
1670                                         if ((state & TAKEN_BEFORE) == 0){\r
1671                                                 ret = ((state & PARENT_TAKING) != 0);\r
1672                                         } else\r
1673                                                 ret = false;\r
1674                                         \r
1675                                         if (ret)\r
1676                                                 state |= TAKING;\r
1677                                         else\r
1678                                                 state &= ~TAKING;\r
1679                                         \r
1680                                         ifstack.Push (state | ELSE_SEEN);\r
1681                                         \r
1682                                         return ret;\r
1683                                 }\r
1684                         }\r
1685 \r
1686                         //\r
1687                         // These are only processed if we are in a `taking' block\r
1688                         //\r
1689                         if (!caller_is_taking)\r
1690                                 return false;\r
1691                                         \r
1692                         switch (cmd){\r
1693                         case "define":\r
1694                                 if (any_token_seen){\r
1695                                         Error_TokensSeen ();\r
1696                                         return true;\r
1697                                 }\r
1698                                 PreProcessDefinition (true, arg);\r
1699                                 return true;\r
1700 \r
1701                         case "undef":\r
1702                                 if (any_token_seen){\r
1703                                         Error_TokensSeen ();\r
1704                                         return true;\r
1705                                 }\r
1706                                 PreProcessDefinition (false, arg);\r
1707                                 return true;\r
1708 \r
1709                         case "error":\r
1710                                 Report.Error (1029, Location, "#error: '" + arg + "'");\r
1711                                 return true;\r
1712 \r
1713                         case "warning":\r
1714                                 Report.Warning (1030, Location, "#warning: '" + arg + "'");\r
1715                                 return true;\r
1716                         }\r
1717 \r
1718                         Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");\r
1719                         return true;\r
1720 \r
1721                 }\r
1722 \r
1723                 private int consume_string (bool quoted) \r
1724                 {\r
1725                         int c;\r
1726                         string_builder.Length = 0;\r
1727                                                                 \r
1728                         while ((c = getChar ()) != -1){\r
1729                                 if (c == '"'){\r
1730                                         if (quoted && peekChar () == '"'){\r
1731                                                 string_builder.Append ((char) c);\r
1732                                                 getChar ();\r
1733                                                 continue;\r
1734                                         } else {\r
1735                                                 val = string_builder.ToString ();\r
1736                                                 return Token.LITERAL_STRING;\r
1737                                         }\r
1738                                 }\r
1739 \r
1740                                 if (c == '\n'){\r
1741                                         if (!quoted)\r
1742                                                 Report.Error (1010, Location, "Newline in constant");\r
1743                                         line++;\r
1744                                         ref_line++;\r
1745                                         col = 0;\r
1746                                 } else\r
1747                                         col++;\r
1748 \r
1749                                 if (!quoted){\r
1750                                         c = escape (c);\r
1751                                         if (c == -1)\r
1752                                                 return Token.ERROR;\r
1753                                 }\r
1754                                 string_builder.Append ((char) c);\r
1755                         }\r
1756 \r
1757                         Report.Error (1039, Location, "Unterminated string literal");\r
1758                         return Token.EOF;\r
1759                 }\r
1760 \r
1761                 private int consume_identifier (int s, bool quoted) \r
1762                 {\r
1763                         int pos = 1;\r
1764                         int c;\r
1765                         \r
1766                         id_builder [0] = (char) s;\r
1767                                         \r
1768                         while ((c = reader.Read ()) != -1) {\r
1769                                 if (is_identifier_part_character ((char) c)){\r
1770                                         if (pos == max_id_size){\r
1771                                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");\r
1772                                                 return Token.ERROR;\r
1773                                         }\r
1774                                         \r
1775                                         id_builder [pos++] = (char) c;\r
1776                                         putback_char = -1;\r
1777                                         col++;\r
1778                                 } else {\r
1779                                         putback_char = c;\r
1780                                         break;\r
1781                                 }\r
1782                         }\r
1783 \r
1784                         //\r
1785                         // Optimization: avoids doing the keyword lookup\r
1786                         // on uppercase letters and _\r
1787                         //\r
1788                         if (s >= 'a'){\r
1789                                 int keyword = GetKeyword (id_builder, pos);\r
1790                                 if (keyword != -1 && !quoted)\r
1791                                 return keyword;\r
1792                         }\r
1793 \r
1794                         //\r
1795                         // Keep identifiers in an array of hashtables to avoid needless\r
1796                         // allocations\r
1797                         //\r
1798 \r
1799                         if (identifiers [pos] != null) {\r
1800                                 val = identifiers [pos][id_builder];\r
1801                                 if (val != null) {\r
1802                                         return Token.IDENTIFIER;\r
1803                                 }\r
1804                         }\r
1805                         else\r
1806                                 identifiers [pos] = new CharArrayHashtable (pos);\r
1807 \r
1808                         val = new String (id_builder, 0, pos);\r
1809 \r
1810                         char [] chars = new char [pos];\r
1811                         Array.Copy (id_builder, chars, pos);\r
1812 \r
1813                         identifiers [pos] [chars] = val;\r
1814 \r
1815                         return Token.IDENTIFIER;\r
1816                 }\r
1817                 \r
1818                 public int xtoken ()\r
1819                 {\r
1820                         int t;\r
1821                         bool doread = false;\r
1822                         int c;\r
1823 \r
1824                         val = null;\r
1825                         // optimization: eliminate col and implement #directive semantic correctly.\r
1826                         for (;(c = getChar ()) != -1; col++) {\r
1827                                 if (c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r' || c == 0xa0){\r
1828                                         \r
1829                                         if (c == '\t')\r
1830                                                 col = (((col + 8) / 8) * 8) - 1;\r
1831                                         continue;\r
1832                                 }\r
1833 \r
1834                                 // Handle double-slash comments.\r
1835                                 if (c == '/'){\r
1836                                         int d = peekChar ();\r
1837                                 \r
1838                                         if (d == '/'){\r
1839                                                 getChar ();\r
1840                                                 while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')\r
1841                                                         col++;\r
1842                                                 if (d == '\n'){\r
1843                                                         line++;\r
1844                                                         ref_line++;\r
1845                                                         col = 0;\r
1846                                                 }\r
1847                                                 any_token_seen |= tokens_seen;\r
1848                                                 tokens_seen = false;\r
1849                                                 continue;\r
1850                                         } else if (d == '*'){\r
1851                                                 getChar ();\r
1852 \r
1853                                                 while ((d = getChar ()) != -1){\r
1854                                                         if (d == '*' && peekChar () == '/'){\r
1855                                                                 getChar ();\r
1856                                                                 col++;\r
1857                                                                 break;\r
1858                                                         }\r
1859                                                         if (d == '\n'){\r
1860                                                                 line++;\r
1861                                                                 ref_line++;\r
1862                                                                 col = 0;\r
1863                                                                 any_token_seen |= tokens_seen;\r
1864                                                                 tokens_seen = false;\r
1865                                                         }\r
1866                                                 }\r
1867                                                 continue;\r
1868                                         }\r
1869                                         goto is_punct_label;\r
1870                                 }\r
1871 \r
1872                                 \r
1873                                 if (is_identifier_start_character ((char)c)){\r
1874                                         tokens_seen = true;\r
1875                                         return consume_identifier (c, false);\r
1876                                 }\r
1877 \r
1878                         is_punct_label:\r
1879                                 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){\r
1880                                         tokens_seen = true;\r
1881                                         if (doread){\r
1882                                                 getChar ();\r
1883                                                 col++;\r
1884                                         }\r
1885                                         return t;\r
1886                                 }\r
1887 \r
1888                                 // white space\r
1889                                 if (c == '\n'){\r
1890                                         line++;\r
1891                                         ref_line++;\r
1892                                         col = 0;\r
1893                                         any_token_seen |= tokens_seen;\r
1894                                         tokens_seen = false;\r
1895                                         continue;\r
1896                                 }\r
1897 \r
1898                                 if (c >= '0' && c <= '9'){\r
1899                                         tokens_seen = true;\r
1900                                         return is_number (c);\r
1901                                 }\r
1902 \r
1903                                 if (c == '.'){\r
1904                                         tokens_seen = true;\r
1905                                         int peek = peekChar ();\r
1906                                         if (peek >= '0' && peek <= '9')\r
1907                                                 return is_number (c);\r
1908                                         return Token.DOT;\r
1909                                 }\r
1910                                 \r
1911                                 /* For now, ignore pre-processor commands */\r
1912                                 // FIXME: In C# the '#' is not limited to appear\r
1913                                 // on the first column.\r
1914                                 if (c == '#' && !tokens_seen){\r
1915                                         bool cont = true;\r
1916                                         \r
1917                                 start_again:\r
1918                                         \r
1919                                         cont = handle_preprocessing_directive (cont);\r
1920 \r
1921                                         if (cont){\r
1922                                                 col = 0;\r
1923                                                 continue;\r
1924                                         }\r
1925                                         col = 1;\r
1926 \r
1927                                         bool skipping = false;\r
1928                                         for (;(c = getChar ()) != -1; col++){\r
1929                                                 if (c == '\n'){\r
1930                                                         col = 0;\r
1931                                                         line++;\r
1932                                                         ref_line++;\r
1933                                                         skipping = false;\r
1934                                                 } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)\r
1935                                                         continue;\r
1936                                                 else if (c != '#')\r
1937                                                         skipping = true;\r
1938                                                 if (c == '#' && !skipping)\r
1939                                                         goto start_again;\r
1940                                         }\r
1941                                         any_token_seen |= tokens_seen;\r
1942                                         tokens_seen = false;\r
1943                                         if (c == -1)\r
1944                                                 Report.Error (1027, Location, "#endif/#endregion expected");\r
1945                                         continue;\r
1946                                 }\r
1947                                 \r
1948                                 if (c == '"') \r
1949                                         return consume_string (false);\r
1950 \r
1951                                 if (c == '\''){\r
1952                                         c = getChar ();\r
1953                                         tokens_seen = true;\r
1954                                         if (c == '\''){\r
1955                                                 error_details = "Empty character literal";\r
1956                                                 Report.Error (1011, Location, error_details);\r
1957                                                 return Token.ERROR;\r
1958                                         }\r
1959                                         c = escape (c);\r
1960                                         if (c == -1)\r
1961                                                 return Token.ERROR;\r
1962                                         val = new System.Char ();\r
1963                                         val = (char) c;\r
1964                                         c = getChar ();\r
1965 \r
1966                                         if (c != '\''){\r
1967                                                 error_details = "Too many characters in character literal";\r
1968                                                 Report.Error (1012, Location, error_details);\r
1969 \r
1970                                                 // Try to recover, read until newline or next "'"\r
1971                                                 while ((c = getChar ()) != -1){\r
1972                                                         if (c == '\n' || c == '\''){\r
1973                                                                 line++;\r
1974                                                                 ref_line++;\r
1975                                                                 col = 0;\r
1976                                                                 break;\r
1977                                                         } else\r
1978                                                                 col++;\r
1979                                                         \r
1980                                                 }\r
1981                                                 return Token.ERROR;\r
1982                                         }\r
1983                                         return Token.LITERAL_CHARACTER;\r
1984                                 }\r
1985                                 \r
1986                                 if (c == '@') {\r
1987                                         c = getChar ();\r
1988                                         if (c == '"') {\r
1989                                                 tokens_seen = true;\r
1990                                                 return consume_string (true);\r
1991                                         } else if (is_identifier_start_character ((char) c)){\r
1992                                                 return consume_identifier (c, true);\r
1993                                         } else {\r
1994                                                 Report.Error (1033, Location, "'@' must be followed by string constant or identifier");\r
1995                                         }\r
1996                                 }\r
1997 \r
1998                                 if (c == '#') {\r
1999                                         error_details = "Preprocessor directives must appear as the first non-whitespace " +\r
2000                                                 "character on a line.";\r
2001 \r
2002                                         Report.Error (1040, Location, error_details);\r
2003 \r
2004                                         return Token.ERROR;\r
2005                                 }\r
2006 \r
2007                                 error_details = ((char)c).ToString ();\r
2008                                 \r
2009                                 return Token.ERROR;\r
2010                         }\r
2011 \r
2012                         return Token.EOF;\r
2013                 }\r
2014 \r
2015                 public void cleanup ()\r
2016                 {\r
2017                         if (ifstack != null && ifstack.Count >= 1) {\r
2018                                 int state = (int) ifstack.Pop ();\r
2019                                 if ((state & REGION) != 0)\r
2020                                         Report.Error (1038, "#endregion directive expected");\r
2021                                 else \r
2022                                         Report.Error (1027, "#endif directive expected");\r
2023                         }\r
2024                                 \r
2025                 }\r
2026         }\r
2027 }\r
2028 \r