2 // Author: Sergey Chaban (serge@wildwestsoftware.com)
\r
7 using System.Collections;
\r
8 using System.Globalization;
\r
10 namespace Mono.ILASM {
\r
12 public delegate void NewTokenEvent (object sender, NewTokenEventArgs args);
\r
14 public class NewTokenEventArgs : EventArgs {
\r
16 public readonly ILToken Token;
\r
18 public NewTokenEventArgs (ILToken token)
\r
26 public class ILTokenizer : ITokenStream {
\r
28 private static readonly string idchars = "_$@?.`";
\r
30 private static Hashtable keywords;
\r
31 private static Hashtable directives;
\r
33 private ILToken lastToken;
\r
34 private ILReader reader;
\r
35 private StringHelper strBuilder;
\r
36 private NumberHelper numBuilder;
\r
37 private bool in_byte_array;
\r
39 public event NewTokenEvent NewTokenEvent;
\r
41 static ILTokenizer()
\r
43 keywords = ILTables.Keywords;
\r
44 directives = ILTables.Directives;
\r
49 /// <param name="reader"></param>
\r
50 public ILTokenizer (StreamReader reader)
\r
52 this.reader = new ILReader (reader);
\r
53 strBuilder = new StringHelper (this);
\r
54 numBuilder = new NumberHelper (this);
\r
55 lastToken = ILToken.Invalid.Clone () as ILToken;
\r
58 public ILReader Reader {
\r
64 public Location Location {
\r
66 return reader.Location;
\r
70 public bool InByteArray {
\r
71 get { return in_byte_array; }
\r
72 set { in_byte_array = value; }
\r
75 public ILToken GetNextToken ()
\r
77 if (lastToken == ILToken.EOF) return ILToken.EOF;
\r
81 ILToken res = ILToken.EOF.Clone () as ILToken;
\r
84 while ((ch = reader.Read ()) != -1) {
\r
88 next = reader.Peek ();
\r
90 // double-slash comment, skip to the end of the line.
\r
91 for (reader.Read ();
\r
92 next != -1 && next != '\n';
\r
93 next = reader.Read ());
\r
95 } else if (next == '*') {
\r
97 for (next = reader.Read (); next != -1; next = reader.Read ()) {
\r
98 if (next == '*' && reader.Peek () == '/') {
\r
108 // HEXBYTES are flagged by the parser otherwise it is
\r
109 // impossible to figure them out
\r
110 if (in_byte_array) {
\r
111 string hx = String.Empty;
\r
113 if (Char.IsWhiteSpace ((char) ch))
\r
117 res = ILToken.CloseParens;
\r
122 throw new ILTokenizingException (reader.Location, ((char) ch).ToString ());
\r
124 if (is_hex (reader.Peek ()))
\r
125 hx += (char) reader.Read ();
\r
126 else if (!Char.IsWhiteSpace ((char) reader.Peek ()) && reader.Peek () != ')')
\r
127 throw new ILTokenizingException (reader.Location,
\r
128 ((char) reader.Peek ()).ToString ());
\r
129 res.token = Token.HEXBYTE;
\r
130 res.val = Byte.Parse (hx, NumberStyles.HexNumber);
\r
132 while (Char.IsWhiteSpace ((char) reader.Peek ()))
\r
138 if (ch == '.' && reader.Peek () == '.') {
\r
139 reader.MarkLocation ();
\r
140 int ch2 = reader.Read ();
\r
141 if (reader.Peek () == '.') {
\r
142 res = ILToken.Ellipsis;
\r
146 reader.Unread (ch2);
\r
147 reader.RestoreLocation ();
\r
150 if (ch == '.' || ch == '#') {
\r
151 next = reader.Peek ();
\r
152 if (ch == '.' && Char.IsDigit((char) next)) {
\r
153 numBuilder.Start (ch);
\r
154 reader.Unread (ch);
\r
155 numBuilder.Build ();
\r
156 if (numBuilder.ResultToken != ILToken.Invalid) {
\r
157 res.CopyFrom (numBuilder.ResultToken);
\r
161 if (strBuilder.Start (next) && strBuilder.TokenId == Token.ID) {
\r
162 reader.MarkLocation ();
\r
163 string dirBody = strBuilder.Build ();
\r
164 string dir = new string ((char) ch, 1) + dirBody;
\r
165 if (IsDirective (dir)) {
\r
166 res = ILTables.Directives [dir] as ILToken;
\r
168 reader.Unread (dirBody.ToCharArray ());
\r
169 reader.RestoreLocation ();
\r
179 // Numbers && Hexbytes
\r
180 if (numBuilder.Start (ch)) {
\r
181 if ((ch == '-') && !(Char.IsDigit ((char) reader.Peek ()))) {
\r
182 res = ILToken.Dash;
\r
185 reader.Unread (ch);
\r
186 numBuilder.Build ();
\r
187 if (numBuilder.ResultToken != ILToken.Invalid) {
\r
188 res.CopyFrom (numBuilder.ResultToken);
\r
195 ILToken punct = ILToken.GetPunctuation (ch);
\r
196 if (punct != null) {
\r
197 if (punct == ILToken.Colon && reader.Peek () == ':') {
\r
199 res = ILToken.DoubleColon;
\r
206 // ID | QSTRING | SQSTRING | INSTR_* | KEYWORD
\r
207 if (strBuilder.Start (ch)) {
\r
208 reader.Unread (ch);
\r
209 string val = strBuilder.Build ();
\r
210 if (strBuilder.TokenId == Token.ID) {
\r
212 next = reader.Peek ();
\r
214 reader.MarkLocation ();
\r
216 next = reader.Peek ();
\r
217 if (IsIdChar ((char) next)) {
\r
218 string opTail = BuildId ();
\r
219 string full_str = String.Format ("{0}.{1}", val, opTail);
\r
220 opcode = InstrTable.GetToken (full_str);
\r
222 if (opcode == null) {
\r
223 if (strBuilder.TokenId != Token.ID) {
\r
224 reader.Unread (opTail.ToCharArray ());
\r
225 reader.Unread ('.');
\r
226 reader.RestoreLocation ();
\r
229 res.token = Token.COMP_NAME;
\r
230 res.val = full_str;
\r
238 } else if (Char.IsWhiteSpace ((char) next)) {
\r
239 // Handle 'tail.' and 'unaligned.'
\r
240 opcode = InstrTable.GetToken (val + ".");
\r
241 if (opcode != null) {
\r
245 // Let the parser handle the dot
\r
246 reader.Unread ('.');
\r
249 opcode = InstrTable.GetToken (val);
\r
250 if (opcode != null) {
\r
254 if (IsKeyword (val)) {
\r
255 res = ILTables.Keywords [val] as ILToken;
\r
260 res.token = strBuilder.TokenId;
\r
267 lastToken.CopyFrom (res);
\r
274 public ILToken NextToken {
\r
276 return GetNextToken ();
\r
283 public ILToken LastToken {
\r
289 bool is_hex (int e)
\r
291 return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
\r
294 private static bool IsIdStartChar (char ch)
\r
296 return (Char.IsLetter (ch) || (idchars.IndexOf (ch) != -1));
\r
300 private static bool IsIdChar (char ch)
\r
302 return (Char.IsLetterOrDigit (ch) || (idchars.IndexOf (ch) != -1));
\r
307 /// <param name="name"></param>
\r
308 /// <returns></returns>
\r
309 public static bool IsOpcode (string name)
\r
311 return InstrTable.IsInstr (name);
\r
317 /// <param name="name"></param>
\r
318 /// <returns></returns>
\r
319 public static bool IsDirective (string name)
\r
321 char ch = name [0];
\r
322 bool res = (ch == '.' || ch == '#');
\r
325 res = directives.Contains (name);
\r
331 private string BuildId ()
\r
333 StringBuilder idsb = new StringBuilder ();
\r
337 while ((ch = reader.Read ()) != -1) {
\r
338 if (IsIdChar ((char) ch) || ch == '.') {
\r
339 idsb.Append ((char) ch);
\r
341 reader.Unread (ch);
\r
342 // Never end an id on a DOT
\r
344 reader.Unread (last);
\r
352 return idsb.ToString ();
\r
357 /// <param name="name"></param>
\r
358 /// <returns></returns>
\r
359 public static bool IsKeyword (string name)
\r
361 return keywords.Contains (name);
\r
364 private void OnNewToken (ILToken token)
\r
366 if (NewTokenEvent != null)
\r
367 NewTokenEvent (this, new NewTokenEventArgs (token));
\r