Mercurial > projects > dil

comparison trunk/src/dil/Token.d @ 487:bccca748d745

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added 'tokenize' command.
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Sat, 01 Dec 2007 20:20:44 +0100
parents ea8c7459f1c4
children 47be6bfe39cd
comparison
equal deleted inserted replaced
486:bd176bc73e43 487:bccca748d745
157 real real_; 157 real real_;
158 } 158 }
159 159
160 alias srcText identifier; 160 alias srcText identifier;
161 161
162 /// Returns the text of the token.
162 string srcText() 163 string srcText()
163 { 164 {
164 assert(start && end); 165 assert(start && end);
165 return start[0 .. end - start]; 166 return start[0 .. end - start];
167 }
168
169 /// Returns the preceding whitespace of the token.
170 string wsChars()
171 {
172 assert(ws && start);
173 return ws[0 .. start - ws];
166 } 174 }
167 175
168 /// Find next non-whitespace token. Returns 'this' token if the next token is TOK.EOF or null. 176 /// Find next non-whitespace token. Returns 'this' token if the next token is TOK.EOF or null.
169 Token* nextNWS() 177 Token* nextNWS()
170 out(token) 178 out(token)
294 delete tok_del; 302 delete tok_del;
295 tok_del = tok_it; 303 tok_del = tok_it;
296 } 304 }
297 } 305 }
298 } 306 }
307 }
308
309 /++
310 Not used at the moment. Could be useful if more
311 info is needed about the location of nodes/tokens.
312 +/
313 struct NewlineInfo
314 {
315 char[] oriPath; /// Original path to the source text.
316 char[] setPath; /// Path set by #line.
317 uint oriLineNum; /// Actual line number in the source text.
318 uint setLineNum; /// Delta line number set by #line.
299 } 319 }
300 320
301 /// A table mapping each TOK to a string. 321 /// A table mapping each TOK to a string.
302 private const string[] tokToString = [ 322 private const string[] tokToString = [
303 "Invalid", 323 "Invalid",