Mercurial > projects > dil

comparison trunk/src/dil/Token.d @ 496:5a607597dc22

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Improved error recovery in the Parser. The Parser skips to the next valid token if an illegal Declaration, Statement or AsmInstruction was found. Refactored a few things in Lexer.d and LexerFuncs.d.
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Sun, 09 Dec 2007 13:04:15 +0100
parents 47be6bfe39cd
children 0ffcc4ff82f3
comparison
equal deleted inserted replaced
495:b60450804b6e 496:5a607597dc22
110 MAX 110 MAX
111 } 111 }
112 112
113 alias TOK.Abstract KeywordsBegin; 113 alias TOK.Abstract KeywordsBegin;
114 alias TOK.With KeywordsEnd; 114 alias TOK.With KeywordsEnd;
115 alias TOK.FILE SpecialTokensBegin;
116 alias TOK.Version SpecialTokensEnd;
115 117
116 struct Token 118 struct Token
117 { 119 {
118 TOK type; /// The type of the token. 120 TOK type; /// The type of the token.
119 /// Pointers to the next and previous tokens (doubly-linked list.) 121 /// Pointers to the next and previous tokens (doubly-linked list.)
210 { 212 {
211 return tokToString[tok]; 213 return tokToString[tok];
212 } 214 }
213 215
214 /++ 216 /++
215 Returns true if this is a token which can have newlines in it. 217 Returns true if this is a token that can have newlines in it.
216 These can be any string literal except for escape literals 218 These can be block and nested comments and any string literal
217 and block and nested comments. 219 except for escape string literals.
218 +/ 220 +/
219 bool isMultiline() 221 bool isMultiline()
220 { 222 {
221 return type == TOK.String && start[0] != '\\' || 223 return type == TOK.String && start[0] != '\\' ||
222 type == TOK.Comment && start[1] != '/'; 224 type == TOK.Comment && start[1] != '/';
235 } 237 }
236 238
237 /// Returns true if this is a special token. 239 /// Returns true if this is a special token.
238 bool isSpecialToken() 240 bool isSpecialToken()
239 { 241 {
240 return *start == '_' && type != TOK.Identifier; 242 return SpecialTokensBegin <= type && type <= SpecialTokensEnd;
241 } 243 }
242 244
243 version(D2) 245 version(D2)
244 { 246 {
245 /// Returns true if this is a token string literal. 247 /// Returns true if this is a token string literal.
246 bool isTokenStringLiteral() 248 bool isTokenStringLiteral()
247 { 249 {
248 return type == TOK.String && tok_str !is null; 250 return type == TOK.String && tok_str !is null;
249 } 251 }
250 } 252 }
253
254 /// Returns true if this token starts a DeclarationDefinition.
255 bool isDeclDefStart()
256 {
257 return isDeclDefStartToken(type);
258 }
259
260 /// Returns true if this token starts a Statement.
261 bool isStatementStart()
262 {
263 return isStatementStartToken(type);
264 }
265
266 /// Returns true if this token starts an AsmInstruction.
267 bool isAsmInstructionStart()
268 {
269 return isAsmInstructionStartToken(type);
270 }
251 271
252 int opEquals(TOK type2) 272 int opEquals(TOK type2)
253 { 273 {
254 return type == type2; 274 return type == type2;
255 } 275 }
288 } 308 }
289 } 309 }
290 return new Location(filePath, lineNum, lineBegin, this.start); 310 return new Location(filePath, lineNum, lineBegin, this.start);
291 } 311 }
292 312
313 uint lineCount()
314 {
315 uint count = 1;
316 if (this.isMultiline)
317 {
318 auto p = this.start, end = this.end;
319 while (p != end)
320 {
321 if (scanNewline(p) == '\n')
322 ++count;
323 else
324 ++p;
325 }
326 }
327 return count;
328 }
329
330 /// Return the source text enclosed by the left and right token.
331 static char[] textSpan(Token* left, Token* right)
332 {
333 assert(left.end <= right.start);
334 return left.start[0 .. right.end - left.start];
335 }
336
293 new(size_t size) 337 new(size_t size)
294 { 338 {
295 void* p = malloc(size); 339 void* p = malloc(size);
296 if (p is null) 340 if (p is null)
297 throw new OutOfMemoryException(__FILE__, __LINE__); 341 throw new OutOfMemoryException(__FILE__, __LINE__);
442 486
443 "HEAD", 487 "HEAD",
444 "EOF" 488 "EOF"
445 ]; 489 ];
446 static assert(tokToString.length == TOK.MAX); 490 static assert(tokToString.length == TOK.MAX);
491
492 /// Returns true if this token starts a DeclarationDefinition.
493 bool isDeclDefStartToken(TOK tok)
494 {
495 switch (tok)
496 {
497 alias TOK T;
498 case T.Align, T.Pragma, T.Export, T.Private, T.Package, T.Protected,
499 T.Public, T.Extern, T.Deprecated, T.Override, T.Abstract,
500 T.Synchronized, T.Static, T.Final, T.Const, T.Invariant/*D 2.0*/,
501 T.Auto, T.Scope, T.Alias, T.Typedef, T.Import, T.Enum, T.Class,
502 T.Interface, T.Struct, T.Union, T.This, T.Tilde, T.Unittest, T.Debug,
503 T.Version, T.Template, T.New, T.Delete, T.Mixin, T.Semicolon,
504 T.Identifier, T.Dot, T.Typeof,
505 T.Char, T.Wchar, T.Dchar, T.Bool,
506 T.Byte, T.Ubyte, T.Short, T.Ushort,
507 T.Int, T.Uint, T.Long, T.Ulong,
508 T.Float, T.Double, T.Real,
509 T.Ifloat, T.Idouble, T.Ireal,
510 T.Cfloat, T.Cdouble, T.Creal, T.Void:
511 return true;
512 default:
513 }
514 return false;
515 }
516
517 /// Returns true if this token starts a Statement.
518 bool isStatementStartToken(TOK tok)
519 {
520 switch (tok)
521 {
522 alias TOK T;
523 case T.Align, T.Extern, T.Final, T.Const, T.Auto, T.Identifier, T.Dot,
524 T.Typeof, T.If, T.While, T.Do, T.For, T.Foreach, T.Foreach_reverse,
525 T.Switch, T.Case, T.Default, T.Continue, T.Break, T.Return, T.Goto,
526 T.With, T.Synchronized, T.Try, T.Throw, T.Scope, T.Volatile, T.Asm,
527 T.Pragma, T.Mixin, T.Static, T.Debug, T.Version, T.Alias, T.Semicolon,
528 T.Enum, T.Class, T.Interface, T.Struct, T.Union, T.LBrace, T.Typedef,
529 T.This, T.Super, T.Null, T.True, T.False, T.Int32, T.Int64, T.Uint32,
530 T.Uint64, T.Float32, T.Float64, T.Float80, T.Imaginary32,
531 T.Imaginary64, T.Imaginary80, T.CharLiteral, T.WCharLiteral,
532 T.DCharLiteral, T.String, T.LBracket, T.Function, T.Delegate,
533 T.Assert, T.Import, T.Typeid, T.Is, T.LParen, T.Traits/*D2.0*/,
534 T.AndBinary, T.PlusPlus, T.MinusMinus, T.Mul,T.Minus, T.Plus, T.Not,
535 T.Tilde, T.New, T.Delete, T.Cast:
536 case T.Char, T.Wchar, T.Dchar, T.Bool,
537 T.Byte, T.Ubyte, T.Short, T.Ushort,
538 T.Int, T.Uint, T.Long, T.Ulong,
539 T.Float, T.Double, T.Real,
540 T.Ifloat, T.Idouble, T.Ireal,
541 T.Cfloat, T.Cdouble, T.Creal, T.Void:
542 return true;
543 default:
544 if (SpecialTokensBegin <= tok && tok <= SpecialTokensEnd)
545 return true;
546 }
547 return false;
548 }
549
550 /// Returns true if this token starts an AsmInstruction.
551 bool isAsmInstructionStartToken(TOK tok)
552 {
553 switch(tok)
554 {
555 alias TOK T;
556 case T.In, T.Int, T.Out, T.Identifier, T.Align, T.Semicolon:
557 return true;
558 default:
559 }
560 return false;
561 }