Mercurial > projects > dil
diff trunk/src/dil/Lexer.d @ 497:0ffcc4ff82f3
Refactored a few things in the Lexer.
Fix: SpecialTokensEnd should be TOK.VERSION not TOK.Version.
Fixed assert statement in textSpan().
| author | Aziz K?ksal <aziz.koeksal@gmail.com> |
|---|---|
| date | Sun, 09 Dec 2007 14:58:38 +0100 |
| parents | 5a607597dc22 |
| children | 49c201b5c465 |
line wrap: on
line diff
--- a/trunk/src/dil/Lexer.d Sun Dec 09 13:04:15 2007 +0100 +++ b/trunk/src/dil/Lexer.d Sun Dec 09 14:58:38 2007 +0100 @@ -297,17 +297,17 @@ } assert(id); t.type = id.type; - if (t.type == TOK.Identifier) + if (t.type == TOK.Identifier || t.isKeyword) return; - if (t.type == TOK.EOF) + else if (t.isSpecialToken) + finalizeSpecialToken(t); + else if (t.type == TOK.EOF) { - t.type = TOK.EOF; - t.end = p; tail = &t; assert(t.srcText == "__EOF__"); } - else if (t.isSpecialToken) - finalizeSpecialToken(t); + else + assert(0, "unexpected token: " ~ t.srcText); return; } @@ -610,9 +610,9 @@ } // Check for EOF - if (c == 0 || c == _Z_) + if (isEOF(c)) { - assert(*p == 0 || *p == _Z_); + assert(isEOF(*p), ""~*p); t.type = TOK.EOF; t.end = p; tail = &t; @@ -1049,17 +1049,17 @@ } assert(id); t.type = id.type; - if (t.type == TOK.Identifier) + if (t.type == TOK.Identifier || t.isKeyword) return; - if (t.type == TOK.EOF) + else if (t.isSpecialToken) + finalizeSpecialToken(t); + else if (t.type == TOK.EOF) { - t.type = TOK.EOF; - t.end = p; tail = &t; assert(t.srcText == "__EOF__"); } - else if (t.isSpecialToken) - finalizeSpecialToken(t); + else + assert(0, "unexpected token: " ~ t.srcText); return; } @@ -1067,9 +1067,9 @@ return scanNumber(t); // Check for EOF - if (c == 0 || c == _Z_) + if (isEOF(c)) { - assert(*p == 0 || *p == _Z_, *p~""); + assert(isEOF(*p), *p~""); t.type = TOK.EOF; t.end = p; tail = &t;