Mercurial > projects > dil
changeset 418:7354f15cd5e9
Applied some fixes to the Lexer.
do-while loop should be while-loop in destructor.\n
Checked for LS and PS using decodeUTF8 in scanShebang(), but 'p' mustn't
be advanced beyond a newline.
Fix: pushing token to token2LocTable only after all whitespace and newlines
have been skipped.
author | Aziz K?ksal <aziz.koeksal@gmail.com> |
---|---|
date | Thu, 27 Sep 2007 17:06:28 +0200 |
parents | 8af5c7e2f722 |
children | 89e40d43065d |
files | trunk/src/dil/Lexer.d |
diffstat | 1 files changed, 18 insertions(+), 11 deletions(-) [+] |
line wrap: on
line diff
--- a/trunk/src/dil/Lexer.d Thu Sep 27 13:59:36 2007 +0200 +++ b/trunk/src/dil/Lexer.d Thu Sep 27 17:06:28 2007 +0200 @@ -99,12 +99,12 @@ ~this() { auto token = head.next; - do + while (token !is null) { assert(token.type == TOK.EOF ? token == tail && token.next is null : 1); delete token.prev; token = token.next; - } while (token !is null) + } delete tail; } @@ -124,13 +124,12 @@ { case '\r', '\n', 0, _Z_: break; + case LS[0]: + if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) + break; default: if (*p & 128) - { - auto c = decodeUTF8(); - if (c == LSd || c == PSd) - break; - } + decodeUTF8(); continue; } break; // Exit loop. @@ -197,6 +196,7 @@ { // Scan whitespace. auto pws = p; + auto old_loc = this.loc; while (1) { switch (*p) @@ -207,8 +207,6 @@ case '\n': ++p; ++loc; - version(token2LocTable) - token2LocTable[&t] = Location(loc, null); continue; case LS[0]: if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) @@ -227,7 +225,12 @@ } if (p != pws) + { t.ws = pws; + if (old_loc != this.loc) + version(token2LocTable) + token2LocTable[&t] = Location(loc, null); + } // Scan token. uint c = *p; @@ -673,6 +676,7 @@ { // Scan whitespace. auto pws = p; + auto old_loc = this.loc; while (1) { switch (*p) @@ -683,8 +687,6 @@ case '\n': ++p; ++loc; - version(token2LocTable) - token2LocTable[&t] = Location(loc, null); continue; case LS[0]: if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) @@ -703,7 +705,12 @@ } if (p != pws) + { t.ws = pws; + if (old_loc != this.loc) + version(token2LocTable) + token2LocTable[&t] = Location(loc, null); + } // Scan token. t.start = p;