Mercurial > projects > dil
changeset 314:ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
- Added method isWhitespace() to Token. Adapted parser accordingly.
author | aziz |
---|---|
date | Wed, 15 Aug 2007 19:57:01 +0000 |
parents | 1c1adededd8f |
children | 29c33ce6c5bb |
files | trunk/src/Parser.d trunk/src/Token.d |
diffstat | 2 files changed, 19 insertions(+), 6 deletions(-) [+] |
line wrap: on
line diff
--- a/trunk/src/Parser.d Wed Aug 15 19:26:04 2007 +0000 +++ b/trunk/src/Parser.d Wed Aug 15 19:57:01 2007 +0000 @@ -49,7 +49,7 @@ writef("%s", prev[0 .. token.end - prev]); prev = token.end; } - } while (token.type == T.Comment) // Skip comments + } while (token.isWhitespace) // Skip whitespace } void skipToOnePast(TOK tok) @@ -99,7 +99,7 @@ Token* next = token; do lx.peek(next); - while (next.type == T.Comment) + while (next.isWhitespace) // Skip whitespace return next.type; }
--- a/trunk/src/Token.d Wed Aug 15 19:26:04 2007 +0000 +++ b/trunk/src/Token.d Wed Aug 15 19:57:01 2007 +0000 @@ -12,12 +12,17 @@ size_t col; } -enum TOK +enum TOK : ushort { Invalid, - Identifier, - Comment, + /// Flag for whitespace tokens that must be ignored in the parsing phase. + Whitespace = 0x8000, + Comment = 1 | Whitespace, + Shebang = 2 | Whitespace, + HashLine = 3 | Whitespace, + + Identifier = 4, String, Special, CharLiteral, WCharLiteral, DCharLiteral, @@ -144,6 +149,11 @@ return KeywordsBegin <= type && type <= KeywordsEnd; } + bool isWhitespace() + { + return !!(type & TOK.Whitespace); + } + int opEquals(TOK type2) { return type == type2; @@ -167,8 +177,11 @@ string[] tokToString = [ "Invalid", + "Comment", + "#! /shebang/", + "#line", + "Identifier", - "Comment", "String", "Special", "CharLiteral", "WCharLiteral", "DCharLiteral",