Mercurial > projects > dil

changeset 314:ebd21bbf296e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such. - Added method isWhitespace() to Token. Adapted parser accordingly.
author aziz
date Wed, 15 Aug 2007 19:57:01 +0000
parents 1c1adededd8f
children 29c33ce6c5bb
files trunk/src/Parser.d trunk/src/Token.d
diffstat 2 files changed, 19 insertions(+), 6 deletions(-) [+]
line wrap: on
line diff
--- a/trunk/src/Parser.d	Wed Aug 15 19:26:04 2007 +0000
+++ b/trunk/src/Parser.d	Wed Aug 15 19:57:01 2007 +0000
@@ -49,7 +49,7 @@
       writef("%s", prev[0 .. token.end - prev]);
       prev = token.end;
 }
-    } while (token.type == T.Comment) // Skip comments
+    } while (token.isWhitespace) // Skip whitespace
   }
 
   void skipToOnePast(TOK tok)
@@ -99,7 +99,7 @@
     Token* next = token;
     do
       lx.peek(next);
-    while (next.type == T.Comment)
+    while (next.isWhitespace) // Skip whitespace
     return next.type;
   }
 
--- a/trunk/src/Token.d	Wed Aug 15 19:26:04 2007 +0000
+++ b/trunk/src/Token.d	Wed Aug 15 19:57:01 2007 +0000
@@ -12,12 +12,17 @@
   size_t col;
 }
 
-enum TOK
+enum TOK : ushort
 {
   Invalid,
 
-  Identifier,
-  Comment,
+  /// Flag for whitespace tokens that must be ignored in the parsing phase.
+  Whitespace = 0x8000,
+  Comment = 1 | Whitespace,
+  Shebang = 2 | Whitespace,
+  HashLine = 3 | Whitespace,
+
+  Identifier = 4,
   String,
   Special,
   CharLiteral, WCharLiteral, DCharLiteral,
@@ -144,6 +149,11 @@
     return KeywordsBegin <= type && type <= KeywordsEnd;
   }
 
+  bool isWhitespace()
+  {
+    return !!(type & TOK.Whitespace);
+  }
+
   int opEquals(TOK type2)
   {
     return type == type2;
@@ -167,8 +177,11 @@
 string[] tokToString = [
   "Invalid",
 
+  "Comment",
+  "#! /shebang/",
+  "#line",
+
   "Identifier",
-  "Comment",
   "String",
   "Special",
   "CharLiteral", "WCharLiteral", "DCharLiteral",