# HG changeset patch
# User Aziz K?ksal <aziz.koeksal@gmail.com>
# Date 1198189603 -3600
# Node ID 3bc7801c207eec6c38f03b56e5517019ad5631bb
# Parent  312da78ab301ebb61939914c5613005322dad2f9
Refactored the way how tokens are flagged as whitespace.

diff -r 312da78ab301 -r 3bc7801c207e trunk/src/dil/Lexer.d
--- a/trunk/src/dil/Lexer.d	Thu Dec 20 20:00:48 2007 +0100
+++ b/trunk/src/dil/Lexer.d	Thu Dec 20 23:26:43 2007 +0100
@@ -74,6 +74,7 @@
     // Add a newline as the first token after the head.
     auto newline = new Token;
     newline.type = TOK.Newline;
+    newline.setWhitespaceFlag();
     newline.start = newline.end = this.p;
     newline.filePath = this.errorPath;
     newline.lineNum = 1;
@@ -108,6 +109,7 @@
     {
       auto t = new Token;
       t.type = TOK.Shebang;
+      t.setWhitespaceFlag();
       t.start = p;
       ++p;
       while (!isEndOfLine(++p))
@@ -253,6 +255,7 @@
         setLineBegin(p);
 //         this.newline = &t;
         t.type = TOK.Newline;
+        t.setWhitespaceFlag();
         t.filePath = this.errorPath;
         t.lineNum = lineNum;
         t.lineNum_hline = lineNum_hline;
@@ -326,6 +329,7 @@
           while (!isEndOfLine(++p))
             isascii(*p) || decodeUTF8();
           t.type = TOK.Comment;
+          t.setWhitespaceFlag();
           t.end = p;
           return;
         default:
@@ -625,6 +629,7 @@
 
       ++p;
       t.type = TOK.Illegal;
+      t.setWhitespaceFlag();
       t.dchar_ = c;
       t.end = p;
       return;
@@ -705,6 +710,7 @@
       setLineBegin(p);
 //       this.newline = &t;
       t.type = TOK.Newline;
+      t.setWhitespaceFlag();
       t.filePath = this.errorPath;
       t.lineNum = lineNum;
       t.lineNum_hline = lineNum_hline;
@@ -817,6 +823,7 @@
       while (!isEndOfLine(++p))
         isascii(*p) || decodeUTF8();
       t.type = TOK.Comment;
+      t.setWhitespaceFlag();
       t.end = p;
       return;
     case toUint!(">="):
@@ -1076,6 +1083,7 @@
 
     ++p;
     t.type = TOK.Illegal;
+    t.setWhitespaceFlag();
     t.dchar_ = c;
     t.end = p;
     return;
@@ -1118,6 +1126,7 @@
       }
     }
     t.type = TOK.Comment;
+    t.setWhitespaceFlag();
     t.end = p;
     return;
   }
@@ -1167,6 +1176,7 @@
       }
     }
     t.type = TOK.Comment;
+    t.setWhitespaceFlag();
     t.end = p;
     return;
   }
@@ -2246,6 +2256,7 @@
   {
     assert(*p == '#');
     t.type = TOK.HashLine;
+    t.setWhitespaceFlag();
 
     MID mid;
     auto errorAtColumn = p;
@@ -2299,6 +2310,7 @@
         t.tokLineFilespec = new Token;
         t.tokLineFilespec.start = p;
         t.tokLineFilespec.type = TOK.Filespec;
+        t.tokLineFilespec.setWhitespaceFlag();
         while (*++p != '"')
         {
           if (isEndOfLine(p))
diff -r 312da78ab301 -r 3bc7801c207e trunk/src/dil/Token.d
--- a/trunk/src/dil/Token.d	Thu Dec 20 20:00:48 2007 +0100
+++ b/trunk/src/dil/Token.d	Thu Dec 20 23:26:43 2007 +0100
@@ -16,7 +16,14 @@
 +/
 struct Token
 {
+  enum Flags : ushort
+  {
+    None,
+    Whitespace = 1, /// Tokens with this flag are ignored by the Parser.
+  }
+
   TOK type; /// The type of the token.
+  Flags flags; /// The flags of the token.
   /// Pointers to the next and previous tokens (doubly-linked list.)
   Token* next, prev;
 
@@ -113,6 +120,12 @@
     return tokToString[tok];
   }
 
+  /// Adds Flags.Whitespace to this token's flags.
+  void setWhitespaceFlag()
+  {
+    this.flags |= Flags.Whitespace;
+  }
+
   /++
     Returns true if this is a token that can have newlines in it.
     These can be block and nested comments and any string literal
@@ -139,7 +152,7 @@
   /// Returns true if this is a whitespace token.
   bool isWhitespace()
   {
-    return !!(type & TOK.Whitespace);
+    return !!(flags & Flags.Whitespace);
   }
 
   /// Returns true if this is a special token.
diff -r 312da78ab301 -r 3bc7801c207e trunk/src/dil/TokensEnum.d
--- a/trunk/src/dil/TokensEnum.d	Thu Dec 20 20:00:48 2007 +0100
+++ b/trunk/src/dil/TokensEnum.d	Thu Dec 20 23:26:43 2007 +0100
@@ -9,17 +9,15 @@
 {
   Invalid,
 
-  /// Flag for whitespace tokens that must be ignored in the parsing phase.
-  Whitespace = 0x8000,
-  Illegal  = 1 | Whitespace,
-  Comment  = 2 | Whitespace,
-  Shebang  = 3 | Whitespace,
-  HashLine = 4 | Whitespace,
-  Filespec = 5 | Whitespace,
-  Newline  = 6 | Whitespace,
-  Empty    = 7,
+  Illegal,
+  Comment,
+  Shebang,
+  HashLine,
+  Filespec,
+  Newline,
+  Empty,
 
-  Identifier = 8,
+  Identifier,
   String,
   CharLiteral,