Mercurial > projects > dil

changeset 497:0ffcc4ff82f3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Refactored a few things in the Lexer. Fix: SpecialTokensEnd should be TOK.VERSION not TOK.Version. Fixed assert statement in textSpan().
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Sun, 09 Dec 2007 14:58:38 +0100
parents 5a607597dc22
children 49c201b5c465
files trunk/src/dil/Keywords.d trunk/src/dil/Lexer.d trunk/src/dil/Token.d
diffstat 3 files changed, 19 insertions(+), 18 deletions(-) [+]
line wrap: on
line diff
--- a/trunk/src/dil/Keywords.d	Sun Dec 09 13:04:15 2007 +0100
+++ b/trunk/src/dil/Keywords.d	Sun Dec 09 14:58:38 2007 +0100
@@ -6,6 +6,7 @@
 import dil.Token;
 import dil.Identifier;
 
+/// Table of reserved identifiers.
 static const Identifier[] keywords = [
   {TOK.Abstract, "abstract"},
   {TOK.Alias, "alias"},
--- a/trunk/src/dil/Lexer.d	Sun Dec 09 13:04:15 2007 +0100
+++ b/trunk/src/dil/Lexer.d	Sun Dec 09 14:58:38 2007 +0100
@@ -297,17 +297,17 @@
         }
         assert(id);
         t.type = id.type;
-        if (t.type == TOK.Identifier)
+        if (t.type == TOK.Identifier || t.isKeyword)
           return;
-        if (t.type == TOK.EOF)
+        else if (t.isSpecialToken)
+          finalizeSpecialToken(t);
+        else if (t.type == TOK.EOF)
         {
-          t.type = TOK.EOF;
-          t.end = p;
           tail = &t;
           assert(t.srcText == "__EOF__");
         }
-        else if (t.isSpecialToken)
-          finalizeSpecialToken(t);
+        else
+          assert(0, "unexpected token: " ~ t.srcText);
         return;
       }
 
@@ -610,9 +610,9 @@
       }
 
       // Check for EOF
-      if (c == 0 || c == _Z_)
+      if (isEOF(c))
       {
-        assert(*p == 0 || *p == _Z_);
+        assert(isEOF(*p), ""~*p);
         t.type = TOK.EOF;
         t.end = p;
         tail = &t;
@@ -1049,17 +1049,17 @@
       }
       assert(id);
       t.type = id.type;
-      if (t.type == TOK.Identifier)
+      if (t.type == TOK.Identifier || t.isKeyword)
         return;
-      if (t.type == TOK.EOF)
+      else if (t.isSpecialToken)
+        finalizeSpecialToken(t);
+      else if (t.type == TOK.EOF)
       {
-        t.type = TOK.EOF;
-        t.end = p;
         tail = &t;
         assert(t.srcText == "__EOF__");
       }
-      else if (t.isSpecialToken)
-        finalizeSpecialToken(t);
+      else
+        assert(0, "unexpected token: " ~ t.srcText);
       return;
     }
 
@@ -1067,9 +1067,9 @@
       return scanNumber(t);
 
     // Check for EOF
-    if (c == 0 || c == _Z_)
+    if (isEOF(c))
     {
-      assert(*p == 0 || *p == _Z_, *p~"");
+      assert(isEOF(*p), *p~"");
       t.type = TOK.EOF;
       t.end = p;
       tail = &t;
--- a/trunk/src/dil/Token.d	Sun Dec 09 13:04:15 2007 +0100
+++ b/trunk/src/dil/Token.d	Sun Dec 09 14:58:38 2007 +0100
@@ -113,7 +113,7 @@
 alias TOK.Abstract KeywordsBegin;
 alias TOK.With KeywordsEnd;
 alias TOK.FILE SpecialTokensBegin;
-alias TOK.Version SpecialTokensEnd;
+alias TOK.VERSION SpecialTokensEnd;
 
 struct Token
 {
@@ -330,7 +330,7 @@
   /// Return the source text enclosed by the left and right token.
   static char[] textSpan(Token* left, Token* right)
   {
-    assert(left.end <= right.start);
+    assert(left.end <= right.start || left is right );
     return left.start[0 .. right.end - left.start];
   }