Mercurial > projects > dil

changeset 411:cca83c0c00fd

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added __EOF__ token. Moved if-statement checking for 0 and _Z_ to the bottom of Lexer.scan().
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Tue, 25 Sep 2007 15:04:41 +0200
parents 4d9ee8e60712
children fb31af0fda73
files trunk/src/dil/Keywords.d trunk/src/dil/Lexer.d
diffstat 2 files changed, 22 insertions(+), 11 deletions(-) [+]
line wrap: on
line diff
--- a/trunk/src/dil/Keywords.d	Tue Sep 25 14:28:35 2007 +0200
+++ b/trunk/src/dil/Keywords.d	Tue Sep 25 15:04:41 2007 +0200
@@ -114,4 +114,5 @@
   {TOK.TIMESTAMP, "__TIMESTAMP__"},
   {TOK.VENDOR, "__VENDOR__"},
   {TOK.VERSION, "__VERSION__"},
+  {TOK.EOF, "__EOF__"}, // D2.0
 ];
--- a/trunk/src/dil/Lexer.d	Tue Sep 25 14:28:35 2007 +0200
+++ b/trunk/src/dil/Lexer.d	Tue Sep 25 15:04:41 2007 +0200
@@ -210,16 +210,6 @@
     {
       t.start = p;
 
-      if (c == 0 || c == _Z_)
-      {
-        assert(*p == 0 || *p == _Z_);
-        t.type = TOK.EOF;
-        t.end = p;
-        tail = &t;
-        assert(t.start == t.end);
-        return;
-      }
-
       if (isidbeg(c))
       {
         if (c == 'r' && p[1] == '"' && ++p)
@@ -250,7 +240,16 @@
         }
         assert(id);
         t.type = id.type;
-        if (t.isSpecialToken)
+        if (t.type == TOK.Identifier)
+          return;
+        if (t.type == TOK.EOF)
+        {
+          t.type = TOK.EOF;
+          t.end = p;
+          tail = &t;
+          assert(t.srcText == "__EOF__");
+        }
+        else if (t.isSpecialToken)
           finalizeSpecialToken(t);
         return;
       }
@@ -657,6 +656,17 @@
       default:
       }
 
+      // Check for EOF
+      if (c == 0 || c == _Z_)
+      {
+        assert(*p == 0 || *p == _Z_);
+        t.type = TOK.EOF;
+        t.end = p;
+        tail = &t;
+        assert(t.start == t.end);
+        return;
+      }
+
       if (c & 128)
       {
         c = decodeUTF8();