# HG changeset patch
# User Aziz K?ksal <aziz.koeksal@gmail.com>
# Date 1190725481 -7200
# Node ID cca83c0c00fd8592ce5f0b82430712a28446582b
# Parent  4d9ee8e60712b64a82555aa04ca26c50aa3aacbb
Added __EOF__ token.

Moved if-statement checking for 0 and _Z_ to the bottom of Lexer.scan().

diff -r 4d9ee8e60712 -r cca83c0c00fd trunk/src/dil/Keywords.d
--- a/trunk/src/dil/Keywords.d	Tue Sep 25 14:28:35 2007 +0200
+++ b/trunk/src/dil/Keywords.d	Tue Sep 25 15:04:41 2007 +0200
@@ -114,4 +114,5 @@
   {TOK.TIMESTAMP, "__TIMESTAMP__"},
   {TOK.VENDOR, "__VENDOR__"},
   {TOK.VERSION, "__VERSION__"},
+  {TOK.EOF, "__EOF__"}, // D2.0
 ];
diff -r 4d9ee8e60712 -r cca83c0c00fd trunk/src/dil/Lexer.d
--- a/trunk/src/dil/Lexer.d	Tue Sep 25 14:28:35 2007 +0200
+++ b/trunk/src/dil/Lexer.d	Tue Sep 25 15:04:41 2007 +0200
@@ -210,16 +210,6 @@
     {
       t.start = p;
 
-      if (c == 0 || c == _Z_)
-      {
-        assert(*p == 0 || *p == _Z_);
-        t.type = TOK.EOF;
-        t.end = p;
-        tail = &t;
-        assert(t.start == t.end);
-        return;
-      }
-
       if (isidbeg(c))
       {
         if (c == 'r' && p[1] == '"' && ++p)
@@ -250,7 +240,16 @@
         }
         assert(id);
         t.type = id.type;
-        if (t.isSpecialToken)
+        if (t.type == TOK.Identifier)
+          return;
+        if (t.type == TOK.EOF)
+        {
+          t.type = TOK.EOF;
+          t.end = p;
+          tail = &t;
+          assert(t.srcText == "__EOF__");
+        }
+        else if (t.isSpecialToken)
           finalizeSpecialToken(t);
         return;
       }
@@ -657,6 +656,17 @@
       default:
       }
 
+      // Check for EOF
+      if (c == 0 || c == _Z_)
+      {
+        assert(*p == 0 || *p == _Z_);
+        t.type = TOK.EOF;
+        t.end = p;
+        tail = &t;
+        assert(t.start == t.end);
+        return;
+      }
+
       if (c & 128)
       {
         c = decodeUTF8();