# HG changeset patch
# User Aziz K?ksal <aziz.koeksal@gmail.com>
# Date 1190905588 -7200
# Node ID 7354f15cd5e907b528dac03d5769207cb8dd0f96
# Parent  8af5c7e2f722abf82b411d4515c1d913d5189a22
Applied some fixes to the Lexer.

do-while loop should be while-loop in destructor.\n
Checked for LS and PS using decodeUTF8 in scanShebang(), but 'p' mustn't
be advanced beyond a newline.

Fix: pushing token to token2LocTable only after all whitespace and newlines
have been skipped.

diff -r 8af5c7e2f722 -r 7354f15cd5e9 trunk/src/dil/Lexer.d
--- a/trunk/src/dil/Lexer.d	Thu Sep 27 13:59:36 2007 +0200
+++ b/trunk/src/dil/Lexer.d	Thu Sep 27 17:06:28 2007 +0200
@@ -99,12 +99,12 @@
   ~this()
   {
     auto token = head.next;
-    do
+    while (token !is null)
     {
       assert(token.type == TOK.EOF ? token == tail && token.next is null : 1);
       delete token.prev;
       token = token.next;
-    } while (token !is null)
+    }
     delete tail;
   }
 
@@ -124,13 +124,12 @@
         {
         case '\r', '\n', 0, _Z_:
           break;
+        case LS[0]:
+          if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
+            break;
         default:
           if (*p & 128)
-          {
-            auto c = decodeUTF8();
-            if (c == LSd || c == PSd)
-              break;
-          }
+            decodeUTF8();
           continue;
         }
         break; // Exit loop.
@@ -197,6 +196,7 @@
   {
     // Scan whitespace.
     auto pws = p;
+    auto old_loc = this.loc;
     while (1)
     {
       switch (*p)
@@ -207,8 +207,6 @@
       case '\n':
         ++p;
         ++loc;
-      version(token2LocTable)
-        token2LocTable[&t] = Location(loc, null);
         continue;
       case LS[0]:
         if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
@@ -227,7 +225,12 @@
     }
 
     if (p != pws)
+    {
       t.ws = pws;
+      if (old_loc != this.loc)
+        version(token2LocTable)
+          token2LocTable[&t] = Location(loc, null);
+    }
 
     // Scan token.
     uint c = *p;
@@ -673,6 +676,7 @@
   {
     // Scan whitespace.
     auto pws = p;
+    auto old_loc = this.loc;
     while (1)
     {
       switch (*p)
@@ -683,8 +687,6 @@
       case '\n':
         ++p;
         ++loc;
-      version(token2LocTable)
-        token2LocTable[&t] = Location(loc, null);
         continue;
       case LS[0]:
         if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
@@ -703,7 +705,12 @@
     }
 
     if (p != pws)
+    {
       t.ws = pws;
+      if (old_loc != this.loc)
+        version(token2LocTable)
+          token2LocTable[&t] = Location(loc, null);
+    }
 
     // Scan token.
     t.start = p;