# HG changeset patch
# User Aziz K?ksal <aziz.koeksal@gmail.com>
# Date 1190881688 -7200
# Node ID 9c69615a4876113f32b5d89d6a64081afeff1278
# Parent  0fd78fdcb982300c60f98555f3169d566cf41909
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.

Fixed the while statement that skips to the next valid UTF-8 sequence.

diff -r 0fd78fdcb982 -r 9c69615a4876 trunk/src/dil/Lexer.d
--- a/trunk/src/dil/Lexer.d	Wed Sep 26 17:12:00 2007 +0200
+++ b/trunk/src/dil/Lexer.d	Thu Sep 27 10:28:08 2007 +0200
@@ -726,6 +726,15 @@
       else
       {
         c <<= 8; c |= p[1]; c <<= 8; c |= p[2]; c <<= 8; c |= p[3];
+        /+
+        c = *cast(uint*)p;
+        asm
+        {
+          mov EDX, c;
+          bswap EDX;
+          mov c, EDX;
+        }
+        +/
       }
     }
 
@@ -896,6 +905,7 @@
     assert(p == t.start);
     assert(*p == c, Format("p={0},c={1}", *p, cast(dchar)c));
     // 1 character tokens.
+    // TODO: consider storing the token type in ptable.
     switch (c)
     {
     case '\'':
@@ -2385,7 +2395,7 @@
     {
       error(MID.InvalidUTF8Sequence);
       // Skip to next valid utf-8 sequence
-      while (p < end && UTF8stride[*++p] != 0xFF) {}
+      while (++p < end && UTF8stride[*p] == 0xFF) {}
       --p;
       assert(p < end);
     }
@@ -2432,6 +2442,28 @@
     return State(this);
   }
 +/
+  /+
+    Insert an empty dummy token before t.
+    Useful in the parsing phase for representing a node in the AST
+    that doesn't consume an actual token from the source text.
+  +/
+  Token* insertEmptyTokenBefore(Token* t)
+  {
+    assert(t !is null && t.prev !is null);
+    assert(text.ptr <= t.start && t.start < end, Token.toString(t.type));
+    assert(text.ptr <= t.end && t.end <= end, Token.toString(t.type));
+
+    auto prev_t = t.prev;
+    auto new_t = new Token;
+    new_t.type = TOK.Empty;
+    new_t.start = new_t.end = prev_t.end;
+    // Link in new token.
+    prev_t.next = new_t;
+    new_t.prev = prev_t;
+    new_t.next = t;
+    t.prev = new_t;
+    return new_t;
+  }
 
   private void scanNext(ref Token* t)
   {
diff -r 0fd78fdcb982 -r 9c69615a4876 trunk/src/dil/Parser.d
--- a/trunk/src/dil/Parser.d	Wed Sep 26 17:12:00 2007 +0200
+++ b/trunk/src/dil/Parser.d	Thu Sep 27 10:28:08 2007 +0200
@@ -2966,7 +2966,12 @@
     default:
       error(MID.ExpectedButFound, "Expression", token.srcText);
       e = new EmptyExpression();
-      break;
+      if (!trying)
+      {
+        // Insert a dummy token and don't consume current one.
+        begin = lx.insertEmptyTokenBefore(token);
+        prevToken = begin;
+      }
     }
     set(e, begin);
     return e;
@@ -3633,6 +3638,12 @@
 
       error(MID.ExpectedButFound, "Expression", token.srcText);
       e = new EmptyExpression();
+      if (!trying)
+      {
+        // Insert a dummy token and don't consume current one.
+        begin = lx.insertEmptyTokenBefore(token);
+        prevToken = begin;
+      }
     }
     set(e, begin);
     return e;
diff -r 0fd78fdcb982 -r 9c69615a4876 trunk/src/dil/Token.d
--- a/trunk/src/dil/Token.d	Wed Sep 26 17:12:00 2007 +0200
+++ b/trunk/src/dil/Token.d	Thu Sep 27 10:28:08 2007 +0200
@@ -19,13 +19,14 @@
 
   /// Flag for whitespace tokens that must be ignored in the parsing phase.
   Whitespace = 0x8000,
-  Illegal = 1 | Whitespace,
-  Comment = 2 | Whitespace,
-  Shebang = 3 | Whitespace,
+  Illegal  = 1 | Whitespace,
+  Comment  = 2 | Whitespace,
+  Shebang  = 3 | Whitespace,
   HashLine = 4 | Whitespace,
   Filespec = 5 | Whitespace,
+  Empty    = 6,
 
-  Identifier = 6,
+  Identifier = 7,
   String,
   CharLiteral, WCharLiteral, DCharLiteral,
 
@@ -287,6 +288,7 @@
   "#! /shebang/",
   "#line",
   `"filespec"`,
+  "Empty",
 
   "Identifier",
   "String",