Mercurial > projects > dil

diff trunk/src/dil/Token.d @ 390:4d36eea1bbc9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Refactored Lexer.scan(). Illegal characters are not ignored anymore. They are reported as errors. Added a new member 'ws' to Token. When a token is scanned the lexer sets ws to the leading whitespace or leaves it at null when no whitespace was found. Added Illegal to enum TOK and IllegalCharacter to enum MID. Added localized messages for MID.IllegalCharacter. Adapted code of cmd.Generate to make use of Token.ws.
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Wed, 12 Sep 2007 21:03:41 +0200
parents 6a5fc22cae34
children 33b566df6af4
line wrap: on
line diff
--- a/trunk/src/dil/Token.d	Wed Sep 12 18:18:29 2007 +0200
+++ b/trunk/src/dil/Token.d	Wed Sep 12 21:03:41 2007 +0200
@@ -18,12 +18,13 @@
 
   /// Flag for whitespace tokens that must be ignored in the parsing phase.
   Whitespace = 0x8000,
-  Comment = 1 | Whitespace,
-  Shebang = 2 | Whitespace,
-  HashLine = 3 | Whitespace,
-  Filespec = 4 | Whitespace,
+  Illegal = 1 | Whitespace,
+  Comment = 2 | Whitespace,
+  Shebang = 3 | Whitespace,
+  HashLine = 4 | Whitespace,
+  Filespec = 5 | Whitespace,
 
-  Identifier = 5,
+  Identifier = 6,
   String,
   CharLiteral, WCharLiteral, DCharLiteral,
 
@@ -121,8 +122,9 @@
 
   Token* next, prev;
 
-  char* start;
-  char* end;
+  char* ws;    /// Start of whitespace characters before token. Null if no WS.
+  char* start; /// Start of token in source text.
+  char* end;   /// Points one past the end of token in source text.
 
   union
   {
@@ -236,6 +238,7 @@
 const string[] tokToString = [
   "Invalid",
 
+  "Illegal",
   "Comment",
   "#! /shebang/",
   "#line",