changeset 502:4e14cd1b24da

Refactored code and added modules related to tabulated Identifiers. Rearranged members of struct Identifier and added new member ID identID. Moved idTableLookup to module dil.IdTable. Renamed module TokenIDs to TokensEnum. Added member Identifier* ident to struct Token. Changed string switchtes in Parser to integer switches using enum ID.
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Tue, 11 Dec 2007 14:19:30 +0100
parents 949a53332c66
children fa63ef408790
files trunk/src/dil/Expressions.d trunk/src/dil/IdTable.d trunk/src/dil/Identifier.d trunk/src/dil/IdentsEnum.d trunk/src/dil/IdentsGenerator.d trunk/src/dil/Keywords.d trunk/src/dil/Lexer.d trunk/src/dil/Parser.d trunk/src/dil/Token.d trunk/src/dil/TokensEnum.d
diffstat 10 files changed, 596 insertions(+), 208 deletions(-) [+]
line wrap: on
line diff
--- a/trunk/src/dil/Expressions.d	Sun Dec 09 23:27:40 2007 +0100
+++ b/trunk/src/dil/Expressions.d	Tue Dec 11 14:19:30 2007 +0100
@@ -8,6 +8,7 @@
 import dil.Types;
 import dil.Declarations;
 import dil.Statements;
+import dil.Identifier;
 import dil.Scope;
 
 abstract class Expression : Node
@@ -1012,9 +1013,9 @@
 
 class AsmRegisterExpression : Expression
 {
-  Token* register;
-  Token* number; // ST(0) - ST(7) or FS:0, FS:4, FS:8
-  this(Token* register, Token* number = null)
+  Identifier* register;
+  int number; // ST(0) - ST(7) or FS:0, FS:4, FS:8
+  this(Identifier* register, int number = -1)
   {
     mixin(set_kind);
     this.register = register;
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/trunk/src/dil/IdTable.d	Tue Dec 11 14:19:30 2007 +0100
@@ -0,0 +1,99 @@
+/++
+  Author: Aziz Köksal
+  License: GPL3
++/
+module dil.IdTable;
+import dil.Identifier;
+import dil.IdentsEnum;
+import dil.TokensEnum;
+import dil.IdentsGenerator;
+import dil.Keywords;
+import common;
+
+struct Ident
+{
+  const static
+  {
+    mixin(generateIdentMembers());
+  }
+
+  static Identifier*[] allIds()
+  {
+    return __allIds;
+  }
+}
+
+struct IdTable
+{
+static:
+  /// A set of common, predefined identifiers for fast lookups.
+  private Identifier*[string] staticTable;
+  /// A table that grows with every newly found, unique identifier.
+  /// Access must be synchronized.
+  private Identifier*[string] growingTable;
+
+  /// Initializes the static table.
+  static this()
+  {
+    foreach (ref k; keywords)
+      staticTable[k.str] = &k;
+    foreach (id; Ident.allIds())
+      staticTable[id.str] = id;
+    staticTable.rehash;
+  }
+
+  /// Looks in both tables.
+  Identifier* lookup(string idString)
+  {
+    auto id = inStatic(idString);
+    if (id)
+      return id;
+    return inGrowing(idString);
+  }
+
+  /// Look up idString in the static table.
+  Identifier* inStatic(string idString)
+  {
+    auto id = idString in staticTable;
+    return id ? *id : null;
+  }
+
+  /++
+    Returns the Identifier for idString.
+    Adds idString to the table if not found.
+    Access to the data structure is synchronized.
+  +/
+  Identifier* inGrowing(string idString)
+  out(id)
+  { assert(id !is null); }
+  body
+  {
+    synchronized
+    {
+      auto id = idString in growingTable;
+      if (id)
+        return *id;
+      auto newID = Identifier(idString, TOK.Identifier);
+      growingTable[idString] = newID;
+      return newID;
+    }
+  }
+
+  /+
+  Identifier* addIdentifiers(char[][] idStrings)
+  {
+    auto ids = new Identifier*[idStrings.length];
+    foreach (i, idString; idStrings)
+    {
+      Identifier** id = idString in tabulatedIds;
+      if (!id)
+      {
+        auto newID = Identifier(TOK.Identifier, idString);
+        tabulatedIds[idString] = newID;
+        id = &newID;
+      }
+      ids[i] = *id;
+    }
+  }
+  +/
+}
--- a/trunk/src/dil/Identifier.d	Sun Dec 09 23:27:40 2007 +0100
+++ b/trunk/src/dil/Identifier.d	Tue Dec 11 14:19:30 2007 +0100
@@ -3,20 +3,24 @@
   License: GPL3
 +/
 module dil.Identifier;
-import dil.Token;
+import dil.TokensEnum;
+import dil.IdentsEnum;
 import common;
 
+align(1)
 struct Identifier
 {
+  string str;
   TOK type;
-  string str;
+  ID identID;
 
-  static Identifier* opCall(TOK type, string str)
+  static Identifier* opCall(string str, TOK type, ID identID = ID.Null)
   {
-    auto i = new Identifier;
-    i.type = type;
-    i.str = str;
-    return i;
+    auto id = new Identifier;
+    id.str = str;
+    id.type = type;
+    id.identID = identID;
+    return id;
   }
 
   uint toHash()
@@ -29,3 +33,4 @@
     return hash;
   }
 }
+// pragma(msg, Identifier.sizeof.stringof);
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/trunk/src/dil/IdentsEnum.d	Tue Dec 11 14:19:30 2007 +0100
@@ -0,0 +1,13 @@
+/++
+  Author: Aziz Köksal
+  License: GPL3
++/
+module dil.IdentsEnum;
+import dil.IdentsGenerator;
+
+mixin(
+  "enum ID : ushort {"
+    "Null,"
+    ~ generateIDMembers ~
+  "}"
+);
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/trunk/src/dil/IdentsGenerator.d	Tue Dec 11 14:19:30 2007 +0100
@@ -0,0 +1,93 @@
+/++
+  Author: Aziz Köksal
+  License: GPL3
++/
+module dil.IdentsGenerator;
+
+struct StrPair
+{
+const:
+  char[] str;   /// Identifier string in code.
+  char[] idStr; /// In table.
+}
+
+static const StrPair[] identPairs = [
+  // scope:
+  {"exit"},
+  {"success"},
+  {"failure"},
+  // pragma:
+  {"msg"}, {"lib"},
+  // Linkage:
+  {"C"}, {"D"}, {"Windows"}, {"Pascal"}, {"System"},
+  // Operator methods:
+  {"opCall"},
+  // ASM identifiers:
+  {"near"}, {"far"}, {"word"}, {"dword"}, {"qword"},
+  {"ptr"}, {"offset"}, {"seg"}, {"__LOCAL_SIZE"},
+  {"FS"}, {"ST"},
+  {"AL"}, {"AH"}, {"AX"}, {"EAX"},
+  {"BL"}, {"BH"}, {"BX"}, {"EBX"},
+  {"CL"}, {"CH"}, {"CX"}, {"ECX"},
+  {"DL"}, {"DH"}, {"DX"}, {"EDX"},
+  {"BP"}, {"EBP"}, {"SP"}, {"ESP"},
+  {"DI"}, {"EDI"}, {"SI"}, {"ESI"},
+  {"ES"}, {"CS"}, {"SS"}, {"DS"}, {"GS"},
+  {"CR0"}, {"CR2"}, {"CR3"}, {"CR4"},
+  {"DR0"}, {"DR1"}, {"DR2"}, {"DR3"}, {"DR6"}, {"DR7"},
+  {"TR3"}, {"TR4"}, {"TR5"}, {"TR6"}, {"TR7"},
+  {"MM0"}, {"MM1"}, {"MM2"}, {"MM3"},
+  {"MM4"}, {"MM5"}, {"MM6"}, {"MM7"},
+  {"XMM0"}, {"XMM1"}, {"XMM2"}, {"XMM3"},
+  {"XMM4"}, {"XMM5"}, {"XMM6"}, {"XMM7"},
+];
+
+/++
+  CTF for generating the members of the struct Ident.
+  The resulting string could look like this:
+  ---
+  private struct Ids {static const:
+    Identifier _str = {"str", TOK.Identifier, ID.str};
+    // more ...
+  }
+  Identifier* str = &Ids._str;
+  // more ...
+  private Identifier*[] __allIds = [
+    str,
+    // more ...
+  ]
+  ---
++/
+char[] generateIdentMembers(char[] private_members = "")
+{
+  private_members = "private struct Ids {static const:";
+
+  char[] public_members = "";
+  char[] array = "private Identifier*[] __allIds = [";
+  foreach (pair; identPairs)
+  {
+    // NB: conditional makes function uneligible for CTE.
+    // char[] idString = pair.idStr ? pair.idStr : pair.str;
+    // Identifier _str = {"str", TOK.Identifier, ID.str};
+    private_members ~= "Identifier _"~pair.str~` = {"`~pair.str~`", TOK.Identifier, ID.`~pair.str~"};\n";
+    // Identifier* str = &_str;
+    public_members ~= "Identifier* "~pair.str~" = &Ids._"~pair.str~";\n";
+    array ~= pair.str~",";
+  }
+
+  private_members ~= "}"; // Close private {
+  array ~= "];";
+
+  return private_members ~ public_members ~ array;
+}
+
+/// CTF for generating the members of the enum ID.
+char[] generateIDMembers(char[] members = "")
+{
+  foreach (pair; identPairs)
+    members ~= pair.str ~ ",\n";
+  return members;
+}
+
+// pragma(msg, generateIdentMembers());
+// pragma(msg, generateIDMembers());
--- a/trunk/src/dil/Keywords.d	Sun Dec 09 23:27:40 2007 +0100
+++ b/trunk/src/dil/Keywords.d	Tue Dec 11 14:19:30 2007 +0100
@@ -8,112 +8,112 @@
 
 /// Table of reserved identifiers.
 static const Identifier[] keywords = [
-  {TOK.Abstract, "abstract"},
-  {TOK.Alias, "alias"},
-  {TOK.Align, "align"},
-  {TOK.Asm, "asm"},
-  {TOK.Assert, "assert"},
-  {TOK.Auto, "auto"},
-  {TOK.Body, "body"},
-  {TOK.Bool, "bool"},
-  {TOK.Break, "break"},
-  {TOK.Byte, "byte"},
-  {TOK.Case, "case"},
-  {TOK.Cast, "cast"},
-  {TOK.Catch, "catch"},
-  {TOK.Cdouble, "cdouble"},
-  {TOK.Cent, "cent"},
-  {TOK.Cfloat, "cfloat"},
-  {TOK.Char, "char"},
-  {TOK.Class, "class"},
-  {TOK.Const, "const"},
-  {TOK.Continue, "continue"},
-  {TOK.Creal, "creal"},
-  {TOK.Dchar, "dchar"},
-  {TOK.Debug, "debug"},
-  {TOK.Default, "default"},
-  {TOK.Delegate, "delegate"},
-  {TOK.Delete, "delete"},
-  {TOK.Deprecated, "deprecated"},
-  {TOK.Do, "do"},
-  {TOK.Double, "double"},
-  {TOK.Else, "else"},
-  {TOK.Enum, "enum"},
-  {TOK.Export, "export"},
-  {TOK.Extern, "extern"},
-  {TOK.False, "false"},
-  {TOK.Final, "final"},
-  {TOK.Finally, "finally"},
-  {TOK.Float, "float"},
-  {TOK.For, "for"},
-  {TOK.Foreach, "foreach"},
-  {TOK.Foreach_reverse, "foreach_reverse"},
-  {TOK.Function, "function"},
-  {TOK.Goto, "goto"},
-  {TOK.Idouble, "idouble"},
-  {TOK.If, "if"},
-  {TOK.Ifloat, "ifloat"},
-  {TOK.Import, "import"},
-  {TOK.In, "in"},
-  {TOK.Inout, "inout"},
-  {TOK.Int, "int"},
-  {TOK.Interface, "interface"},
-  {TOK.Invariant, "invariant"},
-  {TOK.Ireal, "ireal"},
-  {TOK.Is, "is"},
-  {TOK.Lazy, "lazy"},
-  {TOK.Long, "long"},
-  {TOK.Macro, "macro"}, // D2.0
-  {TOK.Mixin, "mixin"},
-  {TOK.Module, "module"},
-  {TOK.New, "new"},
-  {TOK.Null, "null"},
-  {TOK.Out, "out"},
-  {TOK.Override, "override"},
-  {TOK.Package, "package"},
-  {TOK.Pragma, "pragma"},
-  {TOK.Private, "private"},
-  {TOK.Protected, "protected"},
-  {TOK.Public, "public"},
-  {TOK.Real, "real"},
-  {TOK.Ref, "ref"},
-  {TOK.Return, "return"},
-  {TOK.Scope, "scope"},
-  {TOK.Short, "short"},
-  {TOK.Static, "static"},
-  {TOK.Struct, "struct"},
-  {TOK.Super, "super"},
-  {TOK.Switch, "switch"},
-  {TOK.Synchronized, "synchronized"},
-  {TOK.Template, "template"},
-  {TOK.This, "this"},
-  {TOK.Throw, "throw"},
-  {TOK.Traits, "__traits"}, // D2.0
-  {TOK.True, "true"},
-  {TOK.Try, "try"},
-  {TOK.Typedef, "typedef"},
-  {TOK.Typeid, "typeid"},
-  {TOK.Typeof, "typeof"},
-  {TOK.Ubyte, "ubyte"},
-  {TOK.Ucent, "ucent"},
-  {TOK.Uint, "uint"},
-  {TOK.Ulong, "ulong"},
-  {TOK.Union, "union"},
-  {TOK.Unittest, "unittest"},
-  {TOK.Ushort, "ushort"},
-  {TOK.Version, "version"},
-  {TOK.Void, "void"},
-  {TOK.Volatile, "volatile"},
-  {TOK.Wchar, "wchar"},
-  {TOK.While, "while"},
-  {TOK.With, "with"},
+  {"abstract", TOK.Abstract},
+  {"alias", TOK.Alias},
+  {"align", TOK.Align},
+  {"asm", TOK.Asm},
+  {"assert", TOK.Assert},
+  {"auto", TOK.Auto},
+  {"body", TOK.Body},
+  {"bool", TOK.Bool},
+  {"break", TOK.Break},
+  {"byte", TOK.Byte},
+  {"case", TOK.Case},
+  {"cast", TOK.Cast},
+  {"catch", TOK.Catch},
+  {"cdouble", TOK.Cdouble},
+  {"cent", TOK.Cent},
+  {"cfloat", TOK.Cfloat},
+  {"char", TOK.Char},
+  {"class", TOK.Class},
+  {"const", TOK.Const},
+  {"continue", TOK.Continue},
+  {"creal", TOK.Creal},
+  {"dchar", TOK.Dchar},
+  {"debug", TOK.Debug},
+  {"default", TOK.Default},
+  {"delegate", TOK.Delegate},
+  {"delete", TOK.Delete},
+  {"deprecated", TOK.Deprecated},
+  {"do", TOK.Do},
+  {"double", TOK.Double},
+  {"else", TOK.Else},
+  {"enum", TOK.Enum},
+  {"export", TOK.Export},
+  {"extern", TOK.Extern},
+  {"false", TOK.False},
+  {"final", TOK.Final},
+  {"finally", TOK.Finally},
+  {"float", TOK.Float},
+  {"for", TOK.For},
+  {"foreach", TOK.Foreach},
+  {"foreach_reverse", TOK.Foreach_reverse},
+  {"function", TOK.Function},
+  {"goto", TOK.Goto},
+  {"idouble", TOK.Idouble},
+  {"if", TOK.If},
+  {"ifloat", TOK.Ifloat},
+  {"import", TOK.Import},
+  {"in", TOK.In},
+  {"inout", TOK.Inout},
+  {"int", TOK.Int},
+  {"interface", TOK.Interface},
+  {"invariant", TOK.Invariant},
+  {"ireal", TOK.Ireal},
+  {"is", TOK.Is},
+  {"lazy", TOK.Lazy},
+  {"long", TOK.Long},
+  {"macro", TOK.Macro}, // D2.0
+  {"mixin", TOK.Mixin},
+  {"module", TOK.Module},
+  {"new", TOK.New},
+  {"null", TOK.Null},
+  {"out", TOK.Out},
+  {"override", TOK.Override},
+  {"package", TOK.Package},
+  {"pragma", TOK.Pragma},
+  {"private", TOK.Private},
+  {"protected", TOK.Protected},
+  {"public", TOK.Public},
+  {"real", TOK.Real},
+  {"ref", TOK.Ref},
+  {"return", TOK.Return},
+  {"scope", TOK.Scope},
+  {"short", TOK.Short},
+  {"static", TOK.Static},
+  {"struct", TOK.Struct},
+  {"super", TOK.Super},
+  {"switch", TOK.Switch},
+  {"synchronized", TOK.Synchronized},
+  {"template", TOK.Template},
+  {"this", TOK.This},
+  {"throw", TOK.Throw},
+  {"__traits", TOK.Traits}, // D2.0
+  {"true", TOK.True},
+  {"try", TOK.Try},
+  {"typedef", TOK.Typedef},
+  {"typeid", TOK.Typeid},
+  {"typeof", TOK.Typeof},
+  {"ubyte", TOK.Ubyte},
+  {"ucent", TOK.Ucent},
+  {"uint", TOK.Uint},
+  {"ulong", TOK.Ulong},
+  {"union", TOK.Union},
+  {"unittest", TOK.Unittest},
+  {"ushort", TOK.Ushort},
+  {"version", TOK.Version},
+  {"void", TOK.Void},
+  {"volatile", TOK.Volatile},
+  {"wchar", TOK.Wchar},
+  {"while", TOK.While},
+  {"with", TOK.With},
   // Special tokens:
-  {TOK.FILE, "__FILE__"},
-  {TOK.LINE, "__LINE__"},
-  {TOK.DATE, "__DATE__"},
-  {TOK.TIME, "__TIME__"},
-  {TOK.TIMESTAMP, "__TIMESTAMP__"},
-  {TOK.VENDOR, "__VENDOR__"},
-  {TOK.VERSION, "__VERSION__"},
-  {TOK.EOF, "__EOF__"}, // D2.0
+  {"__FILE__", TOK.FILE},
+  {"__LINE__", TOK.LINE},
+  {"__DATE__", TOK.DATE},
+  {"__TIME__", TOK.TIME},
+  {"__TIMESTAMP__", TOK.TIMESTAMP},
+  {"__VENDOR__", TOK.VENDOR},
+  {"__VERSION__", TOK.VERSION},
+  {"__EOF__", TOK.EOF}, // D2.0
 ];
--- a/trunk/src/dil/Lexer.d	Sun Dec 09 23:27:40 2007 +0100
+++ b/trunk/src/dil/Lexer.d	Tue Dec 11 14:19:30 2007 +0100
@@ -10,6 +10,7 @@
 import dil.Messages;
 import dil.HtmlEntities;
 import dil.CompilerInfo;
+import dil.IdTable;
 import tango.stdc.stdlib : strtof, strtod, strtold;
 import tango.stdc.errno : errno, ERANGE;
 import tango.stdc.time : time_t, time, ctime;
@@ -23,31 +24,6 @@
 /// U+FFFD = �. Used to replace invalid Unicode characters.
 const dchar REPLACEMENT_CHAR = '\uFFFD';
 
-/// Global table of identifiers. Access must be synchronized.
-private Identifier*[string] idTable;
-
-static this()
-{
-  foreach(ref k; keywords)
-    idTable[k.str] = &k;
-}
-
-Identifier* idTableLookup(string idString)
-out(id)
-{ assert(id !is null); }
-body
-{
-  synchronized
-  {
-    Identifier** id = idString in idTable;
-    if (id)
-      return *id;
-    auto newID = Identifier(TOK.Identifier, idString);
-    idTable[idString] = newID;
-    return newID;
-  }
-}
-
 /++
   The Lexer analyzes the characters of a source text and
   produces a doubly-linked list of tokens.
@@ -313,8 +289,9 @@
 
         t.end = p;
 
-        auto id = idTableLookup(t.srcText);
+        auto id = IdTable.lookup(t.srcText);
         t.type = id.type;
+        t.ident = id;
 
         if (t.type == TOK.Identifier || t.isKeyword)
           return;
@@ -326,7 +303,7 @@
           assert(t.srcText == "__EOF__");
         }
         else
-          assert(0, "unexpected token: " ~ t.srcText);
+          assert(0, "unexpected token type: " ~ Token.toString(t.type));
         return;
       }
 
@@ -1058,8 +1035,9 @@
 
       t.end = p;
 
-      auto id = idTableLookup(t.srcText);
+      auto id = IdTable.lookup(t.srcText);
       t.type = id.type;
+      t.ident = id;
 
       if (t.type == TOK.Identifier || t.isKeyword)
         return;
@@ -1071,7 +1049,7 @@
         assert(t.srcText == "__EOF__");
       }
       else
-        assert(0, "unexpected token: " ~ t.srcText);
+        assert(0, "unexpected token type: " ~ Token.toString(t.type));
       return;
     }
 
--- a/trunk/src/dil/Parser.d	Sun Dec 09 23:27:40 2007 +0100
+++ b/trunk/src/dil/Parser.d	Tue Dec 11 14:19:30 2007 +0100
@@ -14,10 +14,10 @@
 import dil.Types;
 import dil.Enums;
 import dil.CompilerInfo;
+import dil.IdentsEnum;
+import dil.IdTable;
 import common;
 
-private alias TOK T;
-
 /++
   The Parser produces an abstract syntax tree (AST) by analyzing
   the tokens of the provided source code.
@@ -37,6 +37,8 @@
   StorageClass storageClass;
   uint alignSize = DEFAULT_ALIGN_SIZE;
 
+  private alias TOK T;
+
   this(char[] srcText, string filePath, InformationManager infoMan = null)
   {
     lx = new Lexer(srcText, filePath);
@@ -682,11 +684,13 @@
       return linkageType;
     }
 
-    auto ident = requireId();
-
-    switch (ident ? ident.identifier : null)
+    auto identTok = requireId();
+
+    ID identID = identTok ? identTok.ident.identID : ID.Null;
+
+    switch (identID)
     {
-    case "C":
+    case ID.C:
       if (token.type == T.PlusPlus)
       {
         nT();
@@ -695,16 +699,16 @@
       }
       linkageType = LinkageType.C;
       break;
-    case "D":
+    case ID.D:
       linkageType = LinkageType.D;
       break;
-    case "Windows":
+    case ID.Windows:
       linkageType = LinkageType.Windows;
       break;
-    case "Pascal":
+    case ID.Pascal:
       linkageType = LinkageType.Pascal;
       break;
-    case "System":
+    case ID.System:
       linkageType = LinkageType.System;
       break;
     default:
@@ -2448,14 +2452,13 @@
 
     Token* condition = requireId();
     if (condition)
-      switch (condition.identifier)
+      switch (condition.ident.identID)
       {
-      case "exit":
-      case "success":
-      case "failure":
+      case ID.exit, ID.success, ID.failure:
         break;
       default:
-        // TODO: issue error msg.
+        // TODO: create MID.InvalidScopeIdentifier
+        error(condition, "'exit', 'success', 'failure' are valid scope identifiers, but not '{}';", condition.srcText);
       }
     require(T.RParen);
     Statement scopeBody;
@@ -2724,7 +2727,7 @@
           nT();
         number = null;
         // TODO: report error: number expected after asm align statement.
-        error(MID.ExpectedButFound, "integer", token.srcText);
+        error(token, "expected an integer after align, not '{}'", token.srcText);
       }
       require(T.Semicolon);
       s = new AsmAlignStatement(number);
@@ -2954,23 +2957,23 @@
          T.Float, T.Double, T.Real:
       goto LAsmTypePrefix;
     case T.Identifier:
-      switch (token.identifier)
+      switch (token.ident.identID)
       {
-      case "near", "far",   /*"byte",  "short",  "int",*/
-           "word", "dword", "qword"/*, "float", "double", "real"*/:
+      case ID.near, ID.far,/* "byte",  "short",  "int",*/
+           ID.word, ID.dword, ID.qword/*, "float", "double", "real"*/:
       LAsmTypePrefix:
         nT();
-        if (token.type == T.Identifier && token.identifier == "ptr")
+        if (token.type == T.Identifier && token.ident is Ident.ptr)
           nT();
         else
           error(MID.ExpectedButFound, "ptr", token.srcText);
         e = new AsmTypeExpression(parseAsmExpression());
         break;
-      case "offset":
+      case ID.offset:
         nT();
         e = new AsmOffsetExpression(parseAsmExpression());
         break;
-      case "seg":
+      case ID.seg:
         nT();
         e = new AsmSegExpression(parseAsmExpression());
         break;
@@ -3026,70 +3029,61 @@
       e = new AsmBracketExpression(e);
       break;
     case T.Identifier:
-      switch (token.identifier)
+      auto register = token.ident;
+      switch (register.identID)
       {
       // __LOCAL_SIZE
-      case "__LOCAL_SIZE":
+      case ID.__LOCAL_SIZE:
+        nT();
         e = new AsmLocalSizeExpression();
-        nT();
         break;
       // Register
-      case "ST":
-        auto register = token;
+      case ID.ST:
         nT();
         // (1) - (7)
-        Token* number;
+        int number = -1;
         if (token.type == T.LParen)
         {
           nT();
           if (token.type == T.Int32)
-          {
-            number = token;
-            nT();
-          }
+            (number = token.int_), nT();
           else
             expected(T.Int32);
           require(T.RParen);
         }
         e = new AsmRegisterExpression(register, number);
         break;
-      case "FS":
-        auto register = token;
+      case ID.FS:
         nT();
         // TODO: is the colon-number part optional?
-        Token* number;
+        int number = -1;
         if (token.type == T.Colon)
         {
           // :0, :4, :8
           nT();
-          switch (token.srcText)
-          {
-          case "0", "4", "8":
-            number = token;
-            nT();
-            break;
-          default:
+          if (token.type == T.Int32)
+            (number = token.int_), nT();
+          if (number != 0 && number != 4 && number != 8)
             error(MID.ExpectedButFound, "0, 4 or 8", token.srcText);
-          }
         }
         e = new AsmRegisterExpression(register, number);
         break;
-      case "AL", "AH", "AX", "EAX",
-           "BL", "BH", "BX", "EBX",
-           "CL", "CH", "CX", "ECX",
-           "DL", "DH", "DX", "EDX",
-           "BP", "EBP",
-           "SP", "ESP",
-           "DI", "EDI",
-           "SI", "ESI",
-           "ES", "CS", "SS", "DS", "GS",
-           "CR0", "CR2", "CR3", "CR4",
-           "DR0", "DR1", "DR2", "DR3", "DR6", "DR7",
-           "TR3", "TR4", "TR5", "TR6", "TR7",
-           "MM0", "MM1", "MM2", "MM3", "MM4", "MM5", "MM6", "MM7",
-           "XMM0", "XMM1", "XMM2", "XMM3", "XMM4", "XMM5", "XMM6", "XMM7":
-          e = new AsmRegisterExpression(token);
-          nT();
+      case ID.AL, ID.AH, ID.AX, ID.EAX,
+           ID.BL, ID.BH, ID.BX, ID.EBX,
+           ID.CL, ID.CH, ID.CX, ID.ECX,
+           ID.DL, ID.DH, ID.DX, ID.EDX,
+           ID.BP, ID.EBP, ID.SP, ID.ESP,
+           ID.DI, ID.EDI, ID.SI, ID.ESI,
+           ID.ES, ID.CS, ID.SS, ID.DS, ID.GS,
+           ID.CR0, ID.CR2, ID.CR3, ID.CR4,
+           ID.DR0, ID.DR1, ID.DR2, ID.DR3, ID.DR6, ID.DR7,
+           ID.TR3, ID.TR4, ID.TR5, ID.TR6, ID.TR7,
+           ID.MM0, ID.MM1, ID.MM2, ID.MM3,
+           ID.MM4, ID.MM5, ID.MM6, ID.MM7,
+           ID.XMM0, ID.XMM1, ID.XMM2, ID.XMM3,
+           ID.XMM4, ID.XMM5, ID.XMM6, ID.XMM7:
+        nT();
+        e = new AsmRegisterExpression(register);
         break;
       default:
         // DotIdentifier
@@ -3115,7 +3109,7 @@
       {
         // Insert a dummy token and don't consume current one.
         begin = lx.insertEmptyTokenBefore(token);
-        prevToken = begin;
+        this.prevToken = begin;
       }
     }
     set(e, begin);
@@ -3797,7 +3791,7 @@
       {
         // Insert a dummy token and don't consume current one.
         begin = lx.insertEmptyTokenBefore(token);
-        prevToken = begin;
+        this.prevToken = begin;
       }
     }
     set(e, begin);
--- a/trunk/src/dil/Token.d	Sun Dec 09 23:27:40 2007 +0100
+++ b/trunk/src/dil/Token.d	Tue Dec 11 14:19:30 2007 +0100
@@ -4,11 +4,12 @@
 +/
 module dil.Token;
 import dil.Location;
+import dil.Identifier;
 import tango.stdc.stdlib : malloc, free;
 import tango.core.Exception;
 import common;
 
-public import dil.TokenIDs;
+public import dil.TokensEnum;
 
 /++
   A Token is a sequence of characters formed by the lexical analyzer.
@@ -48,6 +49,7 @@
                       /// doubly linked list. The last token is always '}' or
                       /// EOF in case end of source text is "q{" EOF.
     }
+    Identifier* ident;
     dchar  dchar_;
     long   long_;
     ulong  ulong_;
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/trunk/src/dil/TokensEnum.d	Tue Dec 11 14:19:30 2007 +0100
@@ -0,0 +1,203 @@
+/++
+  Author: Aziz Köksal
+  License: GPL3
++/
+module dil.TokensEnum;
+import common;
+
+enum TOK : ushort
+{
+  Invalid,
+
+  /// Flag for whitespace tokens that must be ignored in the parsing phase.
+  Whitespace = 0x8000,
+  Illegal  = 1 | Whitespace,
+  Comment  = 2 | Whitespace,
+  Shebang  = 3 | Whitespace,
+  HashLine = 4 | Whitespace,
+  Filespec = 5 | Whitespace,
+  Newline  = 6 | Whitespace,
+  Empty    = 7,
+
+  Identifier = 8,
+  String,
+  CharLiteral, WCharLiteral, DCharLiteral,
+
+  // Special tokens
+  FILE,
+  LINE,
+  DATE,
+  TIME,
+  TIMESTAMP,
+  VENDOR,
+  VERSION,
+
+  // Number literals
+  Int32, Int64, Uint32, Uint64,
+  // Floating point number scanner relies on this order. (FloatXY + 3 == ImaginaryXY)
+  Float32, Float64, Float80,
+  Imaginary32, Imaginary64, Imaginary80,
+
+
+  // Brackets
+  LParen,
+  RParen,
+  LBracket,
+  RBracket,
+  LBrace,
+  RBrace,
+
+  Dot, Slice, Ellipses,
+
+  // Floating point number operators
+  Unordered,
+  UorE,
+  UorG,
+  UorGorE,
+  UorL,
+  UorLorE,
+  LorEorG,
+  LorG,
+
+  // Normal operators
+  Assign, Equal, NotEqual, Not,
+  LessEqual, Less,
+  GreaterEqual, Greater,
+  LShiftAssign, LShift,
+  RShiftAssign,RShift,
+  URShiftAssign, URShift,
+  OrAssign, OrLogical, OrBinary,
+  AndAssign, AndLogical, AndBinary,
+  PlusAssign, PlusPlus, Plus,
+  MinusAssign, MinusMinus, Minus,
+  DivAssign, Div,
+  MulAssign, Mul,
+  ModAssign, Mod,
+  XorAssign, Xor,
+  CatAssign, Catenate,
+  Tilde,
+  Identity, NotIdentity,
+
+  Colon,
+  Semicolon,
+  Question,
+  Comma,
+  Dollar,
+
+  /* Keywords:
+     NB.: Token.isKeyword() depends on this list being contiguous.
+  */
+  Abstract,Alias,Align,Asm,Assert,Auto,Body,
+  Bool,Break,Byte,Case,Cast,Catch,Cdouble,
+  Cent,Cfloat,Char,Class,Const,Continue,Creal,
+  Dchar,Debug,Default,Delegate,Delete,Deprecated,Do,
+  Double,Else,Enum,Export,Extern,False,Final,
+  Finally,Float,For,Foreach,Foreach_reverse,Function,Goto,
+  Idouble,If,Ifloat,Import,In,Inout,Int,
+  Interface,Invariant,Ireal,Is,Lazy,Long,Macro/+D2.0+/,
+  Mixin,Module,New,Null,Out,Override,Package,
+  Pragma,Private,Protected,Public,Real,Ref/+D2.0+/,Return,
+  Scope,Short,Static,Struct,Super,Switch,Synchronized,
+  Template,This,Throw,Traits/+D2.0+/,True,Try,Typedef,Typeid,
+  Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest,
+  Ushort,Version,Void,Volatile,Wchar,While,With,
+
+  HEAD, // start of linked list
+  EOF,
+  MAX
+}
+
+alias TOK.Abstract KeywordsBegin;
+alias TOK.With KeywordsEnd;
+alias TOK.FILE SpecialTokensBegin;
+alias TOK.VERSION SpecialTokensEnd;
+
+/// A table mapping each TOK to a string.
+const string[] tokToString = [
+  "Invalid",
+
+  "Illegal",
+  "Comment",
+  "#! /shebang/",
+  "#line",
+  `"filespec"`,
+  "Newline",
+  "Empty",
+
+  "Identifier",
+  "String",
+  "CharLiteral", "WCharLiteral", "DCharLiteral",
+
+  "__FILE__",
+  "__LINE__",
+  "__DATE__",
+  "__TIME__",
+  "__TIMESTAMP__",
+  "__VENDOR__",
+  "__VERSION__",
+
+  "Int32", "Int64", "Uint32", "Uint64",
+  "Float32", "Float64", "Float80",
+  "Imaginary32", "Imaginary64", "Imaginary80",
+
+  "(",
+  ")",
+  "[",
+  "]",
+  "{",
+  "}",
+
+  ".", "..", "...",
+
+  "!<>=", // Unordered
+  "!<>",  // UorE
+  "!<=",  // UorG
+  "!<",   // UorGorE
+  "!>=",  // UorL
+  "!>",   // UorLorE
+  "<>=",  // LorEorG
+  "<>",   // LorG
+
+  "=", "==", "!=", "!",
+  "<=", "<",
+  ">=", ">",
+  "<<=", "<<",
+  ">>=",">>",
+  ">>>=", ">>>",
+  "|=", "||", "|",
+  "&=", "&&", "&",
+  "+=", "++", "+",
+  "-=", "--", "-",
+  "/=", "/",
+  "*=", "*",
+  "%=", "%",
+  "^=", "^",
+  "~=", "~",
+  "~",
+  "is", "!is",
+
+  ":",
+  ";",
+  "?",
+  ",",
+  "$",
+
+  "abstract","alias","align","asm","assert","auto","body",
+  "bool","break","byte","case","cast","catch","cdouble",
+  "cent","cfloat","char","class","const","continue","creal",
+  "dchar","debug","default","delegate","delete","deprecated","do",
+  "double","else","enum","export","extern","false","final",
+  "finally","float","for","foreach","foreach_reverse","function","goto",
+  "idouble","if","ifloat","import","in","inout","int",
+  "interface","invariant","ireal","is","lazy","long","macro",
+  "mixin","module","new","null","out","override","package",
+  "pragma","private","protected","public","real","ref","return",
+  "scope","short","static","struct","super","switch","synchronized",
+  "template","this","throw","__traits","true","try","typedef","typeid",
+  "typeof","ubyte","ucent","uint","ulong","union","unittest",
+  "ushort","version","void","volatile","wchar","while","with",
+
+  "HEAD",
+  "EOF"
+];
+static assert(tokToString.length == TOK.EOF+1);