changeset 499:52447db67938

Implemented global table of identifiers.
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Sun, 09 Dec 2007 22:37:47 +0100
parents 49c201b5c465
children 41b7f9e439bd
files trunk/src/dil/Identifier.d trunk/src/dil/Lexer.d
diffstat 2 files changed, 36 insertions(+), 24 deletions(-) [+]
line wrap: on
line diff
--- a/trunk/src/dil/Identifier.d	Sun Dec 09 19:37:32 2007 +0100
+++ b/trunk/src/dil/Identifier.d	Sun Dec 09 22:37:47 2007 +0100
@@ -11,9 +11,9 @@
   TOK type;
   string str;
 
-  static Identifier opCall(TOK type, string str)
+  static Identifier* opCall(TOK type, string str)
   {
-    Identifier i;
+    auto i = new Identifier;
     i.type = type;
     i.str = str;
     return i;
--- a/trunk/src/dil/Lexer.d	Sun Dec 09 19:37:32 2007 +0100
+++ b/trunk/src/dil/Lexer.d	Sun Dec 09 22:37:47 2007 +0100
@@ -23,6 +23,35 @@
 /// U+FFFD = �. Used to replace invalid Unicode characters.
 const dchar REPLACEMENT_CHAR = '\uFFFD';
 
+/// Global table of identifiers. Access must be synchronized.
+private Identifier*[string] idTable;
+
+static this()
+{
+  foreach(ref k; keywords)
+    idTable[k.str] = &k;
+}
+
+Identifier* idTableLookup(string idString)
+out(id)
+{ assert(id !is null); }
+body
+{
+  synchronized
+  {
+    Identifier** id = idString in idTable;
+    if (id)
+      return *id;
+    auto newID = Identifier(TOK.Identifier, idString);
+    idTable[idString] = newID;
+    return newID;
+  }
+}
+
+/++
+  The Lexer analyzes the characters of a source text and
+  produces a doubly-linked list of tokens.
++/
 class Lexer
 {
   Token* head;      /// The head of the doubly linked token list.
@@ -43,8 +72,6 @@
   uint inTokenString; /// > 0 if inside q{ }
   char[] errorPath;   /// The path displayed in error messages.
 
-  Identifier[string] idtable;
-
   /++
     Construct a Lexer object.
     Params:
@@ -65,7 +92,6 @@
     this.p = this.text.ptr;
     this.end = this.p + this.text.length;
     this.lineBegin = this.p;
-    loadKeywords(this.idtable);
 
     this.head = new Token;
     this.head.type = TOK.HEAD;
@@ -287,16 +313,9 @@
 
         t.end = p;
 
-        string str = t.srcText;
-        Identifier* id = str in idtable;
+        auto id = idTableLookup(t.srcText);
+        t.type = id.type;
 
-        if (!id)
-        {
-          idtable[str] = Identifier(TOK.Identifier, str);
-          id = str in idtable;
-        }
-        assert(id);
-        t.type = id.type;
         if (t.type == TOK.Identifier || t.isKeyword)
           return;
         else if (t.isSpecialToken)
@@ -1039,16 +1058,9 @@
 
       t.end = p;
 
-      string str = t.srcText;
-      Identifier* id = str in idtable;
+      auto id = idTableLookup(t.srcText);
+      t.type = id.type;
 
-      if (!id)
-      {
-        idtable[str] = Identifier(TOK.Identifier, str);
-        id = str in idtable;
-      }
-      assert(id);
-      t.type = id.type;
       if (t.type == TOK.Identifier || t.isKeyword)
         return;
       else if (t.isSpecialToken)
@@ -2446,7 +2458,7 @@
 
     static Identifier[string] reserved_ids_table;
     if (reserved_ids_table is null)
-      loadKeywords(reserved_ids_table);
+      Lexer.loadKeywords(reserved_ids_table);
 
     size_t idx = 1; // Index to the 2nd character in ident.
     dchar isFirstCharUniAlpha()