Mercurial > projects > dil

changeset 28:3a9daccf7d96

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
- Added table for identifiers to Lexer. - Added keywords table. - Added keywords to TOK.
author aziz
date Sun, 24 Jun 2007 17:19:03 +0000
parents 43b6bf56f0e9
children ef83eea26bbd
files trunk/src/Keywords.d trunk/src/Lexer.d trunk/src/Token.d trunk/src/format.css trunk/src/main.d
diffstat 5 files changed, 172 insertions(+), 3 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/trunk/src/Keywords.d	Sun Jun 24 17:19:03 2007 +0000
@@ -0,0 +1,108 @@
+/++
+  Author: Aziz Köksal
+  License: GPL2
++/
+module Keywords;
+import Token;
+import Identifier;
+
+static const Identifier[] keywords = [
+  {TOK.Abstract, "abstract"},
+  {TOK.Alias, "alias"},
+  {TOK.Align, "align"},
+  {TOK.Asm, "asm"},
+  {TOK.Assert, "assert"},
+  {TOK.Auto, "auto"},
+  {TOK.Body, "body"},
+  {TOK.Bool, "bool"},
+  {TOK.Break, "break"},
+  {TOK.Byte, "byte"},
+  {TOK.Case, "case"},
+  {TOK.Cast, "cast"},
+  {TOK.Catch, "catch"},
+  {TOK.Cdouble, "cdouble"},
+  {TOK.Cent, "cent"},
+  {TOK.Cfloat, "cfloat"},
+  {TOK.Char, "char"},
+  {TOK.Class, "class"},
+  {TOK.Const, "const"},
+  {TOK.Continue, "continue"},
+  {TOK.Creal, "creal"},
+  {TOK.Dchar, "dchar"},
+  {TOK.Debug, "debug"},
+  {TOK.Default, "default"},
+  {TOK.Delegate, "delegate"},
+  {TOK.Delete, "delete"},
+  {TOK.Deprecated, "deprecated"},
+  {TOK.Do, "do"},
+  {TOK.Double, "double"},
+  {TOK.Else, "else"},
+  {TOK.Enum, "enum"},
+  {TOK.Export, "export"},
+  {TOK.Extern, "extern"},
+  {TOK.False, "false"},
+  {TOK.Final, "final"},
+  {TOK.Finally, "finally"},
+  {TOK.Float, "float"},
+  {TOK.For, "for"},
+  {TOK.Foreach, "foreach"},
+  {TOK.Foreach_reverse, "foreach_reverse"},
+  {TOK.Function, "function"},
+  {TOK.Goto, "goto"},
+  {TOK.Idouble, "idouble"},
+  {TOK.If, "if"},
+  {TOK.Ifloat, "ifloat"},
+  {TOK.Import, "import"},
+  {TOK.In, "in"},
+  {TOK.Inout, "inout"},
+  {TOK.Int, "int"},
+  {TOK.Interface, "interface"},
+  {TOK.Invariant, "invariant"},
+  {TOK.Ireal, "ireal"},
+  {TOK.Is, "is"},
+  {TOK.Lazy, "lazy"},
+  {TOK.Long, "long"},
+  {TOK.Macro, "macro"},
+  {TOK.Mixin, "mixin"},
+  {TOK.Module, "module"},
+  {TOK.New, "new"},
+  {TOK.Null, "null"},
+  {TOK.Out, "out"},
+  {TOK.Override, "override"},
+  {TOK.Package, "package"},
+  {TOK.Pragma, "pragma"},
+  {TOK.Private, "private"},
+  {TOK.Protected, "protected"},
+  {TOK.Public, "public"},
+  {TOK.Real, "real"},
+  {TOK.Ref, "ref"},
+  {TOK.Return, "return"},
+  {TOK.Scope, "scope"},
+  {TOK.Short, "short"},
+  {TOK.Static, "static"},
+  {TOK.Struct, "struct"},
+  {TOK.Super, "super"},
+  {TOK.Switch, "switch"},
+  {TOK.Synchronized, "synchronized"},
+  {TOK.Template, "template"},
+  {TOK.This, "this"},
+  {TOK.Throw, "throw"},
+  {TOK.True, "true"},
+  {TOK.Try, "try"},
+  {TOK.Typedef, "typedef"},
+  {TOK.Typeid, "typeid"},
+  {TOK.Typeof, "typeof"},
+  {TOK.Ubyte, "ubyte"},
+  {TOK.Ucent, "ucent"},
+  {TOK.Uint, "uint"},
+  {TOK.Ulong, "ulong"},
+  {TOK.Union, "union"},
+  {TOK.Unittest, "unittest"},
+  {TOK.Ushort, "ushort"},
+  {TOK.Version, "version"},
+  {TOK.Void, "void"},
+  {TOK.Volatile, "volatile"},
+  {TOK.Wchar, "wchar"},
+  {TOK.While, "while"},
+  {TOK.With, "with"}
+];
--- a/trunk/src/Lexer.d	Sun Jun 24 15:02:02 2007 +0000
+++ b/trunk/src/Lexer.d	Sun Jun 24 17:19:03 2007 +0000
@@ -4,6 +4,8 @@
 +/
 module Lexer;
 import Token;
+import Keywords;
+import Identifier;
 import std.stdio;
 import std.utf;
 import std.uni;
@@ -128,6 +130,8 @@
 
   Problem[] errors;
 
+  Identifier[string] idtable;
+
   this(char[] text)
   {
     this.text = text;
@@ -136,6 +140,8 @@
 
     this.p = this.text.ptr;
     this.end = this.p + this.text.length;
+
+    loadKeywords();
   }
 
   public void scan(out Token t)
@@ -176,8 +182,19 @@
         do
         { c = *++p; }
         while (isident(c) || c & 128 && isUniAlpha(decodeUTF()))
-        t.type = TOK.Identifier;
+
         t.end = p;
+
+        string str = t.span;
+        Identifier* id = str in idtable;
+
+        if (!id)
+        {
+          idtable[str] = Identifier.Identifier(TOK.Identifier, str);
+          id = str in idtable;
+        }
+        assert(id);
+        t.type = id.type;
         return;
       }
 
@@ -433,6 +450,12 @@
     return d;
   }
 
+  void loadKeywords()
+  {
+    foreach(k; keywords)
+      idtable[k.str] = k;
+  }
+
   void error(MID id)
   {
     errors ~= new Problem(Problem.Type.Lexer, id, loc);
--- a/trunk/src/Token.d	Sun Jun 24 15:02:02 2007 +0000
+++ b/trunk/src/Token.d	Sun Jun 24 17:19:03 2007 +0000
@@ -42,9 +42,30 @@
   Comma,
   Dollar,
 
+  /* Keywords:
+     NB.: Token.isKeyword() depends on this list being contiguous.
+  */
+  Abstract,Alias,Align,Asm,Assert,Auto,Body,
+  Bool,Break,Byte,Case,Cast,Catch,Cdouble,
+  Cent,Cfloat,Char,Class,Const,Continue,Creal,
+  Dchar,Debug,Default,Delegate,Delete,Deprecated,Do,
+  Double,Else,Enum,Export,Extern,False,Final,
+  Finally,Float,For,Foreach,Foreach_reverse,Function,Goto,
+  Idouble,If,Ifloat,Import,In,Inout,Int,
+  Interface,Invariant,Ireal,Is,Lazy,Long,Macro,
+  Mixin,Module,New,Null,Out,Override,Package,
+  Pragma,Private,Protected,Public,Real,Ref,Return,
+  Scope,Short,Static,Struct,Super,Switch,Synchronized,
+  Template,This,Throw,True,Try,Typedef,Typeid,
+  Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest,
+  Ushort,Version,Void,Volatile,Wchar,While,With,
+
   EOF
 }
 
+alias TOK.Abstract KeywordsBegin;
+alias TOK.With KeywordsEnd;
+
 struct Token
 {
   TOK type;
@@ -60,4 +81,16 @@
     float f;
     double d;
   }
+
+  string span()
+  {
+    return start[0 .. end - start];
+  }
+
+  bool isKeyword()
+  {
+    if (KeywordsBegin <= type && type <= KeywordsEnd)
+      return true;
+    return false;
+  }
 }
\ No newline at end of file
--- a/trunk/src/format.css	Sun Jun 24 15:02:02 2007 +0000
+++ b/trunk/src/format.css	Sun Jun 24 17:19:03 2007 +0000
@@ -4,8 +4,10 @@
   font-family: Monospace;
   font-size: 0.8em;
 }
+k { color: darkblue; font-weight: bold; }
 c { color: green; }
 i { color: darkblue; }
 sl { color: red; }
 cl { color: purple; }
 op { color: orange; }
+br { color: orange; }
--- a/trunk/src/main.d	Sun Jun 24 15:02:02 2007 +0000
+++ b/trunk/src/main.d	Sun Jun 24 17:19:03 2007 +0000
@@ -37,7 +37,7 @@
   {
     if (end != token.start)
       writef("%s", xmlescape(end[0 .. token.start - end]));
-    char[] span = xmlescape(token.start[0 .. token.end-token.start]);
+    string span = xmlescape(token.span);
     switch(token.type)
     {
       case TOK.Identifier:
@@ -64,7 +64,10 @@
       break;
       case TOK.EOF: break;
       default:
-        writef("%s", span);
+        if (token.isKeyword())
+          writef("<k>%s</k>", span);
+        else
+          writef("%s", span);
     }
     end = token.end;
   }