--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lexer/Token.d	Fri Apr 18 02:01:38 2008 +0200
@@ -0,0 +1,98 @@
+module lexer.Token;
+
+public 
+import misc.Location;
+
+import Integer = tango.text.convert.Integer;
+
+struct Token
+{
+    Tok type;
+    Location location;
+    uint length;
+
+    static Token opCall (Tok type, Location location, uint length)
+    {
+        Token t;
+        t.type = type;
+        t.location = location;
+        t.length = length;
+        return t;
+    }
+
+    char[] getType ()
+    {
+        return typeToString[this.type];
+    }
+
+    char[] toString ()
+    {
+        return this.getType()~": Len: "~Integer.toString(this.length)
+            ~", Loc: "~location.toString;
+    }
+
+    char[] get ()
+    {
+        return location.get(length);
+    }
+}
+
+enum Tok : ushort
+{
+    /* Non-code related tokens */
+    EOF,
+
+    /* Basic types */
+    Identifier,
+    Integer,
+
+    /* Basic operators */
+    Assign,
+    Add, Sub,
+    Mul, Div,
+    Comma,
+
+    /* Symbols */
+    OpenParentheses,
+    CloseParentheses,
+    OpenBrace,
+    CloseBrace,
+    Seperator,
+
+    /* Keywords */
+    Byte, Ubyte,
+    Short, Ushort,
+    Int, Uint,
+    Long, Ulong,
+
+    Float, Double,
+
+    Return,
+
+}
+
+public char[][Tok] typeToString;
+
+static this()
+{
+    typeToString =
+    [
+        Tok.EOF:"EOF"[],
+        Tok.Identifier:"Identifier",
+        Tok.Byte:"Byte",
+        Tok.Short:"Short",
+        Tok.Int:"Int",
+        Tok.Long:"Long",
+        Tok.OpenParentheses:"OpenParentheses",
+        Tok.CloseParentheses:"CloseParentheses",
+        Tok.OpenBrace:"OpenBrace",
+        Tok.CloseBrace:"CloseBrace",
+        Tok.Assign:"Assign",
+        Tok.Add:"Add",
+        Tok.Sub:"Sub",
+        Tok.Mul:"Mul",
+        Tok.Div:"Div",
+        Tok.Integer:"Interger",
+        Tok.Seperator:"Seperator"
+    ];
+}