diff lexer/Token.d @ 189:75d0544ddc45

Better error handling on unexpected EOF.
author Anders Johnsen <skabet@gmail.com>
date Fri, 25 Jul 2008 13:50:01 +0200
parents dc9bf56b7ace
children 08f68d684047
line wrap: on
line diff
--- a/lexer/Token.d	Fri Jul 25 12:55:38 2008 +0200
+++ b/lexer/Token.d	Fri Jul 25 13:50:01 2008 +0200
@@ -1,7 +1,8 @@
 module lexer.Token;
 
 public 
-import basic.SourceLocation;
+import basic.SourceLocation,
+       basic.SourceManager;
 
 import Integer = tango.text.convert.Integer;
 
@@ -34,8 +35,10 @@
     /**
       Get the type of the Token as a string
       */
-    char[] getType ()
+    char[] get (SourceManager sm)
     {
+        if (isIdentifier)
+            return sm.getText(asRange);
         return typeToString[this.type];
     }
 
@@ -44,7 +47,7 @@
       */
     char[] toString ()
     {
-        return this.getType()~": Len: "~Integer.toString(this.length);
+        return typeToString[this.type];
     }
 
     /// Get the range of this token
@@ -247,77 +250,77 @@
     typeToString =
     [
         Tok.EOF:"EOF"[],
-        Tok.Identifier:"Identifier",
-        Tok.Byte:"Byte",
-        Tok.Short:"Short",
-        Tok.Int:"Int",
-        Tok.Long:"Long",
-        Tok.Char:"Char",
-        Tok.Wchar:"Wchar",
-        Tok.Dchar:"Dchar",
-        Tok.Bool:"Bool",
-        Tok.Void:"Void",
-        Tok.Function:"Function",
-        Tok.Eq:"Eq",
-        Tok.Ne:"Ne",
-        Tok.Lt:"Lt",
-        Tok.Le:"Le",
-        Tok.Gt:"Gt",
-        Tok.Ge:"Ge",
-        Tok.OpenParentheses:"OpenParentheses",
-        Tok.CloseParentheses:"CloseParentheses",
-        Tok.OpenBrace:"OpenBrace",
-        Tok.CloseBrace:"CloseBrace",
-        Tok.OpenBracket:"OpenBracket",
-        Tok.CloseBracket:"CloseBracket",
-        Tok.Dot:"Dot",
-        Tok.Assign:"Assign",
-        Tok.Plus:"Plus",
-        Tok.PlusAssign:"PlusAssign",
-        Tok.Minus:"Minus",
-        Tok.MinusAssign:"MinusAssign",
-        Tok.Star:"Star",
-        Tok.StarAssign:"StarAssign",
-        Tok.Slash:"Slash",
-        Tok.SlashAssign:"SlashAssign",
-        Tok.Percent:"Percent",
-        Tok.PercentAssign:"PercentAssign",
-        Tok.LeftShift:"LeftShift",
-        Tok.RightShift:"RightShift",
-        Tok.UnsignedRightShift:"UnsignedRightShift",
-        Tok.Integer:"Integer",
-        Tok.If:"If",
-        Tok.While:"While",
-        Tok.For:"For",
-        Tok.Switch:"Switch",
-        Tok.Case:"Case",
-        Tok.Default:"Default",
-        Tok.Comma:"Comma",
-        Tok.Return:"Return",
-        Tok.Struct:"Struct",
-        Tok.Class:"Class",
-        Tok.This:"This",
-        Tok.Colon:"Colon",
-        Tok.Seperator:"Seperator",
-        Tok.Cast:"Cast",
-        Tok.Module:"Module",
-        Tok.Import:"Import",
+        Tok.Identifier:"identifier",
+        Tok.Byte:"byte",
+        Tok.Short:"short",
+        Tok.Int:"int",
+        Tok.Long:"long",
+        Tok.Char:"char",
+        Tok.Wchar:"wchar",
+        Tok.Dchar:"dchar",
+        Tok.Bool:"bool",
+        Tok.Void:"void",
+        Tok.Function:"function",
+        Tok.Eq:"==",
+        Tok.Ne:"!=",
+        Tok.Lt:"<",
+        Tok.Le:"<=",
+        Tok.Gt:">",
+        Tok.Ge:">=",
+        Tok.OpenParentheses:"(",
+        Tok.CloseParentheses:")",
+        Tok.OpenBrace:"{",
+        Tok.CloseBrace:"}",
+        Tok.OpenBracket:"[",
+        Tok.CloseBracket:"]",
+        Tok.Dot:"-",
+        Tok.Assign:"=",
+        Tok.Plus:"+",
+        Tok.PlusAssign:"+=",
+        Tok.Minus:"-",
+        Tok.MinusAssign:"-=",
+        Tok.Star:"*",
+        Tok.StarAssign:"*=",
+        Tok.Slash:"/",
+        Tok.SlashAssign:"/=",
+        Tok.Percent:"%",
+        Tok.PercentAssign:"%=",
+        Tok.LeftShift:"<<",
+        Tok.RightShift:">>",
+        Tok.UnsignedRightShift:">>>",
+        Tok.Integer:"int",
+        Tok.If:"if",
+        Tok.While:"while",
+        Tok.For:"for",
+        Tok.Switch:"switch",
+        Tok.Case:"case",
+        Tok.Default:"default",
+        Tok.Comma:",",
+        Tok.Return:"return",
+        Tok.Struct:"struct",
+        Tok.Class:"class",
+        Tok.This:"this",
+        Tok.Colon:":",
+        Tok.Seperator:";",
+        Tok.And:"&",
+        Tok.Cast:"cast",
+        Tok.Module:"module",
+        Tok.Import:"import",
         Tok.String:"String",
-        Tok.Public:"Public",
-        Tok.Private:"Private",
-        Tok.Protected:"Protected",
-        Tok.Package:"Package",
-        Tok.Export:"Export",
-        Tok.Static:"Static",
-        Tok.Final:"Finale",
-        Tok.Public:"Public",
-        Tok.Const:"Const",
-        Tok.Abstract:"Abstract",
-        Tok.Override:"Override",
-        Tok.Deprecated:"Deprecated",
-        Tok.Auto:"Auto",
-        Tok.Extern:"Extern",
-        Tok.New:"New",
-        Tok.And:"And"
+        Tok.Public:"public",
+        Tok.Private:"private",
+        Tok.Protected:"protected",
+        Tok.Package:"package",
+        Tok.Export:"export",
+        Tok.Static:"static",
+        Tok.Final:"finale",
+        Tok.Public:"public",
+        Tok.Const:"const",
+        Tok.Abstract:"abstract",
+        Tok.Override:"override",
+        Tok.Deprecated:"deprecated",
+        Tok.Auto:"auto",
+        Tok.Extern:"extern",
+        Tok.New:"new"
     ];
 }