Mercurial > projects > dang
view lexer/Token.d @ 194:08f68d684047
Rename some files. Hopefully we can get a more iterative sema pass, that's a lot easier to "get startet with".
Also added support for alias.
author | Anders Johnsen <skabet@gmail.com> |
---|---|
date | Tue, 29 Jul 2008 13:54:44 +0200 |
parents | 75d0544ddc45 |
children |
line wrap: on
line source
module lexer.Token; public import basic.SourceLocation, basic.SourceManager; import Integer = tango.text.convert.Integer; /** The Token struct will be used through the Lexer, Parser and other modules as a location into source. The Token should always be optimized for size to limit unnecessary memory usage. */ struct Token { Tok type; SLoc location; uint length; /** Create a new token with a Tok type, Location in source and a length of how many chars the Token span in the source */ static Token opCall (Tok type, SLoc location, uint length) { Token t; t.type = type; t.location = location; t.length = length; return t; } /** Get the type of the Token as a string */ char[] get (SourceManager sm) { if (isIdentifier) return sm.getText(asRange); return typeToString[this.type]; } /** A human readable dump of a Token */ char[] toString () { return typeToString[this.type]; } /// Get the range of this token SourceRange asRange() { return SourceRange(location, location + length); } /** Returns true if the type of this token is a basic type (int, float, ...). Void is included, although a void in it self is not really a type. */ bool isBasicType() { return type >= Tok.Byte && type <= Tok.Void; } /** Returns true for all the various assignments (=, +=, *= ...) */ bool isAssignment() { return type >= Tok.Assign && type <= Tok.PercentAssign; } /** Returns true for all attributes( public, static, private...) */ bool isAttribute() { return type >= Tok.Public && type <= Tok.Extern; } /** Returns true for all attributes( public, static, private...) */ bool isBaseClassProtection() { return type >= Tok.Public && type <= Tok.Export; } /** just a shortcut to avoid `token.type == tok.Switch`. */ bool isSwitch() { return type == Tok.Switch; } /** just a shortcut to avoid `token.type == tok.While`. */ bool isWhile() { return type == Tok.While; } /** just a shortcut to avoid `token.type == tok.For`. */ bool isFor() { return type == Tok.For; } /** just a shortcut to avoid `token.type == tok.If`. */ bool isIf() { return type == Tok.If; } /** just a shortcut to avoid `token.type == tok.Return`. */ bool isReturn() { return type == Tok.Return; } /** Just a shortcut to avoid `token.type == Tok.Identifier`. */ bool isIdentifier() { return type == Tok.Identifier; } } /** Tok is short for TokenType. This enum list is to supply the Token with a type. This enum is used to switch over "many" places. */ enum Tok : ushort { /* Non-code related tokens */ EOF, /* Basic types */ Identifier, Integer, String, /* Basic operators */ Assign, PlusAssign, MinusAssign, StarAssign, SlashAssign, PercentAssign, Plus, Minus, Star, Slash, Percent, LeftShift, RightShift, UnsignedRightShift, Comma, And, /* Symbols */ OpenParentheses, CloseParentheses, OpenBrace, CloseBrace, OpenBracket, CloseBracket, Seperator, Colon, Dot, /* Comparator operators */ Eq, Ne, Lt, Gt, Le, Ge, Not, /* Keywords */ Byte, Ubyte, Short, Ushort, Int, Uint, Long, Ulong, Char, Wchar, Dchar, Float, Double, Bool, Void, Struct, Function, Delegate, Class, This, Interface, Union, Typedef, Typeid, Typeof, Sizeof, Alias, If, Else, While, For, Switch, Case, Default, Break, Return, Cast, Module, Import, New, Null, /* Attributes */ Public, Private, Package, Export, Protected, Static, Final, Const, Abstract, Override, Deprecated, Auto, Extern, Align, Asm, In, Out, Body, Assert, Throw, Try, Catch, Finally, } /** An associative array to supply a Tok to String function. Keep always this list updated when adding a new Tok. */ public char[][Tok] typeToString; static this() { typeToString = [ Tok.EOF:"EOF"[], Tok.Identifier:"identifier", Tok.Byte:"byte", Tok.Short:"short", Tok.Int:"int", Tok.Long:"long", Tok.Char:"char", Tok.Wchar:"wchar", Tok.Dchar:"dchar", Tok.Bool:"bool", Tok.Void:"void", Tok.Function:"function", Tok.Eq:"==", Tok.Ne:"!=", Tok.Lt:"<", Tok.Le:"<=", Tok.Gt:">", Tok.Ge:">=", Tok.OpenParentheses:"(", Tok.CloseParentheses:")", Tok.OpenBrace:"{", Tok.CloseBrace:"}", Tok.OpenBracket:"[", Tok.CloseBracket:"]", Tok.Dot:"-", Tok.Assign:"=", Tok.Plus:"+", Tok.PlusAssign:"+=", Tok.Minus:"-", Tok.MinusAssign:"-=", Tok.Star:"*", Tok.StarAssign:"*=", Tok.Slash:"/", Tok.SlashAssign:"/=", Tok.Percent:"%", Tok.PercentAssign:"%=", Tok.LeftShift:"<<", Tok.RightShift:">>", Tok.UnsignedRightShift:">>>", Tok.Integer:"int", Tok.If:"if", Tok.While:"while", Tok.For:"for", Tok.Switch:"switch", Tok.Case:"case", Tok.Default:"default", Tok.Comma:",", Tok.Return:"return", Tok.Struct:"struct", Tok.Class:"class", Tok.This:"this", Tok.Colon:":", Tok.Seperator:";", Tok.And:"&", Tok.Cast:"cast", Tok.Module:"module", Tok.Import:"import", Tok.String:"String", Tok.Public:"public", Tok.Private:"private", Tok.Protected:"protected", Tok.Package:"package", Tok.Export:"export", Tok.Static:"static", Tok.Final:"finale", Tok.Public:"public", Tok.Const:"const", Tok.Abstract:"abstract", Tok.Override:"override", Tok.Deprecated:"deprecated", Tok.Auto:"auto", Tok.Extern:"extern", Tok.New:"new", Tok.Null:"null", Tok.Alias:"alias" ]; }