Mercurial > projects > dang
view lexer/Token.d @ 185:7b274cfdc1dc
Added support for array literals. Codegen is broken, though.
author | Anders Johnsen <skabet@gmail.com> |
---|---|
date | Fri, 25 Jul 2008 12:18:05 +0200 |
parents | dc9bf56b7ace |
children | 75d0544ddc45 |
line wrap: on
line source
module lexer.Token; public import basic.SourceLocation; import Integer = tango.text.convert.Integer; /** The Token struct will be used through the Lexer, Parser and other modules as a location into source. The Token should always be optimized for size to limit unnecessary memory usage. */ struct Token { Tok type; SLoc location; uint length; /** Create a new token with a Tok type, Location in source and a length of how many chars the Token span in the source */ static Token opCall (Tok type, SLoc location, uint length) { Token t; t.type = type; t.location = location; t.length = length; return t; } /** Get the type of the Token as a string */ char[] getType () { return typeToString[this.type]; } /** A human readable dump of a Token */ char[] toString () { return this.getType()~": Len: "~Integer.toString(this.length); } /// Get the range of this token SourceRange asRange() { return SourceRange(location, location + length); } /** Returns true if the type of this token is a basic type (int, float, ...). Void is included, although a void in it self is not really a type. */ bool isBasicType() { return type >= Tok.Byte && type <= Tok.Void; } /** Returns true for all the various assignments (=, +=, *= ...) */ bool isAssignment() { return type >= Tok.Assign && type <= Tok.PercentAssign; } /** Returns true for all attributes( public, static, private...) */ bool isAttribute() { return type >= Tok.Public && type <= Tok.Extern; } /** Returns true for all attributes( public, static, private...) */ bool isBaseClassProtection() { return type >= Tok.Public && type <= Tok.Export; } /** just a shortcut to avoid `token.type == tok.Switch`. */ bool isSwitch() { return type == Tok.Switch; } /** just a shortcut to avoid `token.type == tok.While`. */ bool isWhile() { return type == Tok.While; } /** just a shortcut to avoid `token.type == tok.For`. */ bool isFor() { return type == Tok.For; } /** just a shortcut to avoid `token.type == tok.If`. */ bool isIf() { return type == Tok.If; } /** just a shortcut to avoid `token.type == tok.Return`. */ bool isReturn() { return type == Tok.Return; } /** Just a shortcut to avoid `token.type == Tok.Identifier`. */ bool isIdentifier() { return type == Tok.Identifier; } } /** Tok is short for TokenType. This enum list is to supply the Token with a type. This enum is used to switch over "many" places. */ enum Tok : ushort { /* Non-code related tokens */ EOF, /* Basic types */ Identifier, Integer, String, /* Basic operators */ Assign, PlusAssign, MinusAssign, StarAssign, SlashAssign, PercentAssign, Plus, Minus, Star, Slash, Percent, LeftShift, RightShift, UnsignedRightShift, Comma, And, /* Symbols */ OpenParentheses, CloseParentheses, OpenBrace, CloseBrace, OpenBracket, CloseBracket, Seperator, Colon, Dot, /* Comparator operators */ Eq, Ne, Lt, Gt, Le, Ge, Not, /* Keywords */ Byte, Ubyte, Short, Ushort, Int, Uint, Long, Ulong, Char, Wchar, Dchar, Float, Double, Bool, Void, Struct, Function, Delegate, Class, This, Interface, Union, Typedef, Typeid, Typeof, Sizeof, Alias, If, Else, While, For, Switch, Case, Default, Break, Return, Cast, Module, Import, New, /* Attributes */ Public, Private, Package, Export, Protected, Static, Final, Const, Abstract, Override, Deprecated, Auto, Extern, Align, Asm, In, Out, Body, Assert, Throw, Try, Catch, Finally, } /** An associative array to supply a Tok to String function. Keep always this list updated when adding a new Tok. */ public char[][Tok] typeToString; static this() { typeToString = [ Tok.EOF:"EOF"[], Tok.Identifier:"Identifier", Tok.Byte:"Byte", Tok.Short:"Short", Tok.Int:"Int", Tok.Long:"Long", Tok.Char:"Char", Tok.Wchar:"Wchar", Tok.Dchar:"Dchar", Tok.Bool:"Bool", Tok.Void:"Void", Tok.Function:"Function", Tok.Eq:"Eq", Tok.Ne:"Ne", Tok.Lt:"Lt", Tok.Le:"Le", Tok.Gt:"Gt", Tok.Ge:"Ge", Tok.OpenParentheses:"OpenParentheses", Tok.CloseParentheses:"CloseParentheses", Tok.OpenBrace:"OpenBrace", Tok.CloseBrace:"CloseBrace", Tok.OpenBracket:"OpenBracket", Tok.CloseBracket:"CloseBracket", Tok.Dot:"Dot", Tok.Assign:"Assign", Tok.Plus:"Plus", Tok.PlusAssign:"PlusAssign", Tok.Minus:"Minus", Tok.MinusAssign:"MinusAssign", Tok.Star:"Star", Tok.StarAssign:"StarAssign", Tok.Slash:"Slash", Tok.SlashAssign:"SlashAssign", Tok.Percent:"Percent", Tok.PercentAssign:"PercentAssign", Tok.LeftShift:"LeftShift", Tok.RightShift:"RightShift", Tok.UnsignedRightShift:"UnsignedRightShift", Tok.Integer:"Integer", Tok.If:"If", Tok.While:"While", Tok.For:"For", Tok.Switch:"Switch", Tok.Case:"Case", Tok.Default:"Default", Tok.Comma:"Comma", Tok.Return:"Return", Tok.Struct:"Struct", Tok.Class:"Class", Tok.This:"This", Tok.Colon:"Colon", Tok.Seperator:"Seperator", Tok.Cast:"Cast", Tok.Module:"Module", Tok.Import:"Import", Tok.String:"String", Tok.Public:"Public", Tok.Private:"Private", Tok.Protected:"Protected", Tok.Package:"Package", Tok.Export:"Export", Tok.Static:"Static", Tok.Final:"Finale", Tok.Public:"Public", Tok.Const:"Const", Tok.Abstract:"Abstract", Tok.Override:"Override", Tok.Deprecated:"Deprecated", Tok.Auto:"Auto", Tok.Extern:"Extern", Tok.New:"New", Tok.And:"And" ]; }