Mercurial > projects > dang
view lexer/Token.d @ 44:495188f9078e new_gen
Big update - Moving towards a better, more seperated parser
The parser no loner creates the AST directly, but through
callbacks(actions). This means the parser can be run with a different set
of actions that do something else.
The parser is not back to full strength yet, the main thing missing is the
various statements and structs.
Also added a SmallArray that uses the stack only until a given size is
exceeded, after which the array is copied unto the heap.
author | Anders Halager <halager@gmail.com> |
---|---|
date | Wed, 23 Apr 2008 00:57:45 +0200 |
parents | 4e879f82dd64 |
children | b6c1dc30ca4b |
line wrap: on
line source
module lexer.Token; public import misc.Location; import Integer = tango.text.convert.Integer; /** The Token struct will be used through the Lexer, Parser and other modules as a location into source. The Token should always be optimized for size to limit unnecessary memory usage. */ struct Token { Tok type; Location location; uint length; /** Create a new token with a Tok type, Location in source and a length of how many chars the Token span in the source */ static Token opCall (Tok type, Location location, uint length) { Token t; t.type = type; t.location = location; t.length = length; return t; } /** Get the type of the Token as a string */ char[] getType () { return typeToString[this.type]; } /** A human readable dump of a Token */ char[] toString () { return this.getType()~": Len: "~Integer.toString(this.length) ~", Loc: "~location.toString; } /** Get the string in the source that matches what this Token is covering. */ char[] get () { return location.get(length); } /** Returns true if the type of this token is a basic type (int, float, ...). Void is included, although a void in it self is not really a type. */ bool isBasicType() { return type >= Tok.Byte && type <= Tok.Void; } /** Just a shortcut to avoid `token.type == Tok.Identifier`. */ bool isIdentifier() { return type == Tok.Identifier; } } /** Tok is short for TokenType. This enum list is to supply the Token with a type. This enum is used to switch over "many" places. */ enum Tok : ushort { /* Non-code related tokens */ EOF, /* Basic types */ Identifier, Integer, /* Basic operators */ Assign, Add, Sub, Mul, Div, Comma, /* Symbols */ OpenParentheses, CloseParentheses, OpenBrace, CloseBrace, Seperator, Colon, Dot, /* Comparator operators */ Eq, Ne, Lt, Gt, Le, Ge, Not, /* Keywords */ Byte, Ubyte, Short, Ushort, Int, Uint, Long, Ulong, Float, Double, Bool, Void, Struct, If, Else, While, Switch, Case, Default, Return, } /** An associative array to supply a Tok to String function. Keep always this list updated when adding a new Tok. */ public char[][Tok] typeToString; static this() { typeToString = [ Tok.EOF:"EOF"[], Tok.Identifier:"Identifier", Tok.Byte:"Byte", Tok.Short:"Short", Tok.Int:"Int", Tok.Long:"Long", Tok.Bool:"Bool", Tok.Void:"Void", Tok.Eq:"Eq", Tok.Ne:"Ne", Tok.Lt:"Lt", Tok.Le:"Le", Tok.Gt:"Gt", Tok.Ge:"Ge", Tok.OpenParentheses:"OpenParentheses", Tok.CloseParentheses:"CloseParentheses", Tok.OpenBrace:"OpenBrace", Tok.CloseBrace:"CloseBrace", Tok.Dot:"Dot", Tok.Assign:"Assign", Tok.Add:"Add", Tok.Sub:"Sub", Tok.Mul:"Mul", Tok.Div:"Div", Tok.Integer:"Integer", Tok.If:"If", Tok.While:"While", Tok.Switch:"Switch", Tok.Case:"Case", Tok.Default:"Default", Tok.Comma:"Comma", Tok.Return:"Return", Tok.Struct:"Struct", Tok.Colon:"Colon", Tok.Seperator:"Seperator" ]; }