Mercurial > projects > dil
view trunk/src/Token.d @ 82:fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
- Added CharaterLiteral tokens with utf-16/32 versions to distinguish between different character literals.
- Checking for valid Unicode codepoint when finished lexing hex escape sequences.
author | aziz |
---|---|
date | Wed, 04 Jul 2007 22:19:01 +0000 |
parents | b3777cca323c |
children | ac8d961d10d1 |
line wrap: on
line source
/++ Author: Aziz Köksal License: GPL2 +/ module Token; struct Position { size_t loc; size_t col; } enum TOK { Identifier, Comment, String, CharLiteral, WCharLiteral, DCharLiteral, // Numbers Number, Int32, Int64, Uint32, Uint64, // Floating point scanner relies on this order. (FloatXY + 3 == ImaginaryXY) Float32, Float64, Float80, Imaginary32, Imaginary64, Imaginary80, // Brackets LParen, RParen, LBracket, RBracket, LBrace, RBrace, Dot, Slice, Ellipses, // Floating point operators Unordered, UorE, UorG, UorGorE, UorL, UorLorE, LorEorG, LorG, // Normal operators Assign, Equal, NotEqual, Not, LessEqual, Less, GreaterEqual, Greater, LShiftAssign, LShift, RShiftAssign,RShift, URShiftAssign, URShift, OrAssign, OrLogical, OrBinary, AndAssign, AndLogical, AndBinary, PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus, DivAssign, Div, MulAssign, Mul, ModAssign, Mod, XorAssign, Xor, CatAssign, Catenate, Tilde, Identity, NotIdentity, Colon, Semicolon, Question, Comma, Dollar, /* Keywords: NB.: Token.isKeyword() depends on this list being contiguous. */ Abstract,Alias,Align,Asm,Assert,Auto,Body, Bool,Break,Byte,Case,Cast,Catch,Cdouble, Cent,Cfloat,Char,Class,Const,Continue,Creal, Dchar,Debug,Default,Delegate,Delete,Deprecated,Do, Double,Else,Enum,Export,Extern,False,Final, Finally,Float,For,Foreach,Foreach_reverse,Function,Goto, Idouble,If,Ifloat,Import,In,Inout,Int, Interface,Invariant,Ireal,Is,Lazy,Long,Macro, Mixin,Module,New,Null,Out,Override,Package, Pragma,Private,Protected,Public,Real,Ref,Return, Scope,Short,Static,Struct,Super,Switch,Synchronized, Template,This,Throw,True,Try,Typedef,Typeid, Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest, Ushort,Version,Void,Volatile,Wchar,While,With, EOF } alias TOK.Abstract KeywordsBegin; alias TOK.With KeywordsEnd; struct Token { TOK type; Position pos; char* start; char* end; union { struct { string str; char pf; } dchar dchar_; long long_; ulong ulong_; int int_; uint uint_; float float_; double double_; real real_; } string srcText() { assert(start && end); return start[0 .. end - start]; } bool isKeyword() { if (KeywordsBegin <= type && type <= KeywordsEnd) return true; return false; } }