Mercurial > projects > dil
annotate trunk/src/Token.d @ 30:426767b94635
- Added code for parsing the '#line' special token.
- Added fileName to Lexer.
- Added _uint to Token.
- Added basic peek() method to Lexer.
| author | aziz |
|---|---|
| date | Mon, 25 Jun 2007 08:37:02 +0000 |
| parents | ef83eea26bbd |
| children | 94f09f4e988e |
| rev | line source |
|---|---|
| 0 | 1 /++ |
| 2 Author: Aziz Köksal | |
| 3 License: GPL2 | |
| 4 +/ | |
| 5 module Token; | |
| 6 | |
| 7 struct Position | |
| 8 { | |
| 9 size_t loc; | |
| 10 size_t col; | |
| 11 } | |
| 12 | |
| 3 | 13 enum TOK |
| 14 { | |
| 15 Identifier, | |
| 4 | 16 Comment, |
|
9
5d6968cc751e
- Parsing string and character literals now (rudimentary implementation.)
aziz
parents:
4
diff
changeset
|
17 String, |
|
5d6968cc751e
- Parsing string and character literals now (rudimentary implementation.)
aziz
parents:
4
diff
changeset
|
18 Character, |
| 14 | 19 DivisionAssign, |
| 15 | 20 Number, |
| 20 | 21 /* Braces */ |
| 22 LParen, | |
| 23 RParen, | |
| 24 LBracket, | |
| 25 RBracket, | |
| 26 LBrace, | |
| 27 RBrace, | |
|
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
28 |
|
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
29 Dot, Slice, Ellipses, |
|
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
30 |
| 26 | 31 Assign, Equal, |
|
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
32 OrAssign, OrLogical, OrBinary, |
|
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
33 AndAssign, AndLogical, AndBinary, |
|
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
34 PlusAssign, PlusPlus, Plus, |
|
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
35 MinusAssign, MinusMinus, Minus, |
|
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
36 MulAssign, Mul, |
|
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
37 ModAssign, Mod, |
|
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
38 XorAssign, Xor, |
| 27 | 39 CatAssign, Catenate, |
|
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
40 |
| 27 | 41 Tilde, |
|
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
42 Colon, |
|
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
43 Semicolon, |
|
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
44 Question, |
|
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
45 Comma, |
|
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
46 Dollar, |
|
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
47 |
| 28 | 48 /* Keywords: |
| 49 NB.: Token.isKeyword() depends on this list being contiguous. | |
| 50 */ | |
| 51 Abstract,Alias,Align,Asm,Assert,Auto,Body, | |
| 52 Bool,Break,Byte,Case,Cast,Catch,Cdouble, | |
| 53 Cent,Cfloat,Char,Class,Const,Continue,Creal, | |
| 54 Dchar,Debug,Default,Delegate,Delete,Deprecated,Do, | |
| 55 Double,Else,Enum,Export,Extern,False,Final, | |
| 56 Finally,Float,For,Foreach,Foreach_reverse,Function,Goto, | |
| 57 Idouble,If,Ifloat,Import,In,Inout,Int, | |
| 58 Interface,Invariant,Ireal,Is,Lazy,Long,Macro, | |
| 59 Mixin,Module,New,Null,Out,Override,Package, | |
| 60 Pragma,Private,Protected,Public,Real,Ref,Return, | |
| 61 Scope,Short,Static,Struct,Super,Switch,Synchronized, | |
| 62 Template,This,Throw,True,Try,Typedef,Typeid, | |
| 63 Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest, | |
| 64 Ushort,Version,Void,Volatile,Wchar,While,With, | |
| 65 | |
| 4 | 66 EOF |
| 3 | 67 } |
| 68 | |
| 28 | 69 alias TOK.Abstract KeywordsBegin; |
| 70 alias TOK.With KeywordsEnd; | |
| 71 | |
| 0 | 72 struct Token |
| 73 { | |
| 3 | 74 TOK type; |
| 0 | 75 Position pos; |
| 76 | |
| 4 | 77 char* start; |
| 78 char* end; | |
| 79 | |
| 0 | 80 union |
| 81 { | |
| 82 char[] str; | |
|
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
83 dchar chr; |
| 30 | 84 uint _uint; |
| 0 | 85 float f; |
| 86 double d; | |
| 87 } | |
| 28 | 88 |
| 89 string span() | |
| 90 { | |
| 30 | 91 assert(start && end); |
| 28 | 92 return start[0 .. end - start]; |
| 93 } | |
| 94 | |
| 95 bool isKeyword() | |
| 96 { | |
| 97 if (KeywordsBegin <= type && type <= KeywordsEnd) | |
| 98 return true; | |
| 99 return false; | |
| 100 } | |
| 0 | 101 } |