Mercurial > projects > dil
annotate trunk/src/Token.d @ 28:3a9daccf7d96
- Added table for identifiers to Lexer.
- Added keywords table.
- Added keywords to TOK.
author | aziz |
---|---|
date | Sun, 24 Jun 2007 17:19:03 +0000 |
parents | 43b6bf56f0e9 |
children | ef83eea26bbd |
rev | line source |
---|---|
0 | 1 /++ |
2 Author: Aziz Köksal | |
3 License: GPL2 | |
4 +/ | |
5 module Token; | |
6 | |
7 struct Position | |
8 { | |
9 size_t loc; | |
10 size_t col; | |
11 } | |
12 | |
3 | 13 enum TOK |
14 { | |
15 Identifier, | |
4 | 16 Comment, |
9
5d6968cc751e
- Parsing string and character literals now (rudimentary implementation.)
aziz
parents:
4
diff
changeset
|
17 String, |
5d6968cc751e
- Parsing string and character literals now (rudimentary implementation.)
aziz
parents:
4
diff
changeset
|
18 Character, |
14 | 19 DivisionAssign, |
15 | 20 Number, |
20 | 21 /* Braces */ |
22 LParen, | |
23 RParen, | |
24 LBracket, | |
25 RBracket, | |
26 LBrace, | |
27 RBrace, | |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
28 |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
29 Dot, Slice, Ellipses, |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
30 |
26 | 31 Assign, Equal, |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
32 OrAssign, OrLogical, OrBinary, |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
33 AndAssign, AndLogical, AndBinary, |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
34 PlusAssign, PlusPlus, Plus, |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
35 MinusAssign, MinusMinus, Minus, |
27 | 36 CatAssign, Catenate, |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
37 |
27 | 38 Tilde, |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
39 Colon, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
40 Semicolon, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
41 Question, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
42 Comma, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
43 Dollar, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
44 |
28 | 45 /* Keywords: |
46 NB.: Token.isKeyword() depends on this list being contiguous. | |
47 */ | |
48 Abstract,Alias,Align,Asm,Assert,Auto,Body, | |
49 Bool,Break,Byte,Case,Cast,Catch,Cdouble, | |
50 Cent,Cfloat,Char,Class,Const,Continue,Creal, | |
51 Dchar,Debug,Default,Delegate,Delete,Deprecated,Do, | |
52 Double,Else,Enum,Export,Extern,False,Final, | |
53 Finally,Float,For,Foreach,Foreach_reverse,Function,Goto, | |
54 Idouble,If,Ifloat,Import,In,Inout,Int, | |
55 Interface,Invariant,Ireal,Is,Lazy,Long,Macro, | |
56 Mixin,Module,New,Null,Out,Override,Package, | |
57 Pragma,Private,Protected,Public,Real,Ref,Return, | |
58 Scope,Short,Static,Struct,Super,Switch,Synchronized, | |
59 Template,This,Throw,True,Try,Typedef,Typeid, | |
60 Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest, | |
61 Ushort,Version,Void,Volatile,Wchar,While,With, | |
62 | |
4 | 63 EOF |
3 | 64 } |
65 | |
28 | 66 alias TOK.Abstract KeywordsBegin; |
67 alias TOK.With KeywordsEnd; | |
68 | |
0 | 69 struct Token |
70 { | |
3 | 71 TOK type; |
0 | 72 Position pos; |
73 | |
4 | 74 char* start; |
75 char* end; | |
76 | |
0 | 77 union |
78 { | |
79 char[] str; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
80 dchar chr; |
0 | 81 float f; |
82 double d; | |
83 } | |
28 | 84 |
85 string span() | |
86 { | |
87 return start[0 .. end - start]; | |
88 } | |
89 | |
90 bool isKeyword() | |
91 { | |
92 if (KeywordsBegin <= type && type <= KeywordsEnd) | |
93 return true; | |
94 return false; | |
95 } | |
0 | 96 } |