Mercurial > projects > dil
annotate trunk/src/Token.d @ 107:722c05bbd5eb
- Implemented parseEnumDeclaration() and added class EnumDeclaration.
- Added 'alias srcText identifier' to Token for Identifiers.
- Replaced all instances of token.srcText with token.identifier where applicable.
author | aziz |
---|---|
date | Sun, 08 Jul 2007 15:22:03 +0000 |
parents | 1a83e5070a84 |
children | ce636f3981cc |
rev | line source |
---|---|
0 | 1 /++ |
2 Author: Aziz Köksal | |
3 License: GPL2 | |
4 +/ | |
5 module Token; | |
6 | |
7 struct Position | |
8 { | |
9 size_t loc; | |
10 size_t col; | |
11 } | |
12 | |
3 | 13 enum TOK |
14 { | |
95 | 15 Invalid, |
16 | |
3 | 17 Identifier, |
4 | 18 Comment, |
9
5d6968cc751e
- Parsing string and character literals now (rudimentary implementation.)
aziz
parents:
4
diff
changeset
|
19 String, |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
20 CharLiteral, WCharLiteral, DCharLiteral, |
58 | 21 |
22 // Numbers | |
15 | 23 Number, |
58 | 24 Int32, Int64, Uint32, Uint64, |
97
1a83e5070a84
- Added code for parsing IntNumber- and RealNumberExpressions.
aziz
parents:
95
diff
changeset
|
25 // Floating point number scanner relies on this order. (FloatXY + 3 == ImaginaryXY) |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
26 Float32, Float64, Float80, |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
27 Imaginary32, Imaginary64, Imaginary80, |
58 | 28 |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
29 |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
30 // Brackets |
20 | 31 LParen, |
32 RParen, | |
33 LBracket, | |
34 RBracket, | |
35 LBrace, | |
36 RBrace, | |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
37 |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
38 Dot, Slice, Ellipses, |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
39 |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
40 // Floating point operators |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
41 Unordered, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
42 UorE, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
43 UorG, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
44 UorGorE, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
45 UorL, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
46 UorLorE, |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
47 LorEorG, |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
48 LorG, |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
49 |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
50 // Normal operators |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
51 Assign, Equal, NotEqual, Not, |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
36
diff
changeset
|
52 LessEqual, Less, |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
36
diff
changeset
|
53 GreaterEqual, Greater, |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
54 LShiftAssign, LShift, |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
36
diff
changeset
|
55 RShiftAssign,RShift, |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
36
diff
changeset
|
56 URShiftAssign, URShift, |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
57 OrAssign, OrLogical, OrBinary, |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
58 AndAssign, AndLogical, AndBinary, |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
59 PlusAssign, PlusPlus, Plus, |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
60 MinusAssign, MinusMinus, Minus, |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
61 DivAssign, Div, |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
62 MulAssign, Mul, |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
63 ModAssign, Mod, |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
64 XorAssign, Xor, |
27 | 65 CatAssign, Catenate, |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
66 Tilde, |
71 | 67 Identity, NotIdentity, |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
68 |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
69 Colon, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
70 Semicolon, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
71 Question, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
72 Comma, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
73 Dollar, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
74 |
28 | 75 /* Keywords: |
76 NB.: Token.isKeyword() depends on this list being contiguous. | |
77 */ | |
78 Abstract,Alias,Align,Asm,Assert,Auto,Body, | |
79 Bool,Break,Byte,Case,Cast,Catch,Cdouble, | |
80 Cent,Cfloat,Char,Class,Const,Continue,Creal, | |
81 Dchar,Debug,Default,Delegate,Delete,Deprecated,Do, | |
82 Double,Else,Enum,Export,Extern,False,Final, | |
83 Finally,Float,For,Foreach,Foreach_reverse,Function,Goto, | |
84 Idouble,If,Ifloat,Import,In,Inout,Int, | |
85 Interface,Invariant,Ireal,Is,Lazy,Long,Macro, | |
86 Mixin,Module,New,Null,Out,Override,Package, | |
87 Pragma,Private,Protected,Public,Real,Ref,Return, | |
88 Scope,Short,Static,Struct,Super,Switch,Synchronized, | |
89 Template,This,Throw,True,Try,Typedef,Typeid, | |
90 Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest, | |
91 Ushort,Version,Void,Volatile,Wchar,While,With, | |
92 | |
4 | 93 EOF |
3 | 94 } |
95 | |
28 | 96 alias TOK.Abstract KeywordsBegin; |
97 alias TOK.With KeywordsEnd; | |
98 | |
0 | 99 struct Token |
100 { | |
3 | 101 TOK type; |
84
ac8d961d10d1
- Added code for parsing This-,Super-,Null-,Bool-,Dollar-,CharLiteral- and StringLiteralExpression.
aziz
parents:
82
diff
changeset
|
102 // Position pos; |
0 | 103 |
4 | 104 char* start; |
105 char* end; | |
106 | |
0 | 107 union |
108 { | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
109 struct |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
110 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
111 string str; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
112 char pf; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
113 } |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
114 dchar dchar_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
115 long long_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
116 ulong ulong_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
117 int int_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
118 uint uint_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
119 float float_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
120 double double_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
121 real real_; |
0 | 122 } |
28 | 123 |
107
722c05bbd5eb
- Implemented parseEnumDeclaration() and added class EnumDeclaration.
aziz
parents:
97
diff
changeset
|
124 alias srcText identifier; |
722c05bbd5eb
- Implemented parseEnumDeclaration() and added class EnumDeclaration.
aziz
parents:
97
diff
changeset
|
125 |
65 | 126 string srcText() |
28 | 127 { |
30 | 128 assert(start && end); |
28 | 129 return start[0 .. end - start]; |
130 } | |
131 | |
132 bool isKeyword() | |
133 { | |
134 if (KeywordsBegin <= type && type <= KeywordsEnd) | |
135 return true; | |
136 return false; | |
137 } | |
0 | 138 } |