Mercurial > projects > dil
annotate trunk/src/dil/Token.d @ 359:511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
- Changed enum Linkage into a class.
- Added method parseLinkage() to Parser.
- Added MAX to enum TOK.
- Added methods nextNWS() and prevNWS() to Token.
- Fixed tokToString table.
author | aziz |
---|---|
date | Tue, 28 Aug 2007 11:41:00 +0000 |
parents | 95f1b6e43214 |
children | eb08126cca56 |
rev | line source |
---|---|
0 | 1 /++ |
2 Author: Aziz Köksal | |
249 | 3 License: GPL3 |
0 | 4 +/ |
326 | 5 module dil.Token; |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
6 import std.c.stdlib : malloc, free; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
7 import std.outofmemory; |
0 | 8 |
9 struct Position | |
10 { | |
11 size_t loc; | |
12 size_t col; | |
13 } | |
14 | |
314
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
15 enum TOK : ushort |
3 | 16 { |
95 | 17 Invalid, |
18 | |
314
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
19 /// Flag for whitespace tokens that must be ignored in the parsing phase. |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
20 Whitespace = 0x8000, |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
21 Comment = 1 | Whitespace, |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
22 Shebang = 2 | Whitespace, |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
23 HashLine = 3 | Whitespace, |
323 | 24 Filespec = 4 | Whitespace, |
314
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
25 |
323 | 26 Identifier = 5, |
9
5d6968cc751e
- Parsing string and character literals now (rudimentary implementation.)
aziz
parents:
4
diff
changeset
|
27 String, |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
28 CharLiteral, WCharLiteral, DCharLiteral, |
58 | 29 |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
30 // Special tokens |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
31 FILE, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
32 LINE, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
33 DATE, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
34 TIME, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
35 TIMESTAMP, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
36 VENDOR, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
37 VERSION, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
38 |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
39 // Number literals |
58 | 40 Int32, Int64, Uint32, Uint64, |
97
1a83e5070a84
- Added code for parsing IntNumber- and RealNumberExpressions.
aziz
parents:
95
diff
changeset
|
41 // Floating point number scanner relies on this order. (FloatXY + 3 == ImaginaryXY) |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
42 Float32, Float64, Float80, |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
43 Imaginary32, Imaginary64, Imaginary80, |
58 | 44 |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
45 |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
46 // Brackets |
20 | 47 LParen, |
48 RParen, | |
49 LBracket, | |
50 RBracket, | |
51 LBrace, | |
52 RBrace, | |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
53 |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
54 Dot, Slice, Ellipses, |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
55 |
131 | 56 // Floating point number operators |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
57 Unordered, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
58 UorE, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
59 UorG, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
60 UorGorE, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
61 UorL, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
62 UorLorE, |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
63 LorEorG, |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
64 LorG, |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
65 |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
66 // Normal operators |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
67 Assign, Equal, NotEqual, Not, |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
36
diff
changeset
|
68 LessEqual, Less, |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
36
diff
changeset
|
69 GreaterEqual, Greater, |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
70 LShiftAssign, LShift, |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
36
diff
changeset
|
71 RShiftAssign,RShift, |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
36
diff
changeset
|
72 URShiftAssign, URShift, |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
73 OrAssign, OrLogical, OrBinary, |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
74 AndAssign, AndLogical, AndBinary, |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
75 PlusAssign, PlusPlus, Plus, |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
76 MinusAssign, MinusMinus, Minus, |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
77 DivAssign, Div, |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
78 MulAssign, Mul, |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
79 ModAssign, Mod, |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
80 XorAssign, Xor, |
27 | 81 CatAssign, Catenate, |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
82 Tilde, |
71 | 83 Identity, NotIdentity, |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
84 |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
85 Colon, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
86 Semicolon, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
87 Question, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
88 Comma, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
89 Dollar, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
90 |
28 | 91 /* Keywords: |
92 NB.: Token.isKeyword() depends on this list being contiguous. | |
93 */ | |
94 Abstract,Alias,Align,Asm,Assert,Auto,Body, | |
95 Bool,Break,Byte,Case,Cast,Catch,Cdouble, | |
96 Cent,Cfloat,Char,Class,Const,Continue,Creal, | |
97 Dchar,Debug,Default,Delegate,Delete,Deprecated,Do, | |
98 Double,Else,Enum,Export,Extern,False,Final, | |
99 Finally,Float,For,Foreach,Foreach_reverse,Function,Goto, | |
100 Idouble,If,Ifloat,Import,In,Inout,Int, | |
269 | 101 Interface,Invariant,Ireal,Is,Lazy,Long,Macro/+D2.0+/, |
28 | 102 Mixin,Module,New,Null,Out,Override,Package, |
269 | 103 Pragma,Private,Protected,Public,Real,Ref/+D2.0+/,Return, |
28 | 104 Scope,Short,Static,Struct,Super,Switch,Synchronized, |
269 | 105 Template,This,Throw,Traits/+D2.0+/,True,Try,Typedef,Typeid, |
28 | 106 Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest, |
107 Ushort,Version,Void,Volatile,Wchar,While,With, | |
108 | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
109 HEAD, // start of linked list |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
110 EOF, |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
111 MAX |
3 | 112 } |
113 | |
28 | 114 alias TOK.Abstract KeywordsBegin; |
115 alias TOK.With KeywordsEnd; | |
116 | |
0 | 117 struct Token |
118 { | |
3 | 119 TOK type; |
84
ac8d961d10d1
- Added code for parsing This-,Super-,Null-,Bool-,Dollar-,CharLiteral- and StringLiteralExpression.
aziz
parents:
82
diff
changeset
|
120 // Position pos; |
0 | 121 |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
122 Token* next, prev; |
131 | 123 |
4 | 124 char* start; |
125 char* end; | |
126 | |
0 | 127 union |
128 { | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
129 struct |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
130 { |
323 | 131 Token* line_num; // #line number |
132 Token* line_filespec; // #line number filespec | |
133 } | |
134 struct | |
135 { | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
136 string str; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
137 char pf; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
138 } |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
139 dchar dchar_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
140 long long_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
141 ulong ulong_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
142 int int_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
143 uint uint_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
144 float float_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
145 double double_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
146 real real_; |
0 | 147 } |
28 | 148 |
107
722c05bbd5eb
- Implemented parseEnumDeclaration() and added class EnumDeclaration.
aziz
parents:
97
diff
changeset
|
149 alias srcText identifier; |
722c05bbd5eb
- Implemented parseEnumDeclaration() and added class EnumDeclaration.
aziz
parents:
97
diff
changeset
|
150 |
65 | 151 string srcText() |
28 | 152 { |
30 | 153 assert(start && end); |
28 | 154 return start[0 .. end - start]; |
155 } | |
156 | |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
157 /// Find next non-whitespace token. Returns 'this' token if the next token is TOK.EOF or null. |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
158 Token* nextNWS() |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
159 out(token) |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
160 { |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
161 assert(token !is null); |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
162 } |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
163 body |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
164 { |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
165 auto token = next; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
166 while (token !is null && token.isWhitespace) |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
167 token = token.next; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
168 if (token is null || token.type == TOK.EOF) |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
169 return this; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
170 return token; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
171 } |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
172 |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
173 /// Find previous non-whitespace token. Returns 'this' token if the previous token is TOK.HEAD or null. |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
174 Token* prevNWS() |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
175 out(token) |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
176 { |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
177 assert(token !is null); |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
178 } |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
179 body |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
180 { |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
181 auto token = prev; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
182 while (token !is null && token.isWhitespace) |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
183 token = token.prev; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
184 if (token is null || token.type == TOK.HEAD) |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
185 return this; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
186 return token; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
187 } |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
188 |
210 | 189 static string toString(TOK tok) |
208 | 190 { |
191 return tokToString[tok]; | |
192 } | |
193 | |
28 | 194 bool isKeyword() |
195 { | |
163
f27a98bb17c7
- Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents:
131
diff
changeset
|
196 return KeywordsBegin <= type && type <= KeywordsEnd; |
28 | 197 } |
208 | 198 |
314
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
199 bool isWhitespace() |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
200 { |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
201 return !!(type & TOK.Whitespace); |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
202 } |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
203 |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
204 bool isSpecialToken() |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
205 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
206 return *start == '_' && type != TOK.Identifier; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
207 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
208 |
163
f27a98bb17c7
- Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents:
131
diff
changeset
|
209 int opEquals(TOK type2) |
f27a98bb17c7
- Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents:
131
diff
changeset
|
210 { |
f27a98bb17c7
- Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents:
131
diff
changeset
|
211 return type == type2; |
f27a98bb17c7
- Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents:
131
diff
changeset
|
212 } |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
213 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
214 new(size_t size) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
215 { |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
216 void* p = malloc(size); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
217 if (p is null) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
218 throw new OutOfMemoryException(); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
219 *cast(Token*)p = Token.init; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
220 return p; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
221 } |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
222 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
223 delete(void* p) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
224 { |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
225 free(p); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
226 } |
208 | 227 } |
228 | |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
229 const string[] tokToString = [ |
208 | 230 "Invalid", |
231 | |
314
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
232 "Comment", |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
233 "#! /shebang/", |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
234 "#line", |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
235 `"filespec"`, |
314
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
236 |
208 | 237 "Identifier", |
238 "String", | |
239 "CharLiteral", "WCharLiteral", "DCharLiteral", | |
240 | |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
241 "__FILE__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
242 "__LINE__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
243 "__DATE__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
244 "__TIME__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
245 "__TIMESTAMP__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
246 "__VENDOR__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
247 "__VERSION__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
248 |
208 | 249 "Int32", "Int64", "Uint32", "Uint64", |
250 "Float32", "Float64", "Float80", | |
251 "Imaginary32", "Imaginary64", "Imaginary80", | |
252 | |
253 "(", | |
254 ")", | |
255 "[", | |
256 "]", | |
257 "{", | |
258 "}", | |
259 | |
260 ".", "..", "...", | |
261 | |
262 "Unordered", | |
263 "UorE", | |
264 "UorG", | |
265 "UorGorE", | |
266 "UorL", | |
267 "UorLorE", | |
268 "LorEorG", | |
269 "LorG", | |
270 | |
271 "=", "==", "!=", "!", | |
272 "<=", "<", | |
273 ">=", ">", | |
274 "<<=", "<<", | |
275 ">>=",">>", | |
276 ">>>=", ">>>", | |
277 "|=", "||", "|", | |
278 "&=", "&&", "&", | |
279 "+=", "++", "+", | |
280 "-=", "--", "-", | |
281 "/=", "/", | |
282 "*=", "*", | |
283 "%=", "%", | |
284 "^=", "^", | |
285 "~=", "~", | |
286 "~", | |
287 "is", "!is", | |
288 | |
289 ":", | |
290 ";", | |
291 "?", | |
292 ",", | |
293 "$", | |
294 | |
295 "abstract","alias","align","asm","assert","auto","body", | |
296 "bool","break","byte","case","cast","catch","cdouble", | |
297 "cent","cfloat","char","class","const","continue","creal", | |
298 "dchar","debug","default","delegate","delete","deprecated","do", | |
299 "double","else","enum","export","extern","false","final", | |
300 "finally","float","for","foreach","foreach_reverse","function","goto", | |
301 "idouble","if","ifloat","import","in","inout","int", | |
302 "interface","invariant","ireal","is","lazy","long","macro", | |
303 "mixin","module","new","null","out","override","package", | |
304 "pragma","private","protected","public","real","ref","return", | |
305 "scope","short","static","struct","super","switch","synchronized", | |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
306 "template","this","throw","__traits","true","try","typedef","typeid", |
208 | 307 "typeof","ubyte","ucent","uint","ulong","union","unittest", |
308 "ushort","version","void","volatile","wchar","while","with", | |
309 | |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
310 "HEAD", |
208 | 311 "EOF" |
312 ]; | |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
313 static assert(tokToString.length == TOK.MAX); |