|
1
|
1
|
|
|
2 // Compiler implementation of the D programming language
|
|
159
|
3 // Copyright (c) 1999-2008 by Digital Mars
|
|
1
|
4 // All Rights Reserved
|
|
|
5 // written by Walter Bright
|
|
|
6 // http://www.digitalmars.com
|
|
|
7 // License for redistribution is by either the Artistic License
|
|
|
8 // in artistic.txt, or the GNU General Public License in gnu.txt.
|
|
|
9 // See the included readme.txt for details.
|
|
|
10
|
|
|
11 #ifndef DMD_LEXER_H
|
|
|
12 #define DMD_LEXER_H
|
|
|
13
|
|
|
14 #ifdef __DMC__
|
|
|
15 #pragma once
|
|
|
16 #endif /* __DMC__ */
|
|
|
17
|
|
|
18 #include "root.h"
|
|
|
19 #include "mars.h"
|
|
|
20
|
|
|
21 struct StringTable;
|
|
|
22 struct Identifier;
|
|
|
23 struct Module;
|
|
|
24
|
|
|
25 /* Tokens:
|
|
|
26 ( )
|
|
|
27 [ ]
|
|
|
28 { }
|
|
|
29 < > <= >= == != === !==
|
|
|
30 << >> <<= >>= >>> >>>=
|
|
|
31 + - += -=
|
|
|
32 * / % *= /= %=
|
|
|
33 & | ^ &= |= ^=
|
|
|
34 = ! ~
|
|
|
35 ++ --
|
|
|
36 . -> : ,
|
|
|
37 ? && ||
|
|
|
38 */
|
|
|
39
|
|
|
40 enum TOK
|
|
|
41 {
|
|
|
42 TOKreserved,
|
|
|
43
|
|
|
44 // Other
|
|
|
45 TOKlparen, TOKrparen,
|
|
|
46 TOKlbracket, TOKrbracket,
|
|
|
47 TOKlcurly, TOKrcurly,
|
|
|
48 TOKcolon, TOKneg,
|
|
|
49 TOKsemicolon, TOKdotdotdot,
|
|
|
50 TOKeof, TOKcast,
|
|
|
51 TOKnull, TOKassert,
|
|
|
52 TOKtrue, TOKfalse,
|
|
|
53 TOKarray, TOKcall,
|
|
|
54 TOKaddress, TOKtypedot,
|
|
|
55 TOKtype, TOKthrow,
|
|
|
56 TOKnew, TOKdelete,
|
|
|
57 TOKstar, TOKsymoff,
|
|
|
58 TOKvar, TOKdotvar,
|
|
|
59 TOKdotti, TOKdotexp,
|
|
|
60 TOKdottype, TOKslice,
|
|
|
61 TOKarraylength, TOKversion,
|
|
|
62 TOKmodule, TOKdollar,
|
|
|
63 TOKtemplate, TOKdottd,
|
|
|
64 TOKdeclaration, TOKtypeof,
|
|
|
65 TOKpragma, TOKdsymbol,
|
|
|
66 TOKtypeid, TOKuadd,
|
|
|
67 TOKremove,
|
|
|
68 TOKnewanonclass, TOKcomment,
|
|
|
69 TOKarrayliteral, TOKassocarrayliteral,
|
|
|
70 TOKstructliteral,
|
|
|
71
|
|
|
72 // Operators
|
|
|
73 TOKlt, TOKgt,
|
|
|
74 TOKle, TOKge,
|
|
|
75 TOKequal, TOKnotequal,
|
|
|
76 TOKidentity, TOKnotidentity,
|
|
|
77 TOKindex, TOKis,
|
|
|
78 TOKtobool,
|
|
|
79
|
|
|
80 // 60
|
|
|
81 // NCEG floating point compares
|
|
|
82 // !<>= <> <>= !> !>= !< !<= !<>
|
|
|
83 TOKunord,TOKlg,TOKleg,TOKule,TOKul,TOKuge,TOKug,TOKue,
|
|
|
84
|
|
|
85 TOKshl, TOKshr,
|
|
|
86 TOKshlass, TOKshrass,
|
|
|
87 TOKushr, TOKushrass,
|
|
|
88 TOKcat, TOKcatass, // ~ ~=
|
|
|
89 TOKadd, TOKmin, TOKaddass, TOKminass,
|
|
|
90 TOKmul, TOKdiv, TOKmod,
|
|
|
91 TOKmulass, TOKdivass, TOKmodass,
|
|
|
92 TOKand, TOKor, TOKxor,
|
|
|
93 TOKandass, TOKorass, TOKxorass,
|
|
|
94 TOKassign, TOKnot, TOKtilde,
|
|
159
|
95 TOKplusplus, TOKminusminus, TOKconstruct, TOKblit,
|
|
1
|
96 TOKdot, TOKarrow, TOKcomma,
|
|
|
97 TOKquestion, TOKandand, TOKoror,
|
|
|
98
|
|
159
|
99 // 104
|
|
1
|
100 // Numeric literals
|
|
|
101 TOKint32v, TOKuns32v,
|
|
|
102 TOKint64v, TOKuns64v,
|
|
|
103 TOKfloat32v, TOKfloat64v, TOKfloat80v,
|
|
|
104 TOKimaginary32v, TOKimaginary64v, TOKimaginary80v,
|
|
|
105
|
|
|
106 // Char constants
|
|
|
107 TOKcharv, TOKwcharv, TOKdcharv,
|
|
|
108
|
|
|
109 // Leaf operators
|
|
|
110 TOKidentifier, TOKstring,
|
|
|
111 TOKthis, TOKsuper,
|
|
|
112 TOKhalt, TOKtuple,
|
|
|
113
|
|
|
114 // Basic types
|
|
|
115 TOKvoid,
|
|
|
116 TOKint8, TOKuns8,
|
|
|
117 TOKint16, TOKuns16,
|
|
|
118 TOKint32, TOKuns32,
|
|
|
119 TOKint64, TOKuns64,
|
|
|
120 TOKfloat32, TOKfloat64, TOKfloat80,
|
|
|
121 TOKimaginary32, TOKimaginary64, TOKimaginary80,
|
|
|
122 TOKcomplex32, TOKcomplex64, TOKcomplex80,
|
|
|
123 TOKchar, TOKwchar, TOKdchar, TOKbit, TOKbool,
|
|
|
124 TOKcent, TOKucent,
|
|
|
125
|
|
|
126 // Aggregates
|
|
|
127 TOKstruct, TOKclass, TOKinterface, TOKunion, TOKenum, TOKimport,
|
|
|
128 TOKtypedef, TOKalias, TOKoverride, TOKdelegate, TOKfunction,
|
|
|
129 TOKmixin,
|
|
|
130
|
|
|
131 TOKalign, TOKextern, TOKprivate, TOKprotected, TOKpublic, TOKexport,
|
|
|
132 TOKstatic, /*TOKvirtual,*/ TOKfinal, TOKconst, TOKabstract, TOKvolatile,
|
|
|
133 TOKdebug, TOKdeprecated, TOKin, TOKout, TOKinout, TOKlazy,
|
|
159
|
134 TOKauto, TOKpackage, TOKmanifest,
|
|
1
|
135
|
|
|
136 // Statements
|
|
|
137 TOKif, TOKelse, TOKwhile, TOKfor, TOKdo, TOKswitch,
|
|
|
138 TOKcase, TOKdefault, TOKbreak, TOKcontinue, TOKwith,
|
|
|
139 TOKsynchronized, TOKreturn, TOKgoto, TOKtry, TOKcatch, TOKfinally,
|
|
|
140 TOKasm, TOKforeach, TOKforeach_reverse,
|
|
|
141 TOKscope,
|
|
|
142 TOKon_scope_exit, TOKon_scope_failure, TOKon_scope_success,
|
|
|
143
|
|
|
144 // Contracts
|
|
|
145 TOKbody, TOKinvariant,
|
|
|
146
|
|
|
147 // Testing
|
|
|
148 TOKunittest,
|
|
|
149
|
|
|
150 // Added after 1.0
|
|
|
151 TOKref,
|
|
|
152 TOKmacro,
|
|
336
|
153 #if DMDV2
|
|
19
|
154 TOKtraits,
|
|
159
|
155 TOKoverloadset,
|
|
|
156 TOKpure,
|
|
336
|
157 TOKnothrow,
|
|
|
158 TOKtls,
|
|
|
159 TOKline,
|
|
|
160 TOKfile,
|
|
19
|
161 #endif
|
|
1
|
162
|
|
|
163 TOKMAX
|
|
|
164 };
|
|
|
165
|
|
|
166 #define CASE_BASIC_TYPES \
|
|
|
167 case TOKwchar: case TOKdchar: \
|
|
|
168 case TOKbit: case TOKbool: case TOKchar: \
|
|
|
169 case TOKint8: case TOKuns8: \
|
|
|
170 case TOKint16: case TOKuns16: \
|
|
|
171 case TOKint32: case TOKuns32: \
|
|
|
172 case TOKint64: case TOKuns64: \
|
|
|
173 case TOKfloat32: case TOKfloat64: case TOKfloat80: \
|
|
|
174 case TOKimaginary32: case TOKimaginary64: case TOKimaginary80: \
|
|
|
175 case TOKcomplex32: case TOKcomplex64: case TOKcomplex80: \
|
|
|
176 case TOKvoid
|
|
|
177
|
|
|
178 #define CASE_BASIC_TYPES_X(t) \
|
|
|
179 case TOKvoid: t = Type::tvoid; goto LabelX; \
|
|
|
180 case TOKint8: t = Type::tint8; goto LabelX; \
|
|
|
181 case TOKuns8: t = Type::tuns8; goto LabelX; \
|
|
|
182 case TOKint16: t = Type::tint16; goto LabelX; \
|
|
|
183 case TOKuns16: t = Type::tuns16; goto LabelX; \
|
|
|
184 case TOKint32: t = Type::tint32; goto LabelX; \
|
|
|
185 case TOKuns32: t = Type::tuns32; goto LabelX; \
|
|
|
186 case TOKint64: t = Type::tint64; goto LabelX; \
|
|
|
187 case TOKuns64: t = Type::tuns64; goto LabelX; \
|
|
|
188 case TOKfloat32: t = Type::tfloat32; goto LabelX; \
|
|
|
189 case TOKfloat64: t = Type::tfloat64; goto LabelX; \
|
|
|
190 case TOKfloat80: t = Type::tfloat80; goto LabelX; \
|
|
|
191 case TOKimaginary32: t = Type::timaginary32; goto LabelX; \
|
|
|
192 case TOKimaginary64: t = Type::timaginary64; goto LabelX; \
|
|
|
193 case TOKimaginary80: t = Type::timaginary80; goto LabelX; \
|
|
|
194 case TOKcomplex32: t = Type::tcomplex32; goto LabelX; \
|
|
|
195 case TOKcomplex64: t = Type::tcomplex64; goto LabelX; \
|
|
|
196 case TOKcomplex80: t = Type::tcomplex80; goto LabelX; \
|
|
|
197 case TOKbit: t = Type::tbit; goto LabelX; \
|
|
|
198 case TOKbool: t = Type::tbool; goto LabelX; \
|
|
|
199 case TOKchar: t = Type::tchar; goto LabelX; \
|
|
|
200 case TOKwchar: t = Type::twchar; goto LabelX; \
|
|
|
201 case TOKdchar: t = Type::tdchar; goto LabelX; \
|
|
|
202 LabelX
|
|
|
203
|
|
|
204 struct Token
|
|
|
205 {
|
|
|
206 Token *next;
|
|
|
207 unsigned char *ptr; // pointer to first character of this token within buffer
|
|
|
208 enum TOK value;
|
|
|
209 unsigned char *blockComment; // doc comment string prior to this token
|
|
|
210 unsigned char *lineComment; // doc comment for previous token
|
|
|
211 union
|
|
|
212 {
|
|
|
213 // Integers
|
|
|
214 d_int32 int32value;
|
|
|
215 d_uns32 uns32value;
|
|
|
216 d_int64 int64value;
|
|
|
217 d_uns64 uns64value;
|
|
|
218
|
|
|
219 // Floats
|
|
|
220 #ifdef IN_GCC
|
|
|
221 // real_t float80value; // can't use this in a union!
|
|
|
222 #else
|
|
|
223 d_float80 float80value;
|
|
|
224 #endif
|
|
|
225
|
|
|
226 struct
|
|
|
227 { unsigned char *ustring; // UTF8 string
|
|
|
228 unsigned len;
|
|
|
229 unsigned char postfix; // 'c', 'w', 'd'
|
|
|
230 };
|
|
|
231
|
|
|
232 Identifier *ident;
|
|
|
233 };
|
|
|
234 #ifdef IN_GCC
|
|
|
235 real_t float80value; // can't use this in a union!
|
|
|
236 #endif
|
|
|
237
|
|
|
238 static char *tochars[TOKMAX];
|
|
|
239 static void *operator new(size_t sz);
|
|
|
240
|
|
|
241 int isKeyword();
|
|
|
242 void print();
|
|
|
243 char *toChars();
|
|
|
244 static char *toChars(enum TOK);
|
|
|
245 };
|
|
|
246
|
|
|
247 struct Lexer
|
|
|
248 {
|
|
|
249 static StringTable stringtable;
|
|
|
250 static OutBuffer stringbuffer;
|
|
|
251 static Token *freelist;
|
|
|
252
|
|
|
253 Loc loc; // for error messages
|
|
|
254
|
|
|
255 unsigned char *base; // pointer to start of buffer
|
|
|
256 unsigned char *end; // past end of buffer
|
|
|
257 unsigned char *p; // current character
|
|
|
258 Token token;
|
|
|
259 Module *mod;
|
|
|
260 int doDocComment; // collect doc comment information
|
|
|
261 int anyToken; // !=0 means seen at least one token
|
|
|
262 int commentToken; // !=0 means comments are TOKcomment's
|
|
|
263
|
|
|
264 Lexer(Module *mod,
|
|
|
265 unsigned char *base, unsigned begoffset, unsigned endoffset,
|
|
|
266 int doDocComment, int commentToken);
|
|
|
267
|
|
|
268 static void initKeywords();
|
|
|
269 static Identifier *idPool(const char *s);
|
|
159
|
270 static Identifier *uniqueId(const char *s);
|
|
|
271 static Identifier *uniqueId(const char *s, int num);
|
|
1
|
272
|
|
|
273 TOK nextToken();
|
|
|
274 void scan(Token *t);
|
|
|
275 Token *peek(Token *t);
|
|
|
276 Token *peekPastParen(Token *t);
|
|
|
277 unsigned escapeSequence();
|
|
|
278 TOK wysiwygStringConstant(Token *t, int tc);
|
|
|
279 TOK hexStringConstant(Token *t);
|
|
336
|
280 #if DMDV2
|
|
19
|
281 TOK delimitedStringConstant(Token *t);
|
|
|
282 TOK tokenStringConstant(Token *t);
|
|
|
283 #endif
|
|
1
|
284 TOK escapeStringConstant(Token *t, int wide);
|
|
|
285 TOK charConstant(Token *t, int wide);
|
|
|
286 void stringPostfix(Token *t);
|
|
|
287 unsigned wchar(unsigned u);
|
|
|
288 TOK number(Token *t);
|
|
|
289 TOK inreal(Token *t);
|
|
|
290 void error(const char *format, ...);
|
|
|
291 void error(Loc loc, const char *format, ...);
|
|
|
292 void pragma();
|
|
|
293 unsigned decodeUTF();
|
|
|
294 void getDocComment(Token *t, unsigned lineComment);
|
|
|
295
|
|
|
296 static int isValidIdentifier(char *p);
|
|
|
297 static unsigned char *combineComments(unsigned char *c1, unsigned char *c2);
|
|
|
298 };
|
|
|
299
|
|
|
300 #endif /* DMD_LEXER_H */
|
