|
1
|
1
|
|
|
2 // Compiler implementation of the D programming language
|
|
159
|
3 // Copyright (c) 1999-2008 by Digital Mars
|
|
1
|
4 // All Rights Reserved
|
|
|
5 // written by Walter Bright
|
|
|
6 // http://www.digitalmars.com
|
|
|
7 // License for redistribution is by either the Artistic License
|
|
|
8 // in artistic.txt, or the GNU General Public License in gnu.txt.
|
|
|
9 // See the included readme.txt for details.
|
|
|
10
|
|
|
11 #ifndef DMD_LEXER_H
|
|
|
12 #define DMD_LEXER_H
|
|
|
13
|
|
|
14 #ifdef __DMC__
|
|
|
15 #pragma once
|
|
|
16 #endif /* __DMC__ */
|
|
|
17
|
|
|
18 #include "root.h"
|
|
|
19 #include "mars.h"
|
|
|
20
|
|
|
21 struct StringTable;
|
|
|
22 struct Identifier;
|
|
|
23 struct Module;
|
|
|
24
|
|
|
25 /* Tokens:
|
|
|
26 ( )
|
|
|
27 [ ]
|
|
|
28 { }
|
|
|
29 < > <= >= == != === !==
|
|
|
30 << >> <<= >>= >>> >>>=
|
|
|
31 + - += -=
|
|
|
32 * / % *= /= %=
|
|
|
33 & | ^ &= |= ^=
|
|
|
34 = ! ~
|
|
|
35 ++ --
|
|
|
36 . -> : ,
|
|
|
37 ? && ||
|
|
|
38 */
|
|
|
39
|
|
|
40 enum TOK
|
|
|
41 {
|
|
|
42 TOKreserved,
|
|
|
43
|
|
|
44 // Other
|
|
|
45 TOKlparen, TOKrparen,
|
|
|
46 TOKlbracket, TOKrbracket,
|
|
|
47 TOKlcurly, TOKrcurly,
|
|
|
48 TOKcolon, TOKneg,
|
|
|
49 TOKsemicolon, TOKdotdotdot,
|
|
|
50 TOKeof, TOKcast,
|
|
|
51 TOKnull, TOKassert,
|
|
|
52 TOKtrue, TOKfalse,
|
|
|
53 TOKarray, TOKcall,
|
|
|
54 TOKaddress, TOKtypedot,
|
|
|
55 TOKtype, TOKthrow,
|
|
|
56 TOKnew, TOKdelete,
|
|
|
57 TOKstar, TOKsymoff,
|
|
|
58 TOKvar, TOKdotvar,
|
|
|
59 TOKdotti, TOKdotexp,
|
|
|
60 TOKdottype, TOKslice,
|
|
|
61 TOKarraylength, TOKversion,
|
|
|
62 TOKmodule, TOKdollar,
|
|
|
63 TOKtemplate, TOKdottd,
|
|
|
64 TOKdeclaration, TOKtypeof,
|
|
|
65 TOKpragma, TOKdsymbol,
|
|
|
66 TOKtypeid, TOKuadd,
|
|
|
67 TOKremove,
|
|
|
68 TOKnewanonclass, TOKcomment,
|
|
|
69 TOKarrayliteral, TOKassocarrayliteral,
|
|
|
70 TOKstructliteral,
|
|
|
71
|
|
|
72 // Operators
|
|
|
73 TOKlt, TOKgt,
|
|
|
74 TOKle, TOKge,
|
|
|
75 TOKequal, TOKnotequal,
|
|
|
76 TOKidentity, TOKnotidentity,
|
|
|
77 TOKindex, TOKis,
|
|
|
78 TOKtobool,
|
|
|
79
|
|
|
80 // 60
|
|
|
81 // NCEG floating point compares
|
|
|
82 // !<>= <> <>= !> !>= !< !<= !<>
|
|
|
83 TOKunord,TOKlg,TOKleg,TOKule,TOKul,TOKuge,TOKug,TOKue,
|
|
|
84
|
|
|
85 TOKshl, TOKshr,
|
|
|
86 TOKshlass, TOKshrass,
|
|
|
87 TOKushr, TOKushrass,
|
|
|
88 TOKcat, TOKcatass, // ~ ~=
|
|
|
89 TOKadd, TOKmin, TOKaddass, TOKminass,
|
|
|
90 TOKmul, TOKdiv, TOKmod,
|
|
|
91 TOKmulass, TOKdivass, TOKmodass,
|
|
|
92 TOKand, TOKor, TOKxor,
|
|
|
93 TOKandass, TOKorass, TOKxorass,
|
|
|
94 TOKassign, TOKnot, TOKtilde,
|
|
159
|
95 TOKplusplus, TOKminusminus, TOKconstruct, TOKblit,
|
|
1
|
96 TOKdot, TOKarrow, TOKcomma,
|
|
|
97 TOKquestion, TOKandand, TOKoror,
|
|
|
98
|
|
159
|
99 // 104
|
|
1
|
100 // Numeric literals
|
|
|
101 TOKint32v, TOKuns32v,
|
|
|
102 TOKint64v, TOKuns64v,
|
|
|
103 TOKfloat32v, TOKfloat64v, TOKfloat80v,
|
|
|
104 TOKimaginary32v, TOKimaginary64v, TOKimaginary80v,
|
|
|
105
|
|
|
106 // Char constants
|
|
|
107 TOKcharv, TOKwcharv, TOKdcharv,
|
|
|
108
|
|
|
109 // Leaf operators
|
|
|
110 TOKidentifier, TOKstring,
|
|
|
111 TOKthis, TOKsuper,
|
|
|
112 TOKhalt, TOKtuple,
|
|
|
113
|
|
|
114 // Basic types
|
|
|
115 TOKvoid,
|
|
|
116 TOKint8, TOKuns8,
|
|
|
117 TOKint16, TOKuns16,
|
|
|
118 TOKint32, TOKuns32,
|
|
|
119 TOKint64, TOKuns64,
|
|
|
120 TOKfloat32, TOKfloat64, TOKfloat80,
|
|
|
121 TOKimaginary32, TOKimaginary64, TOKimaginary80,
|
|
|
122 TOKcomplex32, TOKcomplex64, TOKcomplex80,
|
|
|
123 TOKchar, TOKwchar, TOKdchar, TOKbit, TOKbool,
|
|
|
124 TOKcent, TOKucent,
|
|
|
125
|
|
|
126 // Aggregates
|
|
|
127 TOKstruct, TOKclass, TOKinterface, TOKunion, TOKenum, TOKimport,
|
|
|
128 TOKtypedef, TOKalias, TOKoverride, TOKdelegate, TOKfunction,
|
|
|
129 TOKmixin,
|
|
|
130
|
|
|
131 TOKalign, TOKextern, TOKprivate, TOKprotected, TOKpublic, TOKexport,
|
|
|
132 TOKstatic, /*TOKvirtual,*/ TOKfinal, TOKconst, TOKabstract, TOKvolatile,
|
|
|
133 TOKdebug, TOKdeprecated, TOKin, TOKout, TOKinout, TOKlazy,
|
|
159
|
134 TOKauto, TOKpackage, TOKmanifest,
|
|
1
|
135
|
|
|
136 // Statements
|
|
|
137 TOKif, TOKelse, TOKwhile, TOKfor, TOKdo, TOKswitch,
|
|
|
138 TOKcase, TOKdefault, TOKbreak, TOKcontinue, TOKwith,
|
|
|
139 TOKsynchronized, TOKreturn, TOKgoto, TOKtry, TOKcatch, TOKfinally,
|
|
|
140 TOKasm, TOKforeach, TOKforeach_reverse,
|
|
|
141 TOKscope,
|
|
|
142 TOKon_scope_exit, TOKon_scope_failure, TOKon_scope_success,
|
|
|
143
|
|
|
144 // Contracts
|
|
|
145 TOKbody, TOKinvariant,
|
|
|
146
|
|
|
147 // Testing
|
|
|
148 TOKunittest,
|
|
|
149
|
|
|
150 // Added after 1.0
|
|
|
151 TOKref,
|
|
|
152 TOKmacro,
|
|
19
|
153 #if V2
|
|
|
154 TOKtraits,
|
|
159
|
155 TOKoverloadset,
|
|
|
156 TOKpure,
|
|
19
|
157 #endif
|
|
1
|
158
|
|
|
159 TOKMAX
|
|
|
160 };
|
|
|
161
|
|
|
162 #define CASE_BASIC_TYPES \
|
|
|
163 case TOKwchar: case TOKdchar: \
|
|
|
164 case TOKbit: case TOKbool: case TOKchar: \
|
|
|
165 case TOKint8: case TOKuns8: \
|
|
|
166 case TOKint16: case TOKuns16: \
|
|
|
167 case TOKint32: case TOKuns32: \
|
|
|
168 case TOKint64: case TOKuns64: \
|
|
|
169 case TOKfloat32: case TOKfloat64: case TOKfloat80: \
|
|
|
170 case TOKimaginary32: case TOKimaginary64: case TOKimaginary80: \
|
|
|
171 case TOKcomplex32: case TOKcomplex64: case TOKcomplex80: \
|
|
|
172 case TOKvoid
|
|
|
173
|
|
|
174 #define CASE_BASIC_TYPES_X(t) \
|
|
|
175 case TOKvoid: t = Type::tvoid; goto LabelX; \
|
|
|
176 case TOKint8: t = Type::tint8; goto LabelX; \
|
|
|
177 case TOKuns8: t = Type::tuns8; goto LabelX; \
|
|
|
178 case TOKint16: t = Type::tint16; goto LabelX; \
|
|
|
179 case TOKuns16: t = Type::tuns16; goto LabelX; \
|
|
|
180 case TOKint32: t = Type::tint32; goto LabelX; \
|
|
|
181 case TOKuns32: t = Type::tuns32; goto LabelX; \
|
|
|
182 case TOKint64: t = Type::tint64; goto LabelX; \
|
|
|
183 case TOKuns64: t = Type::tuns64; goto LabelX; \
|
|
|
184 case TOKfloat32: t = Type::tfloat32; goto LabelX; \
|
|
|
185 case TOKfloat64: t = Type::tfloat64; goto LabelX; \
|
|
|
186 case TOKfloat80: t = Type::tfloat80; goto LabelX; \
|
|
|
187 case TOKimaginary32: t = Type::timaginary32; goto LabelX; \
|
|
|
188 case TOKimaginary64: t = Type::timaginary64; goto LabelX; \
|
|
|
189 case TOKimaginary80: t = Type::timaginary80; goto LabelX; \
|
|
|
190 case TOKcomplex32: t = Type::tcomplex32; goto LabelX; \
|
|
|
191 case TOKcomplex64: t = Type::tcomplex64; goto LabelX; \
|
|
|
192 case TOKcomplex80: t = Type::tcomplex80; goto LabelX; \
|
|
|
193 case TOKbit: t = Type::tbit; goto LabelX; \
|
|
|
194 case TOKbool: t = Type::tbool; goto LabelX; \
|
|
|
195 case TOKchar: t = Type::tchar; goto LabelX; \
|
|
|
196 case TOKwchar: t = Type::twchar; goto LabelX; \
|
|
|
197 case TOKdchar: t = Type::tdchar; goto LabelX; \
|
|
|
198 LabelX
|
|
|
199
|
|
|
200 struct Token
|
|
|
201 {
|
|
|
202 Token *next;
|
|
|
203 unsigned char *ptr; // pointer to first character of this token within buffer
|
|
|
204 enum TOK value;
|
|
|
205 unsigned char *blockComment; // doc comment string prior to this token
|
|
|
206 unsigned char *lineComment; // doc comment for previous token
|
|
|
207 union
|
|
|
208 {
|
|
|
209 // Integers
|
|
|
210 d_int32 int32value;
|
|
|
211 d_uns32 uns32value;
|
|
|
212 d_int64 int64value;
|
|
|
213 d_uns64 uns64value;
|
|
|
214
|
|
|
215 // Floats
|
|
|
216 #ifdef IN_GCC
|
|
|
217 // real_t float80value; // can't use this in a union!
|
|
|
218 #else
|
|
|
219 d_float80 float80value;
|
|
|
220 #endif
|
|
|
221
|
|
|
222 struct
|
|
|
223 { unsigned char *ustring; // UTF8 string
|
|
|
224 unsigned len;
|
|
|
225 unsigned char postfix; // 'c', 'w', 'd'
|
|
|
226 };
|
|
|
227
|
|
|
228 Identifier *ident;
|
|
|
229 };
|
|
|
230 #ifdef IN_GCC
|
|
|
231 real_t float80value; // can't use this in a union!
|
|
|
232 #endif
|
|
|
233
|
|
|
234 static char *tochars[TOKMAX];
|
|
|
235 static void *operator new(size_t sz);
|
|
|
236
|
|
|
237 int isKeyword();
|
|
|
238 void print();
|
|
|
239 char *toChars();
|
|
|
240 static char *toChars(enum TOK);
|
|
|
241 };
|
|
|
242
|
|
|
243 struct Lexer
|
|
|
244 {
|
|
|
245 static StringTable stringtable;
|
|
|
246 static OutBuffer stringbuffer;
|
|
|
247 static Token *freelist;
|
|
|
248
|
|
|
249 Loc loc; // for error messages
|
|
|
250
|
|
|
251 unsigned char *base; // pointer to start of buffer
|
|
|
252 unsigned char *end; // past end of buffer
|
|
|
253 unsigned char *p; // current character
|
|
|
254 Token token;
|
|
|
255 Module *mod;
|
|
|
256 int doDocComment; // collect doc comment information
|
|
|
257 int anyToken; // !=0 means seen at least one token
|
|
|
258 int commentToken; // !=0 means comments are TOKcomment's
|
|
|
259
|
|
|
260 Lexer(Module *mod,
|
|
|
261 unsigned char *base, unsigned begoffset, unsigned endoffset,
|
|
|
262 int doDocComment, int commentToken);
|
|
|
263
|
|
|
264 static void initKeywords();
|
|
|
265 static Identifier *idPool(const char *s);
|
|
159
|
266 static Identifier *uniqueId(const char *s);
|
|
|
267 static Identifier *uniqueId(const char *s, int num);
|
|
1
|
268
|
|
|
269 TOK nextToken();
|
|
|
270 void scan(Token *t);
|
|
|
271 Token *peek(Token *t);
|
|
|
272 Token *peekPastParen(Token *t);
|
|
|
273 unsigned escapeSequence();
|
|
|
274 TOK wysiwygStringConstant(Token *t, int tc);
|
|
|
275 TOK hexStringConstant(Token *t);
|
|
19
|
276 #if V2
|
|
|
277 TOK delimitedStringConstant(Token *t);
|
|
|
278 TOK tokenStringConstant(Token *t);
|
|
|
279 #endif
|
|
1
|
280 TOK escapeStringConstant(Token *t, int wide);
|
|
|
281 TOK charConstant(Token *t, int wide);
|
|
|
282 void stringPostfix(Token *t);
|
|
|
283 unsigned wchar(unsigned u);
|
|
|
284 TOK number(Token *t);
|
|
|
285 TOK inreal(Token *t);
|
|
|
286 void error(const char *format, ...);
|
|
|
287 void error(Loc loc, const char *format, ...);
|
|
|
288 void pragma();
|
|
|
289 unsigned decodeUTF();
|
|
|
290 void getDocComment(Token *t, unsigned lineComment);
|
|
|
291
|
|
|
292 static int isValidIdentifier(char *p);
|
|
|
293 static unsigned char *combineComments(unsigned char *c1, unsigned char *c2);
|
|
|
294 };
|
|
|
295
|
|
|
296 #endif /* DMD_LEXER_H */
|
