|
1
|
1
|
|
|
2 // Compiler implementation of the D programming language
|
|
|
3 // Copyright (c) 1999-2007 by Digital Mars
|
|
|
4 // All Rights Reserved
|
|
|
5 // written by Walter Bright
|
|
|
6 // http://www.digitalmars.com
|
|
|
7 // License for redistribution is by either the Artistic License
|
|
|
8 // in artistic.txt, or the GNU General Public License in gnu.txt.
|
|
|
9 // See the included readme.txt for details.
|
|
|
10
|
|
|
11 #ifndef DMD_LEXER_H
|
|
|
12 #define DMD_LEXER_H
|
|
|
13
|
|
|
14 #ifdef __DMC__
|
|
|
15 #pragma once
|
|
|
16 #endif /* __DMC__ */
|
|
|
17
|
|
|
18 #include "root.h"
|
|
|
19 #include "mars.h"
|
|
|
20
|
|
|
21 struct StringTable;
|
|
|
22 struct Identifier;
|
|
|
23 struct Module;
|
|
|
24
|
|
|
25 /* Tokens:
|
|
|
26 ( )
|
|
|
27 [ ]
|
|
|
28 { }
|
|
|
29 < > <= >= == != === !==
|
|
|
30 << >> <<= >>= >>> >>>=
|
|
|
31 + - += -=
|
|
|
32 * / % *= /= %=
|
|
|
33 & | ^ &= |= ^=
|
|
|
34 = ! ~
|
|
|
35 ++ --
|
|
|
36 . -> : ,
|
|
|
37 ? && ||
|
|
|
38 */
|
|
|
39
|
|
|
40 enum TOK
|
|
|
41 {
|
|
|
42 TOKreserved,
|
|
|
43
|
|
|
44 // Other
|
|
|
45 TOKlparen, TOKrparen,
|
|
|
46 TOKlbracket, TOKrbracket,
|
|
|
47 TOKlcurly, TOKrcurly,
|
|
|
48 TOKcolon, TOKneg,
|
|
|
49 TOKsemicolon, TOKdotdotdot,
|
|
|
50 TOKeof, TOKcast,
|
|
|
51 TOKnull, TOKassert,
|
|
|
52 TOKtrue, TOKfalse,
|
|
|
53 TOKarray, TOKcall,
|
|
|
54 TOKaddress, TOKtypedot,
|
|
|
55 TOKtype, TOKthrow,
|
|
|
56 TOKnew, TOKdelete,
|
|
|
57 TOKstar, TOKsymoff,
|
|
|
58 TOKvar, TOKdotvar,
|
|
|
59 TOKdotti, TOKdotexp,
|
|
|
60 TOKdottype, TOKslice,
|
|
|
61 TOKarraylength, TOKversion,
|
|
|
62 TOKmodule, TOKdollar,
|
|
|
63 TOKtemplate, TOKdottd,
|
|
|
64 TOKdeclaration, TOKtypeof,
|
|
|
65 TOKpragma, TOKdsymbol,
|
|
|
66 TOKtypeid, TOKuadd,
|
|
|
67 TOKremove,
|
|
|
68 TOKnewanonclass, TOKcomment,
|
|
|
69 TOKarrayliteral, TOKassocarrayliteral,
|
|
|
70 TOKstructliteral,
|
|
|
71
|
|
|
72 // Operators
|
|
|
73 TOKlt, TOKgt,
|
|
|
74 TOKle, TOKge,
|
|
|
75 TOKequal, TOKnotequal,
|
|
|
76 TOKidentity, TOKnotidentity,
|
|
|
77 TOKindex, TOKis,
|
|
|
78 TOKtobool,
|
|
|
79
|
|
|
80 // 60
|
|
|
81 // NCEG floating point compares
|
|
|
82 // !<>= <> <>= !> !>= !< !<= !<>
|
|
|
83 TOKunord,TOKlg,TOKleg,TOKule,TOKul,TOKuge,TOKug,TOKue,
|
|
|
84
|
|
|
85 TOKshl, TOKshr,
|
|
|
86 TOKshlass, TOKshrass,
|
|
|
87 TOKushr, TOKushrass,
|
|
|
88 TOKcat, TOKcatass, // ~ ~=
|
|
|
89 TOKadd, TOKmin, TOKaddass, TOKminass,
|
|
|
90 TOKmul, TOKdiv, TOKmod,
|
|
|
91 TOKmulass, TOKdivass, TOKmodass,
|
|
|
92 TOKand, TOKor, TOKxor,
|
|
|
93 TOKandass, TOKorass, TOKxorass,
|
|
|
94 TOKassign, TOKnot, TOKtilde,
|
|
|
95 TOKplusplus, TOKminusminus, TOKconstruct,
|
|
|
96 TOKdot, TOKarrow, TOKcomma,
|
|
|
97 TOKquestion, TOKandand, TOKoror,
|
|
|
98
|
|
|
99 // 103
|
|
|
100 // Numeric literals
|
|
|
101 TOKint32v, TOKuns32v,
|
|
|
102 TOKint64v, TOKuns64v,
|
|
|
103 TOKfloat32v, TOKfloat64v, TOKfloat80v,
|
|
|
104 TOKimaginary32v, TOKimaginary64v, TOKimaginary80v,
|
|
|
105
|
|
|
106 // Char constants
|
|
|
107 TOKcharv, TOKwcharv, TOKdcharv,
|
|
|
108
|
|
|
109 // Leaf operators
|
|
|
110 TOKidentifier, TOKstring,
|
|
|
111 TOKthis, TOKsuper,
|
|
|
112 TOKhalt, TOKtuple,
|
|
|
113
|
|
|
114 // Basic types
|
|
|
115 TOKvoid,
|
|
|
116 TOKint8, TOKuns8,
|
|
|
117 TOKint16, TOKuns16,
|
|
|
118 TOKint32, TOKuns32,
|
|
|
119 TOKint64, TOKuns64,
|
|
|
120 TOKfloat32, TOKfloat64, TOKfloat80,
|
|
|
121 TOKimaginary32, TOKimaginary64, TOKimaginary80,
|
|
|
122 TOKcomplex32, TOKcomplex64, TOKcomplex80,
|
|
|
123 TOKchar, TOKwchar, TOKdchar, TOKbit, TOKbool,
|
|
|
124 TOKcent, TOKucent,
|
|
|
125
|
|
|
126 // Aggregates
|
|
|
127 TOKstruct, TOKclass, TOKinterface, TOKunion, TOKenum, TOKimport,
|
|
|
128 TOKtypedef, TOKalias, TOKoverride, TOKdelegate, TOKfunction,
|
|
|
129 TOKmixin,
|
|
|
130
|
|
|
131 TOKalign, TOKextern, TOKprivate, TOKprotected, TOKpublic, TOKexport,
|
|
|
132 TOKstatic, /*TOKvirtual,*/ TOKfinal, TOKconst, TOKabstract, TOKvolatile,
|
|
|
133 TOKdebug, TOKdeprecated, TOKin, TOKout, TOKinout, TOKlazy,
|
|
|
134 TOKauto, TOKpackage,
|
|
|
135
|
|
|
136 // Statements
|
|
|
137 TOKif, TOKelse, TOKwhile, TOKfor, TOKdo, TOKswitch,
|
|
|
138 TOKcase, TOKdefault, TOKbreak, TOKcontinue, TOKwith,
|
|
|
139 TOKsynchronized, TOKreturn, TOKgoto, TOKtry, TOKcatch, TOKfinally,
|
|
|
140 TOKasm, TOKforeach, TOKforeach_reverse,
|
|
|
141 TOKscope,
|
|
|
142 TOKon_scope_exit, TOKon_scope_failure, TOKon_scope_success,
|
|
|
143
|
|
|
144 // Contracts
|
|
|
145 TOKbody, TOKinvariant,
|
|
|
146
|
|
|
147 // Testing
|
|
|
148 TOKunittest,
|
|
|
149
|
|
|
150 // Added after 1.0
|
|
|
151 TOKref,
|
|
|
152 TOKmacro,
|
|
19
|
153 #if V2
|
|
|
154 TOKtraits,
|
|
|
155 #endif
|
|
1
|
156
|
|
|
157 TOKMAX
|
|
|
158 };
|
|
|
159
|
|
|
160 #define CASE_BASIC_TYPES \
|
|
|
161 case TOKwchar: case TOKdchar: \
|
|
|
162 case TOKbit: case TOKbool: case TOKchar: \
|
|
|
163 case TOKint8: case TOKuns8: \
|
|
|
164 case TOKint16: case TOKuns16: \
|
|
|
165 case TOKint32: case TOKuns32: \
|
|
|
166 case TOKint64: case TOKuns64: \
|
|
|
167 case TOKfloat32: case TOKfloat64: case TOKfloat80: \
|
|
|
168 case TOKimaginary32: case TOKimaginary64: case TOKimaginary80: \
|
|
|
169 case TOKcomplex32: case TOKcomplex64: case TOKcomplex80: \
|
|
|
170 case TOKvoid
|
|
|
171
|
|
|
172 #define CASE_BASIC_TYPES_X(t) \
|
|
|
173 case TOKvoid: t = Type::tvoid; goto LabelX; \
|
|
|
174 case TOKint8: t = Type::tint8; goto LabelX; \
|
|
|
175 case TOKuns8: t = Type::tuns8; goto LabelX; \
|
|
|
176 case TOKint16: t = Type::tint16; goto LabelX; \
|
|
|
177 case TOKuns16: t = Type::tuns16; goto LabelX; \
|
|
|
178 case TOKint32: t = Type::tint32; goto LabelX; \
|
|
|
179 case TOKuns32: t = Type::tuns32; goto LabelX; \
|
|
|
180 case TOKint64: t = Type::tint64; goto LabelX; \
|
|
|
181 case TOKuns64: t = Type::tuns64; goto LabelX; \
|
|
|
182 case TOKfloat32: t = Type::tfloat32; goto LabelX; \
|
|
|
183 case TOKfloat64: t = Type::tfloat64; goto LabelX; \
|
|
|
184 case TOKfloat80: t = Type::tfloat80; goto LabelX; \
|
|
|
185 case TOKimaginary32: t = Type::timaginary32; goto LabelX; \
|
|
|
186 case TOKimaginary64: t = Type::timaginary64; goto LabelX; \
|
|
|
187 case TOKimaginary80: t = Type::timaginary80; goto LabelX; \
|
|
|
188 case TOKcomplex32: t = Type::tcomplex32; goto LabelX; \
|
|
|
189 case TOKcomplex64: t = Type::tcomplex64; goto LabelX; \
|
|
|
190 case TOKcomplex80: t = Type::tcomplex80; goto LabelX; \
|
|
|
191 case TOKbit: t = Type::tbit; goto LabelX; \
|
|
|
192 case TOKbool: t = Type::tbool; goto LabelX; \
|
|
|
193 case TOKchar: t = Type::tchar; goto LabelX; \
|
|
|
194 case TOKwchar: t = Type::twchar; goto LabelX; \
|
|
|
195 case TOKdchar: t = Type::tdchar; goto LabelX; \
|
|
|
196 LabelX
|
|
|
197
|
|
|
198 struct Token
|
|
|
199 {
|
|
|
200 Token *next;
|
|
|
201 unsigned char *ptr; // pointer to first character of this token within buffer
|
|
|
202 enum TOK value;
|
|
|
203 unsigned char *blockComment; // doc comment string prior to this token
|
|
|
204 unsigned char *lineComment; // doc comment for previous token
|
|
|
205 union
|
|
|
206 {
|
|
|
207 // Integers
|
|
|
208 d_int32 int32value;
|
|
|
209 d_uns32 uns32value;
|
|
|
210 d_int64 int64value;
|
|
|
211 d_uns64 uns64value;
|
|
|
212
|
|
|
213 // Floats
|
|
|
214 #ifdef IN_GCC
|
|
|
215 // real_t float80value; // can't use this in a union!
|
|
|
216 #else
|
|
|
217 d_float80 float80value;
|
|
|
218 #endif
|
|
|
219
|
|
|
220 struct
|
|
|
221 { unsigned char *ustring; // UTF8 string
|
|
|
222 unsigned len;
|
|
|
223 unsigned char postfix; // 'c', 'w', 'd'
|
|
|
224 };
|
|
|
225
|
|
|
226 Identifier *ident;
|
|
|
227 };
|
|
|
228 #ifdef IN_GCC
|
|
|
229 real_t float80value; // can't use this in a union!
|
|
|
230 #endif
|
|
|
231
|
|
|
232 static char *tochars[TOKMAX];
|
|
|
233 static void *operator new(size_t sz);
|
|
|
234
|
|
|
235 int isKeyword();
|
|
|
236 void print();
|
|
|
237 char *toChars();
|
|
|
238 static char *toChars(enum TOK);
|
|
|
239 };
|
|
|
240
|
|
|
241 struct Lexer
|
|
|
242 {
|
|
|
243 static StringTable stringtable;
|
|
|
244 static OutBuffer stringbuffer;
|
|
|
245 static Token *freelist;
|
|
|
246
|
|
|
247 Loc loc; // for error messages
|
|
|
248
|
|
|
249 unsigned char *base; // pointer to start of buffer
|
|
|
250 unsigned char *end; // past end of buffer
|
|
|
251 unsigned char *p; // current character
|
|
|
252 Token token;
|
|
|
253 Module *mod;
|
|
|
254 int doDocComment; // collect doc comment information
|
|
|
255 int anyToken; // !=0 means seen at least one token
|
|
|
256 int commentToken; // !=0 means comments are TOKcomment's
|
|
|
257
|
|
|
258 Lexer(Module *mod,
|
|
|
259 unsigned char *base, unsigned begoffset, unsigned endoffset,
|
|
|
260 int doDocComment, int commentToken);
|
|
|
261
|
|
|
262 static void initKeywords();
|
|
|
263 static Identifier *idPool(const char *s);
|
|
|
264
|
|
|
265 TOK nextToken();
|
|
|
266 void scan(Token *t);
|
|
|
267 Token *peek(Token *t);
|
|
|
268 Token *peekPastParen(Token *t);
|
|
|
269 unsigned escapeSequence();
|
|
|
270 TOK wysiwygStringConstant(Token *t, int tc);
|
|
|
271 TOK hexStringConstant(Token *t);
|
|
19
|
272 #if V2
|
|
|
273 TOK delimitedStringConstant(Token *t);
|
|
|
274 TOK tokenStringConstant(Token *t);
|
|
|
275 #endif
|
|
1
|
276 TOK escapeStringConstant(Token *t, int wide);
|
|
|
277 TOK charConstant(Token *t, int wide);
|
|
|
278 void stringPostfix(Token *t);
|
|
|
279 unsigned wchar(unsigned u);
|
|
|
280 TOK number(Token *t);
|
|
|
281 TOK inreal(Token *t);
|
|
|
282 void error(const char *format, ...);
|
|
|
283 void error(Loc loc, const char *format, ...);
|
|
|
284 void pragma();
|
|
|
285 unsigned decodeUTF();
|
|
|
286 void getDocComment(Token *t, unsigned lineComment);
|
|
|
287
|
|
|
288 static int isValidIdentifier(char *p);
|
|
|
289 static unsigned char *combineComments(unsigned char *c1, unsigned char *c2);
|
|
|
290 };
|
|
|
291
|
|
|
292 #endif /* DMD_LEXER_H */
|
