Mercurial > projects > ldc

comparison dmd2/lexer.h @ 758:f04dde6e882c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added initial D2 support, D2 frontend and changes to codegen to make things compile.
author Tomas Lindquist Olsen <tomas.l.olsen@gmail.com>
date Tue, 11 Nov 2008 01:38:48 +0100
parents
children 638d16625da2
comparison
equal deleted inserted replaced
757:2c730d530c98 758:f04dde6e882c
1
2 // Compiler implementation of the D programming language
3 // Copyright (c) 1999-2008 by Digital Mars
4 // All Rights Reserved
5 // written by Walter Bright
6 // http://www.digitalmars.com
7 // License for redistribution is by either the Artistic License
8 // in artistic.txt, or the GNU General Public License in gnu.txt.
9 // See the included readme.txt for details.
10
11 #ifndef DMD_LEXER_H
12 #define DMD_LEXER_H
13
14 #ifdef __DMC__
15 #pragma once
16 #endif /* __DMC__ */
17
18 #include "root.h"
19 #include "mars.h"
20
21 struct StringTable;
22 struct Identifier;
23 struct Module;
24
25 /* Tokens:
26 ( )
27 [ ]
28 { }
29 < > <= >= == != === !==
30 << >> <<= >>= >>> >>>=
31 + - += -=
32 * / % *= /= %=
33 & | ^ &= |= ^=
34 = ! ~
35 ++ --
36 . -> : ,
37 ? && ||
38 */
39
40 enum TOK
41 {
42 TOKreserved,
43
44 // Other
45 TOKlparen, TOKrparen,
46 TOKlbracket, TOKrbracket,
47 TOKlcurly, TOKrcurly,
48 TOKcolon, TOKneg,
49 TOKsemicolon, TOKdotdotdot,
50 TOKeof, TOKcast,
51 TOKnull, TOKassert,
52 TOKtrue, TOKfalse,
53 TOKarray, TOKcall,
54 TOKaddress, TOKtypedot,
55 TOKtype, TOKthrow,
56 TOKnew, TOKdelete,
57 TOKstar, TOKsymoff,
58 TOKvar, TOKdotvar,
59 TOKdotti, TOKdotexp,
60 TOKdottype, TOKslice,
61 TOKarraylength, TOKversion,
62 TOKmodule, TOKdollar,
63 TOKtemplate, TOKdottd,
64 TOKdeclaration, TOKtypeof,
65 TOKpragma, TOKdsymbol,
66 TOKtypeid, TOKuadd,
67 TOKremove,
68 TOKnewanonclass, TOKcomment,
69 TOKarrayliteral, TOKassocarrayliteral,
70 TOKstructliteral,
71
72 // Operators
73 TOKlt, TOKgt,
74 TOKle, TOKge,
75 TOKequal, TOKnotequal,
76 TOKidentity, TOKnotidentity,
77 TOKindex, TOKis,
78 TOKtobool,
79
80 // 60
81 // NCEG floating point compares
82 // !<>= <> <>= !> !>= !< !<= !<>
83 TOKunord,TOKlg,TOKleg,TOKule,TOKul,TOKuge,TOKug,TOKue,
84
85 TOKshl, TOKshr,
86 TOKshlass, TOKshrass,
87 TOKushr, TOKushrass,
88 TOKcat, TOKcatass, // ~ ~=
89 TOKadd, TOKmin, TOKaddass, TOKminass,
90 TOKmul, TOKdiv, TOKmod,
91 TOKmulass, TOKdivass, TOKmodass,
92 TOKand, TOKor, TOKxor,
93 TOKandass, TOKorass, TOKxorass,
94 TOKassign, TOKnot, TOKtilde,
95 TOKplusplus, TOKminusminus, TOKconstruct, TOKblit,
96 TOKdot, TOKarrow, TOKcomma,
97 TOKquestion, TOKandand, TOKoror,
98
99 // 104
100 // Numeric literals
101 TOKint32v, TOKuns32v,
102 TOKint64v, TOKuns64v,
103 TOKfloat32v, TOKfloat64v, TOKfloat80v,
104 TOKimaginary32v, TOKimaginary64v, TOKimaginary80v,
105
106 // Char constants
107 TOKcharv, TOKwcharv, TOKdcharv,
108
109 // Leaf operators
110 TOKidentifier, TOKstring,
111 TOKthis, TOKsuper,
112 TOKhalt, TOKtuple,
113
114 // Basic types
115 TOKvoid,
116 TOKint8, TOKuns8,
117 TOKint16, TOKuns16,
118 TOKint32, TOKuns32,
119 TOKint64, TOKuns64,
120 TOKfloat32, TOKfloat64, TOKfloat80,
121 TOKimaginary32, TOKimaginary64, TOKimaginary80,
122 TOKcomplex32, TOKcomplex64, TOKcomplex80,
123 TOKchar, TOKwchar, TOKdchar, TOKbit, TOKbool,
124 TOKcent, TOKucent,
125
126 // Aggregates
127 TOKstruct, TOKclass, TOKinterface, TOKunion, TOKenum, TOKimport,
128 TOKtypedef, TOKalias, TOKoverride, TOKdelegate, TOKfunction,
129 TOKmixin,
130
131 TOKalign, TOKextern, TOKprivate, TOKprotected, TOKpublic, TOKexport,
132 TOKstatic, /*TOKvirtual,*/ TOKfinal, TOKconst, TOKabstract, TOKvolatile,
133 TOKdebug, TOKdeprecated, TOKin, TOKout, TOKinout, TOKlazy,
134 TOKauto, TOKpackage, TOKmanifest, TOKimmutable,
135
136 // Statements
137 TOKif, TOKelse, TOKwhile, TOKfor, TOKdo, TOKswitch,
138 TOKcase, TOKdefault, TOKbreak, TOKcontinue, TOKwith,
139 TOKsynchronized, TOKreturn, TOKgoto, TOKtry, TOKcatch, TOKfinally,
140 TOKasm, TOKforeach, TOKforeach_reverse,
141 TOKscope,
142 TOKon_scope_exit, TOKon_scope_failure, TOKon_scope_success,
143
144 // Contracts
145 TOKbody, TOKinvariant,
146
147 // Testing
148 TOKunittest,
149
150 // Added after 1.0
151 TOKref,
152 TOKmacro,
153 #if DMDV2
154 TOKtraits,
155 TOKoverloadset,
156 TOKpure,
157 TOKnothrow,
158 TOKtls,
159 TOKline,
160 TOKfile,
161 TOKshared,
162 #endif
163
164 // LDC specific
165 #if IN_LLVM
166 TOKgep,
167 #endif
168
169 TOKMAX
170 };
171
172 #define CASE_BASIC_TYPES \
173 case TOKwchar: case TOKdchar: \
174 case TOKbit: case TOKbool: case TOKchar: \
175 case TOKint8: case TOKuns8: \
176 case TOKint16: case TOKuns16: \
177 case TOKint32: case TOKuns32: \
178 case TOKint64: case TOKuns64: \
179 case TOKfloat32: case TOKfloat64: case TOKfloat80: \
180 case TOKimaginary32: case TOKimaginary64: case TOKimaginary80: \
181 case TOKcomplex32: case TOKcomplex64: case TOKcomplex80: \
182 case TOKvoid
183
184 #define CASE_BASIC_TYPES_X(t) \
185 case TOKvoid: t = Type::tvoid; goto LabelX; \
186 case TOKint8: t = Type::tint8; goto LabelX; \
187 case TOKuns8: t = Type::tuns8; goto LabelX; \
188 case TOKint16: t = Type::tint16; goto LabelX; \
189 case TOKuns16: t = Type::tuns16; goto LabelX; \
190 case TOKint32: t = Type::tint32; goto LabelX; \
191 case TOKuns32: t = Type::tuns32; goto LabelX; \
192 case TOKint64: t = Type::tint64; goto LabelX; \
193 case TOKuns64: t = Type::tuns64; goto LabelX; \
194 case TOKfloat32: t = Type::tfloat32; goto LabelX; \
195 case TOKfloat64: t = Type::tfloat64; goto LabelX; \
196 case TOKfloat80: t = Type::tfloat80; goto LabelX; \
197 case TOKimaginary32: t = Type::timaginary32; goto LabelX; \
198 case TOKimaginary64: t = Type::timaginary64; goto LabelX; \
199 case TOKimaginary80: t = Type::timaginary80; goto LabelX; \
200 case TOKcomplex32: t = Type::tcomplex32; goto LabelX; \
201 case TOKcomplex64: t = Type::tcomplex64; goto LabelX; \
202 case TOKcomplex80: t = Type::tcomplex80; goto LabelX; \
203 case TOKbit: t = Type::tbit; goto LabelX; \
204 case TOKbool: t = Type::tbool; goto LabelX; \
205 case TOKchar: t = Type::tchar; goto LabelX; \
206 case TOKwchar: t = Type::twchar; goto LabelX; \
207 case TOKdchar: t = Type::tdchar; goto LabelX; \
208 LabelX
209
210 struct Token
211 {
212 Token *next;
213 unsigned char *ptr; // pointer to first character of this token within buffer
214 enum TOK value;
215 unsigned char *blockComment; // doc comment string prior to this token
216 unsigned char *lineComment; // doc comment for previous token
217 union
218 {
219 // Integers
220 d_int32 int32value;
221 d_uns32 uns32value;
222 d_int64 int64value;
223 d_uns64 uns64value;
224
225 // Floats
226 #ifdef IN_GCC
227 // real_t float80value; // can't use this in a union!
228 #else
229 d_float80 float80value;
230 #endif
231
232 struct
233 { unsigned char *ustring; // UTF8 string
234 unsigned len;
235 unsigned char postfix; // 'c', 'w', 'd'
236 };
237
238 Identifier *ident;
239 };
240 #ifdef IN_GCC
241 real_t float80value; // can't use this in a union!
242 #endif
243
244 static const char *tochars[TOKMAX];
245 static void *operator new(size_t sz);
246
247 int isKeyword();
248 void print();
249 const char *toChars();
250 static const char *toChars(enum TOK);
251 };
252
253 struct Lexer
254 {
255 static StringTable stringtable;
256 static OutBuffer stringbuffer;
257 static Token *freelist;
258
259 Loc loc; // for error messages
260
261 unsigned char *base; // pointer to start of buffer
262 unsigned char *end; // past end of buffer
263 unsigned char *p; // current character
264 Token token;
265 Module *mod;
266 int doDocComment; // collect doc comment information
267 int anyToken; // !=0 means seen at least one token
268 int commentToken; // !=0 means comments are TOKcomment's
269
270 Lexer(Module *mod,
271 unsigned char *base, unsigned begoffset, unsigned endoffset,
272 int doDocComment, int commentToken);
273
274 static void initKeywords();
275 static Identifier *idPool(const char *s);
276 static Identifier *uniqueId(const char *s);
277 static Identifier *uniqueId(const char *s, int num);
278
279 TOK nextToken();
280 TOK peekNext();
281 void scan(Token *t);
282 Token *peek(Token *t);
283 Token *peekPastParen(Token *t);
284 unsigned escapeSequence();
285 TOK wysiwygStringConstant(Token *t, int tc);
286 TOK hexStringConstant(Token *t);
287 #if DMDV2
288 TOK delimitedStringConstant(Token *t);
289 TOK tokenStringConstant(Token *t);
290 #endif
291 TOK escapeStringConstant(Token *t, int wide);
292 TOK charConstant(Token *t, int wide);
293 void stringPostfix(Token *t);
294 unsigned wchar(unsigned u);
295 TOK number(Token *t);
296 TOK inreal(Token *t);
297 void error(const char *format, ...);
298 void error(Loc loc, const char *format, ...);
299 void pragma();
300 unsigned decodeUTF();
301 void getDocComment(Token *t, unsigned lineComment);
302
303 static int isValidIdentifier(char *p);
304 static unsigned char *combineComments(unsigned char *c1, unsigned char *c2);
305 };
306
307 #endif /* DMD_LEXER_H */