Mercurial > projects > ldc
annotate dmd/lexer.h @ 650:aa6a0b7968f7
Added test case for bug #100
Removed dubious check for not emitting static private global in other modules without access. This should be handled properly somewhere else, it's causing unresolved global errors for stuff that should work (in MiniD)
| author | Tomas Lindquist Olsen <tomas.l.olsen@gmail.com> |
|---|---|
| date | Sun, 05 Oct 2008 17:28:15 +0200 |
| parents | 4435f57956e7 |
| children | 50383e476c7e |
| rev | line source |
|---|---|
| 1 | 1 |
| 2 // Compiler implementation of the D programming language | |
| 159 | 3 // Copyright (c) 1999-2008 by Digital Mars |
| 1 | 4 // All Rights Reserved |
| 5 // written by Walter Bright | |
| 6 // http://www.digitalmars.com | |
| 7 // License for redistribution is by either the Artistic License | |
| 8 // in artistic.txt, or the GNU General Public License in gnu.txt. | |
| 9 // See the included readme.txt for details. | |
| 10 | |
| 11 #ifndef DMD_LEXER_H | |
| 12 #define DMD_LEXER_H | |
| 13 | |
| 14 #ifdef __DMC__ | |
| 15 #pragma once | |
| 16 #endif /* __DMC__ */ | |
| 17 | |
| 18 #include "root.h" | |
| 19 #include "mars.h" | |
| 20 | |
| 21 struct StringTable; | |
| 22 struct Identifier; | |
| 23 struct Module; | |
| 24 | |
| 25 /* Tokens: | |
| 26 ( ) | |
| 27 [ ] | |
| 28 { } | |
| 29 < > <= >= == != === !== | |
| 30 << >> <<= >>= >>> >>>= | |
| 31 + - += -= | |
| 32 * / % *= /= %= | |
| 33 & | ^ &= |= ^= | |
| 34 = ! ~ | |
| 35 ++ -- | |
| 36 . -> : , | |
| 37 ? && || | |
| 38 */ | |
| 39 | |
| 40 enum TOK | |
| 41 { | |
| 42 TOKreserved, | |
| 43 | |
| 44 // Other | |
| 45 TOKlparen, TOKrparen, | |
| 46 TOKlbracket, TOKrbracket, | |
| 47 TOKlcurly, TOKrcurly, | |
| 48 TOKcolon, TOKneg, | |
| 49 TOKsemicolon, TOKdotdotdot, | |
| 50 TOKeof, TOKcast, | |
| 51 TOKnull, TOKassert, | |
| 52 TOKtrue, TOKfalse, | |
| 53 TOKarray, TOKcall, | |
| 54 TOKaddress, TOKtypedot, | |
| 55 TOKtype, TOKthrow, | |
| 56 TOKnew, TOKdelete, | |
| 57 TOKstar, TOKsymoff, | |
| 58 TOKvar, TOKdotvar, | |
| 59 TOKdotti, TOKdotexp, | |
| 60 TOKdottype, TOKslice, | |
| 61 TOKarraylength, TOKversion, | |
| 62 TOKmodule, TOKdollar, | |
| 63 TOKtemplate, TOKdottd, | |
| 64 TOKdeclaration, TOKtypeof, | |
| 65 TOKpragma, TOKdsymbol, | |
| 66 TOKtypeid, TOKuadd, | |
| 67 TOKremove, | |
| 68 TOKnewanonclass, TOKcomment, | |
| 69 TOKarrayliteral, TOKassocarrayliteral, | |
| 70 TOKstructliteral, | |
| 71 | |
| 72 // Operators | |
| 73 TOKlt, TOKgt, | |
| 74 TOKle, TOKge, | |
| 75 TOKequal, TOKnotequal, | |
| 76 TOKidentity, TOKnotidentity, | |
| 77 TOKindex, TOKis, | |
| 78 TOKtobool, | |
| 79 | |
| 80 // 60 | |
| 81 // NCEG floating point compares | |
| 82 // !<>= <> <>= !> !>= !< !<= !<> | |
| 83 TOKunord,TOKlg,TOKleg,TOKule,TOKul,TOKuge,TOKug,TOKue, | |
| 84 | |
| 85 TOKshl, TOKshr, | |
| 86 TOKshlass, TOKshrass, | |
| 87 TOKushr, TOKushrass, | |
| 88 TOKcat, TOKcatass, // ~ ~= | |
| 89 TOKadd, TOKmin, TOKaddass, TOKminass, | |
| 90 TOKmul, TOKdiv, TOKmod, | |
| 91 TOKmulass, TOKdivass, TOKmodass, | |
| 92 TOKand, TOKor, TOKxor, | |
| 93 TOKandass, TOKorass, TOKxorass, | |
| 94 TOKassign, TOKnot, TOKtilde, | |
| 159 | 95 TOKplusplus, TOKminusminus, TOKconstruct, TOKblit, |
| 1 | 96 TOKdot, TOKarrow, TOKcomma, |
| 97 TOKquestion, TOKandand, TOKoror, | |
| 98 | |
| 159 | 99 // 104 |
| 1 | 100 // Numeric literals |
| 101 TOKint32v, TOKuns32v, | |
| 102 TOKint64v, TOKuns64v, | |
| 103 TOKfloat32v, TOKfloat64v, TOKfloat80v, | |
| 104 TOKimaginary32v, TOKimaginary64v, TOKimaginary80v, | |
| 105 | |
| 106 // Char constants | |
| 107 TOKcharv, TOKwcharv, TOKdcharv, | |
| 108 | |
| 109 // Leaf operators | |
| 110 TOKidentifier, TOKstring, | |
| 111 TOKthis, TOKsuper, | |
| 112 TOKhalt, TOKtuple, | |
| 113 | |
| 114 // Basic types | |
| 115 TOKvoid, | |
| 116 TOKint8, TOKuns8, | |
| 117 TOKint16, TOKuns16, | |
| 118 TOKint32, TOKuns32, | |
| 119 TOKint64, TOKuns64, | |
| 120 TOKfloat32, TOKfloat64, TOKfloat80, | |
| 121 TOKimaginary32, TOKimaginary64, TOKimaginary80, | |
| 122 TOKcomplex32, TOKcomplex64, TOKcomplex80, | |
| 123 TOKchar, TOKwchar, TOKdchar, TOKbit, TOKbool, | |
| 124 TOKcent, TOKucent, | |
| 125 | |
| 126 // Aggregates | |
| 127 TOKstruct, TOKclass, TOKinterface, TOKunion, TOKenum, TOKimport, | |
| 128 TOKtypedef, TOKalias, TOKoverride, TOKdelegate, TOKfunction, | |
| 129 TOKmixin, | |
| 130 | |
| 131 TOKalign, TOKextern, TOKprivate, TOKprotected, TOKpublic, TOKexport, | |
| 132 TOKstatic, /*TOKvirtual,*/ TOKfinal, TOKconst, TOKabstract, TOKvolatile, | |
| 133 TOKdebug, TOKdeprecated, TOKin, TOKout, TOKinout, TOKlazy, | |
| 159 | 134 TOKauto, TOKpackage, TOKmanifest, |
| 1 | 135 |
| 136 // Statements | |
| 137 TOKif, TOKelse, TOKwhile, TOKfor, TOKdo, TOKswitch, | |
| 138 TOKcase, TOKdefault, TOKbreak, TOKcontinue, TOKwith, | |
| 139 TOKsynchronized, TOKreturn, TOKgoto, TOKtry, TOKcatch, TOKfinally, | |
| 140 TOKasm, TOKforeach, TOKforeach_reverse, | |
| 141 TOKscope, | |
| 142 TOKon_scope_exit, TOKon_scope_failure, TOKon_scope_success, | |
| 143 | |
| 144 // Contracts | |
| 145 TOKbody, TOKinvariant, | |
| 146 | |
| 147 // Testing | |
| 148 TOKunittest, | |
| 149 | |
| 150 // Added after 1.0 | |
| 151 TOKref, | |
| 152 TOKmacro, | |
| 336 | 153 #if DMDV2 |
| 19 | 154 TOKtraits, |
| 159 | 155 TOKoverloadset, |
| 156 TOKpure, | |
| 336 | 157 TOKnothrow, |
| 158 TOKtls, | |
| 159 TOKline, | |
| 160 TOKfile, | |
| 19 | 161 #endif |
| 1 | 162 |
|
599
4435f57956e7
Fixed .funcptr property of delegates, no longer uses the infamous DMD rewrites to pointer arithmetic, instead a GEPExp has been introduced.
Tomas Lindquist Olsen <tomas.l.olsen@gmail.com>
parents:
336
diff
changeset
|
163 // LLVMDC specific |
|
4435f57956e7
Fixed .funcptr property of delegates, no longer uses the infamous DMD rewrites to pointer arithmetic, instead a GEPExp has been introduced.
Tomas Lindquist Olsen <tomas.l.olsen@gmail.com>
parents:
336
diff
changeset
|
164 #if IN_LLVM |
|
4435f57956e7
Fixed .funcptr property of delegates, no longer uses the infamous DMD rewrites to pointer arithmetic, instead a GEPExp has been introduced.
Tomas Lindquist Olsen <tomas.l.olsen@gmail.com>
parents:
336
diff
changeset
|
165 TOKgep, |
|
4435f57956e7
Fixed .funcptr property of delegates, no longer uses the infamous DMD rewrites to pointer arithmetic, instead a GEPExp has been introduced.
Tomas Lindquist Olsen <tomas.l.olsen@gmail.com>
parents:
336
diff
changeset
|
166 #endif |
|
4435f57956e7
Fixed .funcptr property of delegates, no longer uses the infamous DMD rewrites to pointer arithmetic, instead a GEPExp has been introduced.
Tomas Lindquist Olsen <tomas.l.olsen@gmail.com>
parents:
336
diff
changeset
|
167 |
| 1 | 168 TOKMAX |
| 169 }; | |
| 170 | |
| 171 #define CASE_BASIC_TYPES \ | |
| 172 case TOKwchar: case TOKdchar: \ | |
| 173 case TOKbit: case TOKbool: case TOKchar: \ | |
| 174 case TOKint8: case TOKuns8: \ | |
| 175 case TOKint16: case TOKuns16: \ | |
| 176 case TOKint32: case TOKuns32: \ | |
| 177 case TOKint64: case TOKuns64: \ | |
| 178 case TOKfloat32: case TOKfloat64: case TOKfloat80: \ | |
| 179 case TOKimaginary32: case TOKimaginary64: case TOKimaginary80: \ | |
| 180 case TOKcomplex32: case TOKcomplex64: case TOKcomplex80: \ | |
| 181 case TOKvoid | |
| 182 | |
| 183 #define CASE_BASIC_TYPES_X(t) \ | |
| 184 case TOKvoid: t = Type::tvoid; goto LabelX; \ | |
| 185 case TOKint8: t = Type::tint8; goto LabelX; \ | |
| 186 case TOKuns8: t = Type::tuns8; goto LabelX; \ | |
| 187 case TOKint16: t = Type::tint16; goto LabelX; \ | |
| 188 case TOKuns16: t = Type::tuns16; goto LabelX; \ | |
| 189 case TOKint32: t = Type::tint32; goto LabelX; \ | |
| 190 case TOKuns32: t = Type::tuns32; goto LabelX; \ | |
| 191 case TOKint64: t = Type::tint64; goto LabelX; \ | |
| 192 case TOKuns64: t = Type::tuns64; goto LabelX; \ | |
| 193 case TOKfloat32: t = Type::tfloat32; goto LabelX; \ | |
| 194 case TOKfloat64: t = Type::tfloat64; goto LabelX; \ | |
| 195 case TOKfloat80: t = Type::tfloat80; goto LabelX; \ | |
| 196 case TOKimaginary32: t = Type::timaginary32; goto LabelX; \ | |
| 197 case TOKimaginary64: t = Type::timaginary64; goto LabelX; \ | |
| 198 case TOKimaginary80: t = Type::timaginary80; goto LabelX; \ | |
| 199 case TOKcomplex32: t = Type::tcomplex32; goto LabelX; \ | |
| 200 case TOKcomplex64: t = Type::tcomplex64; goto LabelX; \ | |
| 201 case TOKcomplex80: t = Type::tcomplex80; goto LabelX; \ | |
| 202 case TOKbit: t = Type::tbit; goto LabelX; \ | |
| 203 case TOKbool: t = Type::tbool; goto LabelX; \ | |
| 204 case TOKchar: t = Type::tchar; goto LabelX; \ | |
| 205 case TOKwchar: t = Type::twchar; goto LabelX; \ | |
| 206 case TOKdchar: t = Type::tdchar; goto LabelX; \ | |
| 207 LabelX | |
| 208 | |
| 209 struct Token | |
| 210 { | |
| 211 Token *next; | |
| 212 unsigned char *ptr; // pointer to first character of this token within buffer | |
| 213 enum TOK value; | |
| 214 unsigned char *blockComment; // doc comment string prior to this token | |
| 215 unsigned char *lineComment; // doc comment for previous token | |
| 216 union | |
| 217 { | |
| 218 // Integers | |
| 219 d_int32 int32value; | |
| 220 d_uns32 uns32value; | |
| 221 d_int64 int64value; | |
| 222 d_uns64 uns64value; | |
| 223 | |
| 224 // Floats | |
| 225 #ifdef IN_GCC | |
| 226 // real_t float80value; // can't use this in a union! | |
| 227 #else | |
| 228 d_float80 float80value; | |
| 229 #endif | |
| 230 | |
| 231 struct | |
| 232 { unsigned char *ustring; // UTF8 string | |
| 233 unsigned len; | |
| 234 unsigned char postfix; // 'c', 'w', 'd' | |
| 235 }; | |
| 236 | |
| 237 Identifier *ident; | |
| 238 }; | |
| 239 #ifdef IN_GCC | |
| 240 real_t float80value; // can't use this in a union! | |
| 241 #endif | |
| 242 | |
| 243 static char *tochars[TOKMAX]; | |
| 244 static void *operator new(size_t sz); | |
| 245 | |
| 246 int isKeyword(); | |
| 247 void print(); | |
| 248 char *toChars(); | |
| 249 static char *toChars(enum TOK); | |
| 250 }; | |
| 251 | |
| 252 struct Lexer | |
| 253 { | |
| 254 static StringTable stringtable; | |
| 255 static OutBuffer stringbuffer; | |
| 256 static Token *freelist; | |
| 257 | |
| 258 Loc loc; // for error messages | |
| 259 | |
| 260 unsigned char *base; // pointer to start of buffer | |
| 261 unsigned char *end; // past end of buffer | |
| 262 unsigned char *p; // current character | |
| 263 Token token; | |
| 264 Module *mod; | |
| 265 int doDocComment; // collect doc comment information | |
| 266 int anyToken; // !=0 means seen at least one token | |
| 267 int commentToken; // !=0 means comments are TOKcomment's | |
| 268 | |
| 269 Lexer(Module *mod, | |
| 270 unsigned char *base, unsigned begoffset, unsigned endoffset, | |
| 271 int doDocComment, int commentToken); | |
| 272 | |
| 273 static void initKeywords(); | |
| 274 static Identifier *idPool(const char *s); | |
| 159 | 275 static Identifier *uniqueId(const char *s); |
| 276 static Identifier *uniqueId(const char *s, int num); | |
| 1 | 277 |
| 278 TOK nextToken(); | |
| 279 void scan(Token *t); | |
| 280 Token *peek(Token *t); | |
| 281 Token *peekPastParen(Token *t); | |
| 282 unsigned escapeSequence(); | |
| 283 TOK wysiwygStringConstant(Token *t, int tc); | |
| 284 TOK hexStringConstant(Token *t); | |
| 336 | 285 #if DMDV2 |
| 19 | 286 TOK delimitedStringConstant(Token *t); |
| 287 TOK tokenStringConstant(Token *t); | |
| 288 #endif | |
| 1 | 289 TOK escapeStringConstant(Token *t, int wide); |
| 290 TOK charConstant(Token *t, int wide); | |
| 291 void stringPostfix(Token *t); | |
| 292 unsigned wchar(unsigned u); | |
| 293 TOK number(Token *t); | |
| 294 TOK inreal(Token *t); | |
| 295 void error(const char *format, ...); | |
| 296 void error(Loc loc, const char *format, ...); | |
| 297 void pragma(); | |
| 298 unsigned decodeUTF(); | |
| 299 void getDocComment(Token *t, unsigned lineComment); | |
| 300 | |
| 301 static int isValidIdentifier(char *p); | |
| 302 static unsigned char *combineComments(unsigned char *c1, unsigned char *c2); | |
| 303 }; | |
| 304 | |
| 305 #endif /* DMD_LEXER_H */ |