Mercurial > projects > ddmd
annotate dmd/Lexer.d @ 178:e3afd1303184
Many small bugs fixed
Made all classes derive from TObject to detect memory leaks (functionality is disabled for now)
Began work on overriding backend memory allocations (to avoid memory leaks)
| author | korDen |
|---|---|
| date | Sun, 17 Oct 2010 07:42:00 +0400 |
| parents | 94b6033c07f3 |
| children |
| rev | line source |
|---|---|
| 0 | 1 module dmd.Lexer; |
| 2 | |
| 114 | 3 import dmd.common; |
| 0 | 4 import dmd.StringTable; |
| 5 import dmd.OutBuffer; | |
| 6 import dmd.Token; | |
| 7 import dmd.Loc; | |
| 8 import dmd.Module; | |
| 9 import dmd.Identifier; | |
| 10 import dmd.TOK; | |
| 11 import dmd.Keyword; | |
| 12 import dmd.StringValue; | |
| 13 import dmd.Global; | |
| 14 import dmd.Util; | |
| 15 import dmd.Id; | |
| 16 import dmd.Dchar; | |
| 17 import dmd.Utf; | |
| 18 | |
| 19 import std.stdio : writeln; | |
| 20 | |
| 4 | 21 import core.memory; |
| 2 | 22 |
| 0 | 23 import core.stdc.ctype; |
| 24 import core.stdc.stdlib; | |
| 25 import core.stdc.string; | |
| 26 import core.stdc.errno; | |
| 27 | |
| 28 enum LS = 0x2028; // UTF line separator | |
| 29 enum PS = 0x2029; // UTF paragraph separator | |
| 30 | |
| 31 extern (C) extern | |
| 32 { | |
| 33 __gshared char* __locale_decpoint; | |
| 34 } | |
| 35 | |
| 162 | 36 bool isUniAlpha(uint u) |
| 0 | 37 { |
|
168
ceed63f310fb
stringtable, stringbuffer and freelist moved to Global
korDen
parents:
163
diff
changeset
|
38 enum ushort table[][2] = |
| 162 | 39 [ |
| 40 [ 0x00AA, 0x00AA ], | |
| 41 [ 0x00B5, 0x00B5 ], | |
| 42 [ 0x00B7, 0x00B7 ], | |
| 43 [ 0x00BA, 0x00BA ], | |
| 44 [ 0x00C0, 0x00D6 ], | |
| 45 [ 0x00D8, 0x00F6 ], | |
| 46 [ 0x00F8, 0x01F5 ], | |
| 47 [ 0x01FA, 0x0217 ], | |
| 48 [ 0x0250, 0x02A8 ], | |
| 49 [ 0x02B0, 0x02B8 ], | |
| 50 [ 0x02BB, 0x02BB ], | |
| 51 [ 0x02BD, 0x02C1 ], | |
| 52 [ 0x02D0, 0x02D1 ], | |
| 53 [ 0x02E0, 0x02E4 ], | |
| 54 [ 0x037A, 0x037A ], | |
| 55 [ 0x0386, 0x0386 ], | |
| 56 [ 0x0388, 0x038A ], | |
| 57 [ 0x038C, 0x038C ], | |
| 58 [ 0x038E, 0x03A1 ], | |
| 59 [ 0x03A3, 0x03CE ], | |
| 60 [ 0x03D0, 0x03D6 ], | |
| 61 [ 0x03DA, 0x03DA ], | |
| 62 [ 0x03DC, 0x03DC ], | |
| 63 [ 0x03DE, 0x03DE ], | |
| 64 [ 0x03E0, 0x03E0 ], | |
| 65 [ 0x03E2, 0x03F3 ], | |
| 66 [ 0x0401, 0x040C ], | |
| 67 [ 0x040E, 0x044F ], | |
| 68 [ 0x0451, 0x045C ], | |
| 69 [ 0x045E, 0x0481 ], | |
| 70 [ 0x0490, 0x04C4 ], | |
| 71 [ 0x04C7, 0x04C8 ], | |
| 72 [ 0x04CB, 0x04CC ], | |
| 73 [ 0x04D0, 0x04EB ], | |
| 74 [ 0x04EE, 0x04F5 ], | |
| 75 [ 0x04F8, 0x04F9 ], | |
| 76 [ 0x0531, 0x0556 ], | |
| 77 [ 0x0559, 0x0559 ], | |
| 78 [ 0x0561, 0x0587 ], | |
| 79 [ 0x05B0, 0x05B9 ], | |
| 80 [ 0x05BB, 0x05BD ], | |
| 81 [ 0x05BF, 0x05BF ], | |
| 82 [ 0x05C1, 0x05C2 ], | |
| 83 [ 0x05D0, 0x05EA ], | |
| 84 [ 0x05F0, 0x05F2 ], | |
| 85 [ 0x0621, 0x063A ], | |
| 86 [ 0x0640, 0x0652 ], | |
| 87 [ 0x0660, 0x0669 ], | |
| 88 [ 0x0670, 0x06B7 ], | |
| 89 [ 0x06BA, 0x06BE ], | |
| 90 [ 0x06C0, 0x06CE ], | |
| 91 [ 0x06D0, 0x06DC ], | |
| 92 [ 0x06E5, 0x06E8 ], | |
| 93 [ 0x06EA, 0x06ED ], | |
| 94 [ 0x06F0, 0x06F9 ], | |
| 95 [ 0x0901, 0x0903 ], | |
| 96 [ 0x0905, 0x0939 ], | |
| 97 [ 0x093D, 0x093D ], | |
| 98 [ 0x093E, 0x094D ], | |
| 99 [ 0x0950, 0x0952 ], | |
| 100 [ 0x0958, 0x0963 ], | |
| 101 [ 0x0966, 0x096F ], | |
| 102 [ 0x0981, 0x0983 ], | |
| 103 [ 0x0985, 0x098C ], | |
| 104 [ 0x098F, 0x0990 ], | |
| 105 [ 0x0993, 0x09A8 ], | |
| 106 [ 0x09AA, 0x09B0 ], | |
| 107 [ 0x09B2, 0x09B2 ], | |
| 108 [ 0x09B6, 0x09B9 ], | |
| 109 [ 0x09BE, 0x09C4 ], | |
| 110 [ 0x09C7, 0x09C8 ], | |
| 111 [ 0x09CB, 0x09CD ], | |
| 112 [ 0x09DC, 0x09DD ], | |
| 113 [ 0x09DF, 0x09E3 ], | |
| 114 [ 0x09E6, 0x09EF ], | |
| 115 [ 0x09F0, 0x09F1 ], | |
| 116 [ 0x0A02, 0x0A02 ], | |
| 117 [ 0x0A05, 0x0A0A ], | |
| 118 [ 0x0A0F, 0x0A10 ], | |
| 119 [ 0x0A13, 0x0A28 ], | |
| 120 [ 0x0A2A, 0x0A30 ], | |
| 121 [ 0x0A32, 0x0A33 ], | |
| 122 [ 0x0A35, 0x0A36 ], | |
| 123 [ 0x0A38, 0x0A39 ], | |
| 124 [ 0x0A3E, 0x0A42 ], | |
| 125 [ 0x0A47, 0x0A48 ], | |
| 126 [ 0x0A4B, 0x0A4D ], | |
| 127 [ 0x0A59, 0x0A5C ], | |
| 128 [ 0x0A5E, 0x0A5E ], | |
| 129 [ 0x0A66, 0x0A6F ], | |
| 130 [ 0x0A74, 0x0A74 ], | |
| 131 [ 0x0A81, 0x0A83 ], | |
| 132 [ 0x0A85, 0x0A8B ], | |
| 133 [ 0x0A8D, 0x0A8D ], | |
| 134 [ 0x0A8F, 0x0A91 ], | |
| 135 [ 0x0A93, 0x0AA8 ], | |
| 136 [ 0x0AAA, 0x0AB0 ], | |
| 137 [ 0x0AB2, 0x0AB3 ], | |
| 138 [ 0x0AB5, 0x0AB9 ], | |
| 139 [ 0x0ABD, 0x0AC5 ], | |
| 140 [ 0x0AC7, 0x0AC9 ], | |
| 141 [ 0x0ACB, 0x0ACD ], | |
| 142 [ 0x0AD0, 0x0AD0 ], | |
| 143 [ 0x0AE0, 0x0AE0 ], | |
| 144 [ 0x0AE6, 0x0AEF ], | |
| 145 [ 0x0B01, 0x0B03 ], | |
| 146 [ 0x0B05, 0x0B0C ], | |
| 147 [ 0x0B0F, 0x0B10 ], | |
| 148 [ 0x0B13, 0x0B28 ], | |
| 149 [ 0x0B2A, 0x0B30 ], | |
| 150 [ 0x0B32, 0x0B33 ], | |
| 151 [ 0x0B36, 0x0B39 ], | |
| 152 [ 0x0B3D, 0x0B3D ], | |
| 153 [ 0x0B3E, 0x0B43 ], | |
| 154 [ 0x0B47, 0x0B48 ], | |
| 155 [ 0x0B4B, 0x0B4D ], | |
| 156 [ 0x0B5C, 0x0B5D ], | |
| 157 [ 0x0B5F, 0x0B61 ], | |
| 158 [ 0x0B66, 0x0B6F ], | |
| 159 [ 0x0B82, 0x0B83 ], | |
| 160 [ 0x0B85, 0x0B8A ], | |
| 161 [ 0x0B8E, 0x0B90 ], | |
| 162 [ 0x0B92, 0x0B95 ], | |
| 163 [ 0x0B99, 0x0B9A ], | |
| 164 [ 0x0B9C, 0x0B9C ], | |
| 165 [ 0x0B9E, 0x0B9F ], | |
| 166 [ 0x0BA3, 0x0BA4 ], | |
| 167 [ 0x0BA8, 0x0BAA ], | |
| 168 [ 0x0BAE, 0x0BB5 ], | |
| 169 [ 0x0BB7, 0x0BB9 ], | |
| 170 [ 0x0BBE, 0x0BC2 ], | |
| 171 [ 0x0BC6, 0x0BC8 ], | |
| 172 [ 0x0BCA, 0x0BCD ], | |
| 173 [ 0x0BE7, 0x0BEF ], | |
| 174 [ 0x0C01, 0x0C03 ], | |
| 175 [ 0x0C05, 0x0C0C ], | |
| 176 [ 0x0C0E, 0x0C10 ], | |
| 177 [ 0x0C12, 0x0C28 ], | |
| 178 [ 0x0C2A, 0x0C33 ], | |
| 179 [ 0x0C35, 0x0C39 ], | |
| 180 [ 0x0C3E, 0x0C44 ], | |
| 181 [ 0x0C46, 0x0C48 ], | |
| 182 [ 0x0C4A, 0x0C4D ], | |
| 183 [ 0x0C60, 0x0C61 ], | |
| 184 [ 0x0C66, 0x0C6F ], | |
| 185 [ 0x0C82, 0x0C83 ], | |
| 186 [ 0x0C85, 0x0C8C ], | |
| 187 [ 0x0C8E, 0x0C90 ], | |
| 188 [ 0x0C92, 0x0CA8 ], | |
| 189 [ 0x0CAA, 0x0CB3 ], | |
| 190 [ 0x0CB5, 0x0CB9 ], | |
| 191 [ 0x0CBE, 0x0CC4 ], | |
| 192 [ 0x0CC6, 0x0CC8 ], | |
| 193 [ 0x0CCA, 0x0CCD ], | |
| 194 [ 0x0CDE, 0x0CDE ], | |
| 195 [ 0x0CE0, 0x0CE1 ], | |
| 196 [ 0x0CE6, 0x0CEF ], | |
| 197 [ 0x0D02, 0x0D03 ], | |
| 198 [ 0x0D05, 0x0D0C ], | |
| 199 [ 0x0D0E, 0x0D10 ], | |
| 200 [ 0x0D12, 0x0D28 ], | |
| 201 [ 0x0D2A, 0x0D39 ], | |
| 202 [ 0x0D3E, 0x0D43 ], | |
| 203 [ 0x0D46, 0x0D48 ], | |
| 204 [ 0x0D4A, 0x0D4D ], | |
| 205 [ 0x0D60, 0x0D61 ], | |
| 206 [ 0x0D66, 0x0D6F ], | |
| 207 [ 0x0E01, 0x0E3A ], | |
| 208 [ 0x0E40, 0x0E5B ], | |
| 209 // { 0x0E50, 0x0E59 }, | |
| 210 [ 0x0E81, 0x0E82 ], | |
| 211 [ 0x0E84, 0x0E84 ], | |
| 212 [ 0x0E87, 0x0E88 ], | |
| 213 [ 0x0E8A, 0x0E8A ], | |
| 214 [ 0x0E8D, 0x0E8D ], | |
| 215 [ 0x0E94, 0x0E97 ], | |
| 216 [ 0x0E99, 0x0E9F ], | |
| 217 [ 0x0EA1, 0x0EA3 ], | |
| 218 [ 0x0EA5, 0x0EA5 ], | |
| 219 [ 0x0EA7, 0x0EA7 ], | |
| 220 [ 0x0EAA, 0x0EAB ], | |
| 221 [ 0x0EAD, 0x0EAE ], | |
| 222 [ 0x0EB0, 0x0EB9 ], | |
| 223 [ 0x0EBB, 0x0EBD ], | |
| 224 [ 0x0EC0, 0x0EC4 ], | |
| 225 [ 0x0EC6, 0x0EC6 ], | |
| 226 [ 0x0EC8, 0x0ECD ], | |
| 227 [ 0x0ED0, 0x0ED9 ], | |
| 228 [ 0x0EDC, 0x0EDD ], | |
| 229 [ 0x0F00, 0x0F00 ], | |
| 230 [ 0x0F18, 0x0F19 ], | |
| 231 [ 0x0F20, 0x0F33 ], | |
| 232 [ 0x0F35, 0x0F35 ], | |
| 233 [ 0x0F37, 0x0F37 ], | |
| 234 [ 0x0F39, 0x0F39 ], | |
| 235 [ 0x0F3E, 0x0F47 ], | |
| 236 [ 0x0F49, 0x0F69 ], | |
| 237 [ 0x0F71, 0x0F84 ], | |
| 238 [ 0x0F86, 0x0F8B ], | |
| 239 [ 0x0F90, 0x0F95 ], | |
| 240 [ 0x0F97, 0x0F97 ], | |
| 241 [ 0x0F99, 0x0FAD ], | |
| 242 [ 0x0FB1, 0x0FB7 ], | |
| 243 [ 0x0FB9, 0x0FB9 ], | |
| 244 [ 0x10A0, 0x10C5 ], | |
| 245 [ 0x10D0, 0x10F6 ], | |
| 246 [ 0x1E00, 0x1E9B ], | |
| 247 [ 0x1EA0, 0x1EF9 ], | |
| 248 [ 0x1F00, 0x1F15 ], | |
| 249 [ 0x1F18, 0x1F1D ], | |
| 250 [ 0x1F20, 0x1F45 ], | |
| 251 [ 0x1F48, 0x1F4D ], | |
| 252 [ 0x1F50, 0x1F57 ], | |
| 253 [ 0x1F59, 0x1F59 ], | |
| 254 [ 0x1F5B, 0x1F5B ], | |
| 255 [ 0x1F5D, 0x1F5D ], | |
| 256 [ 0x1F5F, 0x1F7D ], | |
| 257 [ 0x1F80, 0x1FB4 ], | |
| 258 [ 0x1FB6, 0x1FBC ], | |
| 259 [ 0x1FBE, 0x1FBE ], | |
| 260 [ 0x1FC2, 0x1FC4 ], | |
| 261 [ 0x1FC6, 0x1FCC ], | |
| 262 [ 0x1FD0, 0x1FD3 ], | |
| 263 [ 0x1FD6, 0x1FDB ], | |
| 264 [ 0x1FE0, 0x1FEC ], | |
| 265 [ 0x1FF2, 0x1FF4 ], | |
| 266 [ 0x1FF6, 0x1FFC ], | |
| 267 [ 0x203F, 0x2040 ], | |
| 268 [ 0x207F, 0x207F ], | |
| 269 [ 0x2102, 0x2102 ], | |
| 270 [ 0x2107, 0x2107 ], | |
| 271 [ 0x210A, 0x2113 ], | |
| 272 [ 0x2115, 0x2115 ], | |
| 273 [ 0x2118, 0x211D ], | |
| 274 [ 0x2124, 0x2124 ], | |
| 275 [ 0x2126, 0x2126 ], | |
| 276 [ 0x2128, 0x2128 ], | |
| 277 [ 0x212A, 0x2131 ], | |
| 278 [ 0x2133, 0x2138 ], | |
| 279 [ 0x2160, 0x2182 ], | |
| 280 [ 0x3005, 0x3007 ], | |
| 281 [ 0x3021, 0x3029 ], | |
| 282 [ 0x3041, 0x3093 ], | |
| 283 [ 0x309B, 0x309C ], | |
| 284 [ 0x30A1, 0x30F6 ], | |
| 285 [ 0x30FB, 0x30FC ], | |
| 286 [ 0x3105, 0x312C ], | |
| 287 [ 0x4E00, 0x9FA5 ], | |
| 288 [ 0xAC00, 0xD7A3 ], | |
| 289 ]; | |
| 290 | |
| 291 debug { | |
| 292 for (int i = 0; i < table.length; i++) | |
| 293 { | |
| 294 //printf("%x\n", table[i][0]); | |
| 295 assert(table[i][0] <= table[i][1]); | |
| 296 if (i < table.length - 1) | |
| 297 assert(table[i][1] < table[i + 1][0]); | |
| 298 } | |
| 299 } | |
| 300 | |
| 301 if (u > 0xD7A3) | |
| 302 goto Lisnot; | |
| 303 | |
| 304 // Binary search | |
| 305 int mid; | |
| 306 int low; | |
| 307 int high; | |
| 308 | |
| 309 low = 0; | |
| 310 high = table.length - 1; | |
| 311 while (low <= high) | |
| 312 { | |
| 313 mid = (low + high) >> 1; | |
| 314 if (u < table[mid][0]) | |
| 315 high = mid - 1; | |
| 316 else if (u > table[mid][1]) | |
| 317 low = mid + 1; | |
| 318 else | |
| 319 goto Lis; | |
| 320 } | |
| 321 | |
| 322 Lisnot: | |
| 323 debug { | |
| 324 for (int i = 0; i < table.length; i++) | |
| 325 { | |
| 326 assert(u < table[i][0] || u > table[i][1]); | |
| 327 } | |
| 328 } | |
| 329 return false; | |
| 330 | |
| 331 Lis: | |
| 332 debug { | |
| 333 for (int i = 0; i < table.length; i++) | |
| 334 { | |
| 335 if (u >= table[i][0] && u <= table[i][1]) | |
| 336 return 1; | |
| 337 } | |
| 338 assert(0); // should have been in table | |
| 339 } | |
| 340 return true; | |
| 0 | 341 } |
| 342 | |
| 178 | 343 import dmd.TObject; |
| 344 | |
| 345 class Lexer : TObject | |
| 0 | 346 { |
| 347 Loc loc; // for error messages | |
| 348 | |
| 349 ubyte* base; // pointer to start of buffer | |
| 350 ubyte* end; // past end of buffer | |
| 351 ubyte* p; // current character | |
| 352 Token token; | |
| 353 Module mod; | |
| 354 int doDocComment; // collect doc comment information | |
| 355 int anyToken; // !=0 means seen at least one token | |
| 356 int commentToken; // !=0 means comments are TOKcomment's | |
| 357 | |
| 358 this(Module mod, ubyte* base, uint begoffset, uint endoffset, int doDocComment, int commentToken) | |
| 359 { | |
| 178 | 360 register(); |
| 0 | 361 loc = Loc(mod, 1); |
| 178 | 362 |
| 0 | 363 memset(&token,0,token.sizeof); |
| 364 this.base = base; | |
| 365 this.end = base + endoffset; | |
| 366 p = base + begoffset; | |
| 367 this.mod = mod; | |
| 368 this.doDocComment = doDocComment; | |
| 369 this.anyToken = 0; | |
| 370 this.commentToken = commentToken; | |
| 371 //initKeywords(); | |
| 372 | |
| 373 /* If first line starts with '#!', ignore the line | |
| 374 */ | |
| 375 | |
| 376 if (p[0] == '#' && p[1] =='!') | |
| 377 { | |
| 378 p += 2; | |
| 379 while (1) | |
| 178 | 380 { |
| 0 | 381 ubyte c = *p; |
| 382 switch (c) | |
| 383 { | |
| 384 case '\n': | |
| 385 p++; | |
| 386 break; | |
| 387 | |
| 388 case '\r': | |
| 389 p++; | |
| 390 if (*p == '\n') | |
| 391 p++; | |
| 392 break; | |
| 393 | |
| 394 case 0: | |
| 395 case 0x1A: | |
| 396 break; | |
| 397 | |
| 398 default: | |
| 399 if (c & 0x80) | |
| 400 { | |
| 401 uint u = decodeUTF(); | |
| 402 if (u == PS || u == LS) | |
| 403 break; | |
| 404 } | |
| 405 p++; | |
| 406 continue; | |
| 407 } | |
| 408 break; | |
| 409 } | |
| 410 loc.linnum = 2; | |
| 411 } | |
| 412 } | |
| 413 | |
| 414 version (DMDV2) { | |
| 169 | 415 enum Keyword[] keywords = |
| 0 | 416 [ |
| 417 // { "", TOK }, | |
| 418 | |
| 419 { "this", TOK.TOKthis }, | |
| 420 { "super", TOK.TOKsuper }, | |
| 421 { "assert", TOK.TOKassert }, | |
| 422 { "null", TOK.TOKnull }, | |
| 423 { "true", TOK.TOKtrue }, | |
| 424 { "false", TOK.TOKfalse }, | |
| 425 { "cast", TOK.TOKcast }, | |
| 426 { "new", TOK.TOKnew }, | |
| 427 { "delete", TOK.TOKdelete }, | |
| 428 { "throw", TOK.TOKthrow }, | |
| 429 { "module", TOK.TOKmodule }, | |
| 430 { "pragma", TOK.TOKpragma }, | |
| 431 { "typeof", TOK.TOKtypeof }, | |
| 432 { "typeid", TOK.TOKtypeid }, | |
| 433 | |
| 434 { "template", TOK.TOKtemplate }, | |
| 435 | |
| 436 { "void", TOK.TOKvoid }, | |
| 437 { "byte", TOK.TOKint8 }, | |
| 438 { "ubyte", TOK.TOKuns8 }, | |
| 439 { "short", TOK.TOKint16 }, | |
| 440 { "ushort", TOK.TOKuns16 }, | |
| 441 { "int", TOK.TOKint32 }, | |
| 442 { "uint", TOK.TOKuns32 }, | |
| 443 { "long", TOK.TOKint64 }, | |
| 444 { "ulong", TOK.TOKuns64 }, | |
| 445 { "cent", TOK.TOKcent, }, | |
| 446 { "ucent", TOK.TOKucent, }, | |
| 447 { "float", TOK.TOKfloat32 }, | |
| 448 { "double", TOK.TOKfloat64 }, | |
| 449 { "real", TOK.TOKfloat80 }, | |
| 450 | |
| 451 { "bool", TOK.TOKbool }, | |
| 452 { "char", TOK.TOKchar }, | |
| 453 { "wchar", TOK.TOKwchar }, | |
| 454 { "dchar", TOK.TOKdchar }, | |
| 455 | |
| 456 { "ifloat", TOK.TOKimaginary32 }, | |
| 457 { "idouble", TOK.TOKimaginary64 }, | |
| 458 { "ireal", TOK.TOKimaginary80 }, | |
| 459 | |
| 460 { "cfloat", TOK.TOKcomplex32 }, | |
| 461 { "cdouble", TOK.TOKcomplex64 }, | |
| 462 { "creal", TOK.TOKcomplex80 }, | |
| 463 | |
| 464 { "delegate", TOK.TOKdelegate }, | |
| 465 { "function", TOK.TOKfunction }, | |
| 466 | |
| 467 { "is", TOK.TOKis }, | |
| 468 { "if", TOK.TOKif }, | |
| 469 { "else", TOK.TOKelse }, | |
| 470 { "while", TOK.TOKwhile }, | |
| 471 { "for", TOK.TOKfor }, | |
| 472 { "do", TOK.TOKdo }, | |
| 473 { "switch", TOK.TOKswitch }, | |
| 474 { "case", TOK.TOKcase }, | |
| 475 { "default", TOK.TOKdefault }, | |
| 476 { "break", TOK.TOKbreak }, | |
| 477 { "continue", TOK.TOKcontinue }, | |
| 478 { "synchronized", TOK.TOKsynchronized }, | |
| 479 { "return", TOK.TOKreturn }, | |
| 480 { "goto", TOK.TOKgoto }, | |
| 481 { "try", TOK.TOKtry }, | |
| 482 { "catch", TOK.TOKcatch }, | |
| 483 { "finally", TOK.TOKfinally }, | |
| 484 { "with", TOK.TOKwith }, | |
| 485 { "asm", TOK.TOKasm }, | |
| 486 { "foreach", TOK.TOKforeach }, | |
| 487 { "foreach_reverse", TOK.TOKforeach_reverse }, | |
| 488 { "scope", TOK.TOKscope }, | |
| 489 | |
| 490 { "struct", TOK.TOKstruct }, | |
| 491 { "class", TOK.TOKclass }, | |
| 492 { "interface", TOK.TOKinterface }, | |
| 493 { "union", TOK.TOKunion }, | |
| 494 { "enum", TOK.TOKenum }, | |
| 495 { "import", TOK.TOKimport }, | |
| 496 { "mixin", TOK.TOKmixin }, | |
| 497 { "static", TOK.TOKstatic }, | |
| 498 { "final", TOK.TOKfinal }, | |
| 499 { "const", TOK.TOKconst }, | |
| 500 { "typedef", TOK.TOKtypedef }, | |
| 501 { "alias", TOK.TOKalias }, | |
| 502 { "override", TOK.TOKoverride }, | |
| 503 { "abstract", TOK.TOKabstract }, | |
| 504 { "volatile", TOK.TOKvolatile }, | |
| 505 { "debug", TOK.TOKdebug }, | |
| 506 { "deprecated", TOK.TOKdeprecated }, | |
| 507 { "in", TOK.TOKin }, | |
| 508 { "out", TOK.TOKout }, | |
| 509 { "inout", TOK.TOKinout }, | |
| 510 { "lazy", TOK.TOKlazy }, | |
| 511 { "auto", TOK.TOKauto }, | |
| 512 | |
| 513 { "align", TOK.TOKalign }, | |
| 514 { "extern", TOK.TOKextern }, | |
| 515 { "private", TOK.TOKprivate }, | |
| 516 { "package", TOK.TOKpackage }, | |
| 517 { "protected", TOK.TOKprotected }, | |
| 518 { "public", TOK.TOKpublic }, | |
| 519 { "export", TOK.TOKexport }, | |
| 520 | |
| 521 { "body", TOK.TOKbody }, | |
| 522 { "invariant", TOK.TOKinvariant }, | |
| 523 { "unittest", TOK.TOKunittest }, | |
| 524 { "version", TOK.TOKversion }, | |
| 525 //{ "manifest", TOK.TOKmanifest }, | |
| 526 | |
| 527 // Added after 1.0 | |
| 528 { "ref", TOK.TOKref }, | |
| 529 { "macro", TOK.TOKmacro }, | |
| 530 { "pure", TOK.TOKpure }, | |
| 531 { "nothrow", TOK.TOKnothrow }, | |
| 532 { "__thread", TOK.TOKtls }, | |
| 533 { "__gshared", TOK.TOKgshared }, | |
| 534 { "__traits", TOK.TOKtraits }, | |
| 535 { "__overloadset", TOK.TOKoverloadset }, | |
| 536 { "__FILE__", TOK.TOKfile }, | |
| 537 { "__LINE__", TOK.TOKline }, | |
| 538 { "shared", TOK.TOKshared }, | |
| 539 { "immutable", TOK.TOKimmutable }, | |
| 540 ]; | |
| 541 } else { | |
| 169 | 542 enum Keyword[] keywords = |
| 0 | 543 [ |
| 544 // { "", TOK }, | |
| 545 | |
| 546 { "this", TOK.TOKthis }, | |
| 547 { "super", TOK.TOKsuper }, | |
| 548 { "assert", TOK.TOKassert }, | |
| 549 { "null", TOK.TOKnull }, | |
| 550 { "true", TOK.TOKtrue }, | |
| 551 { "false", TOK.TOKfalse }, | |
| 552 { "cast", TOK.TOKcast }, | |
| 553 { "new", TOK.TOKnew }, | |
| 554 { "delete", TOK.TOKdelete }, | |
| 555 { "throw", TOK.TOKthrow }, | |
| 556 { "module", TOK.TOKmodule }, | |
| 557 { "pragma", TOK.TOKpragma }, | |
| 558 { "typeof", TOK.TOKtypeof }, | |
| 559 { "typeid", TOK.TOKtypeid }, | |
| 560 | |
| 561 { "template", TOK.TOKtemplate }, | |
| 562 | |
| 563 { "void", TOK.TOKvoid }, | |
| 564 { "byte", TOK.TOKint8 }, | |
| 565 { "ubyte", TOK.TOKuns8 }, | |
| 566 { "short", TOK.TOKint16 }, | |
| 567 { "ushort", TOK.TOKuns16 }, | |
| 568 { "int", TOK.TOKint32 }, | |
| 569 { "uint", TOK.TOKuns32 }, | |
| 570 { "long", TOK.TOKint64 }, | |
| 571 { "ulong", TOK.TOKuns64 }, | |
| 572 { "cent", TOK.TOKcent, }, | |
| 573 { "ucent", TOK.TOKucent, }, | |
| 574 { "float", TOK.TOKfloat32 }, | |
| 575 { "double", TOK.TOKfloat64 }, | |
| 576 { "real", TOK.TOKfloat80 }, | |
| 577 | |
| 578 { "bool", TOK.TOKbool }, | |
| 579 { "char", TOK.TOKchar }, | |
| 580 { "wchar", TOK.TOKwchar }, | |
| 581 { "dchar", TOK.TOKdchar }, | |
| 582 | |
| 583 { "ifloat", TOK.TOKimaginary32 }, | |
| 584 { "idouble", TOK.TOKimaginary64 }, | |
| 585 { "ireal", TOK.TOKimaginary80 }, | |
| 586 | |
| 587 { "cfloat", TOK.TOKcomplex32 }, | |
| 588 { "cdouble", TOK.TOKcomplex64 }, | |
| 589 { "creal", TOK.TOKcomplex80 }, | |
| 590 | |
| 591 { "delegate", TOK.TOKdelegate }, | |
| 592 { "function", TOK.TOKfunction }, | |
| 593 | |
| 594 { "is", TOK.TOKis }, | |
| 595 { "if", TOK.TOKif }, | |
| 596 { "else", TOK.TOKelse }, | |
| 597 { "while", TOK.TOKwhile }, | |
| 598 { "for", TOK.TOKfor }, | |
| 599 { "do", TOK.TOKdo }, | |
| 600 { "switch", TOK.TOKswitch }, | |
| 601 { "case", TOK.TOKcase }, | |
| 602 { "default", TOK.TOKdefault }, | |
| 603 { "break", TOK.TOKbreak }, | |
| 604 { "continue", TOK.TOKcontinue }, | |
| 605 { "synchronized", TOK.TOKsynchronized }, | |
| 606 { "return", TOK.TOKreturn }, | |
| 607 { "goto", TOK.TOKgoto }, | |
| 608 { "try", TOK.TOKtry }, | |
| 609 { "catch", TOK.TOKcatch }, | |
| 610 { "finally", TOK.TOKfinally }, | |
| 611 { "with", TOK.TOKwith }, | |
| 612 { "asm", TOK.TOKasm }, | |
| 613 { "foreach", TOK.TOKforeach }, | |
| 614 { "foreach_reverse", TOK.TOKforeach_reverse }, | |
| 615 { "scope", TOK.TOKscope }, | |
| 616 | |
| 617 { "struct", TOK.TOKstruct }, | |
| 618 { "class", TOK.TOKclass }, | |
| 619 { "interface", TOK.TOKinterface }, | |
| 620 { "union", TOK.TOKunion }, | |
| 621 { "enum", TOK.TOKenum }, | |
| 622 { "import", TOK.TOKimport }, | |
| 623 { "mixin", TOK.TOKmixin }, | |
| 624 { "static", TOK.TOKstatic }, | |
| 625 { "final", TOK.TOKfinal }, | |
| 626 { "const", TOK.TOKconst }, | |
| 627 { "typedef", TOK.TOKtypedef }, | |
| 628 { "alias", TOK.TOKalias }, | |
| 629 { "override", TOK.TOKoverride }, | |
| 630 { "abstract", TOK.TOKabstract }, | |
| 631 { "volatile", TOK.TOKvolatile }, | |
| 632 { "debug", TOK.TOKdebug }, | |
| 633 { "deprecated", TOK.TOKdeprecated }, | |
| 634 { "in", TOK.TOKin }, | |
| 635 { "out", TOK.TOKout }, | |
| 636 { "inout", TOK.TOKinout }, | |
| 637 { "lazy", TOK.TOKlazy }, | |
| 638 { "auto", TOK.TOKauto }, | |
| 639 | |
| 640 { "align", TOK.TOKalign }, | |
| 641 { "extern", TOK.TOKextern }, | |
| 642 { "private", TOK.TOKprivate }, | |
| 643 { "package", TOK.TOKpackage }, | |
| 644 { "protected", TOK.TOKprotected }, | |
| 645 { "public", TOK.TOKpublic }, | |
| 646 { "export", TOK.TOKexport }, | |
| 647 | |
| 648 { "body", TOK.TOKbody }, | |
| 649 { "invariant", TOK.TOKinvariant }, | |
| 650 { "unittest", TOK.TOKunittest }, | |
| 651 { "version", TOK.TOKversion }, | |
| 652 //{ "manifest", TOK.TOKmanifest }, | |
| 653 | |
| 654 // Added after 1.0 | |
| 655 { "ref", TOK.TOKref }, | |
| 656 { "macro", TOK.TOKmacro }, | |
| 657 ]; | |
| 658 } | |
| 659 | |
| 174 | 660 static __gshared ubyte cmtable[256]; |
| 0 | 661 enum CMoctal = 0x1; |
| 662 enum CMhex = 0x2; | |
| 663 enum CMidchar = 0x4; | |
| 178 | 664 |
| 0 | 665 ubyte isoctal (ubyte c) { return cmtable[c] & CMoctal; } |
| 666 ubyte ishex (ubyte c) { return cmtable[c] & CMhex; } | |
| 667 ubyte isidchar(ubyte c) { return cmtable[c] & CMidchar; } | |
| 668 | |
| 669 static void cmtable_init() | |
| 670 { | |
| 671 for (uint c = 0; c < cmtable.length; c++) | |
| 672 { | |
| 673 if ('0' <= c && c <= '7') | |
| 674 cmtable[c] |= CMoctal; | |
| 675 if (isdigit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F')) | |
| 676 cmtable[c] |= CMhex; | |
| 677 if (isalnum(c) || c == '_') | |
| 678 cmtable[c] |= CMidchar; | |
| 679 } | |
| 680 } | |
| 178 | 681 |
| 682 static ref StringTable stringtable() | |
|
168
ceed63f310fb
stringtable, stringbuffer and freelist moved to Global
korDen
parents:
163
diff
changeset
|
683 { |
|
ceed63f310fb
stringtable, stringbuffer and freelist moved to Global
korDen
parents:
163
diff
changeset
|
684 return global.stringtable; |
|
ceed63f310fb
stringtable, stringbuffer and freelist moved to Global
korDen
parents:
163
diff
changeset
|
685 } |
| 178 | 686 |
|
168
ceed63f310fb
stringtable, stringbuffer and freelist moved to Global
korDen
parents:
163
diff
changeset
|
687 static OutBuffer stringbuffer() |
|
ceed63f310fb
stringtable, stringbuffer and freelist moved to Global
korDen
parents:
163
diff
changeset
|
688 { |
|
ceed63f310fb
stringtable, stringbuffer and freelist moved to Global
korDen
parents:
163
diff
changeset
|
689 return global.stringbuffer; |
|
ceed63f310fb
stringtable, stringbuffer and freelist moved to Global
korDen
parents:
163
diff
changeset
|
690 } |
| 178 | 691 |
| 0 | 692 static void initKeywords() |
| 693 { | |
| 694 uint nkeywords = keywords.length; | |
| 695 | |
| 696 if (global.params.Dversion == 1) | |
| 697 nkeywords -= 2; | |
| 698 | |
| 699 cmtable_init(); | |
| 178 | 700 |
| 0 | 701 for (uint u = 0; u < nkeywords; u++) |
| 702 { | |
| 34 | 703 //printf("keyword[%d] = '%.*s'\n",u, keywords[u].name); |
| 0 | 704 string s = keywords[u].name; |
| 705 TOK v = keywords[u].value; | |
| 178 | 706 Object* sv = stringtable.insert(s); |
| 707 *sv = new Identifier(s, v); | |
| 0 | 708 |
| 709 //printf("tochars[%d] = '%s'\n",v, s); | |
| 710 Token.tochars[v] = s; | |
| 711 } | |
| 712 | |
| 713 Token.tochars[TOK.TOKeof] = "EOF"; | |
| 714 Token.tochars[TOK.TOKlcurly] = "{"; | |
| 715 Token.tochars[TOK.TOKrcurly] = "}"; | |
| 716 Token.tochars[TOK.TOKlparen] = "("; | |
| 717 Token.tochars[TOK.TOKrparen] = ")"; | |
| 718 Token.tochars[TOK.TOKlbracket] = "["; | |
| 719 Token.tochars[TOK.TOKrbracket] = "]"; | |
| 720 Token.tochars[TOK.TOKsemicolon] = ";"; | |
| 721 Token.tochars[TOK.TOKcolon] = ":"; | |
| 722 Token.tochars[TOK.TOKcomma] = ","; | |
| 723 Token.tochars[TOK.TOKdot] = "."; | |
| 724 Token.tochars[TOK.TOKxor] = "^"; | |
| 725 Token.tochars[TOK.TOKxorass] = "^="; | |
| 726 Token.tochars[TOK.TOKassign] = "="; | |
| 727 Token.tochars[TOK.TOKconstruct] = "="; | |
| 728 version (DMDV2) { | |
| 729 Token.tochars[TOK.TOKblit] = "="; | |
| 730 } | |
| 731 Token.tochars[TOK.TOKlt] = "<"; | |
| 732 Token.tochars[TOK.TOKgt] = ">"; | |
| 733 Token.tochars[TOK.TOKle] = "<="; | |
| 734 Token.tochars[TOK.TOKge] = ">="; | |
| 735 Token.tochars[TOK.TOKequal] = "=="; | |
| 736 Token.tochars[TOK.TOKnotequal] = "!="; | |
| 737 Token.tochars[TOK.TOKnotidentity] = "!is"; | |
| 738 Token.tochars[TOK.TOKtobool] = "!!"; | |
| 739 | |
| 740 Token.tochars[TOK.TOKunord] = "!<>="; | |
| 741 Token.tochars[TOK.TOKue] = "!<>"; | |
| 742 Token.tochars[TOK.TOKlg] = "<>"; | |
| 743 Token.tochars[TOK.TOKleg] = "<>="; | |
| 744 Token.tochars[TOK.TOKule] = "!>"; | |
| 745 Token.tochars[TOK.TOKul] = "!>="; | |
| 746 Token.tochars[TOK.TOKuge] = "!<"; | |
| 747 Token.tochars[TOK.TOKug] = "!<="; | |
| 748 | |
| 749 Token.tochars[TOK.TOKnot] = "!"; | |
| 750 Token.tochars[TOK.TOKtobool] = "!!"; | |
| 751 Token.tochars[TOK.TOKshl] = "<<"; | |
| 752 Token.tochars[TOK.TOKshr] = ">>"; | |
| 753 Token.tochars[TOK.TOKushr] = ">>>"; | |
| 754 Token.tochars[TOK.TOKadd] = "+"; | |
| 755 Token.tochars[TOK.TOKmin] = "-"; | |
| 756 Token.tochars[TOK.TOKmul] = "*"; | |
| 757 Token.tochars[TOK.TOKdiv] = "/"; | |
| 758 Token.tochars[TOK.TOKmod] = "%"; | |
| 759 Token.tochars[TOK.TOKslice] = ".."; | |
| 760 Token.tochars[TOK.TOKdotdotdot] = "..."; | |
| 761 Token.tochars[TOK.TOKand] = "&"; | |
| 762 Token.tochars[TOK.TOKandand] = "&&"; | |
| 763 Token.tochars[TOK.TOKor] = "|"; | |
| 764 Token.tochars[TOK.TOKoror] = "||"; | |
| 765 Token.tochars[TOK.TOKarray] = "[]"; | |
| 766 Token.tochars[TOK.TOKindex] = "[i]"; | |
| 767 Token.tochars[TOK.TOKaddress] = "&"; | |
| 768 Token.tochars[TOK.TOKstar] = "*"; | |
| 769 Token.tochars[TOK.TOKtilde] = "~"; | |
| 770 Token.tochars[TOK.TOKdollar] = "$"; | |
| 771 Token.tochars[TOK.TOKcast] = "cast"; | |
| 772 Token.tochars[TOK.TOKplusplus] = "++"; | |
| 773 Token.tochars[TOK.TOKminusminus] = "--"; | |
| 774 Token.tochars[TOK.TOKtype] = "type"; | |
| 775 Token.tochars[TOK.TOKquestion] = "?"; | |
| 776 Token.tochars[TOK.TOKneg] = "-"; | |
| 777 Token.tochars[TOK.TOKuadd] = "+"; | |
| 778 Token.tochars[TOK.TOKvar] = "var"; | |
| 779 Token.tochars[TOK.TOKaddass] = "+="; | |
| 780 Token.tochars[TOK.TOKminass] = "-="; | |
| 781 Token.tochars[TOK.TOKmulass] = "*="; | |
| 782 Token.tochars[TOK.TOKdivass] = "/="; | |
| 783 Token.tochars[TOK.TOKmodass] = "%="; | |
| 784 Token.tochars[TOK.TOKshlass] = "<<="; | |
| 785 Token.tochars[TOK.TOKshrass] = ">>="; | |
| 786 Token.tochars[TOK.TOKushrass] = ">>>="; | |
| 787 Token.tochars[TOK.TOKandass] = "&="; | |
| 788 Token.tochars[TOK.TOKorass] = "|="; | |
| 789 Token.tochars[TOK.TOKcatass] = "~="; | |
| 790 Token.tochars[TOK.TOKcat] = "~"; | |
| 791 Token.tochars[TOK.TOKcall] = "call"; | |
| 792 Token.tochars[TOK.TOKidentity] = "is"; | |
| 793 Token.tochars[TOK.TOKnotidentity] = "!is"; | |
| 794 | |
| 795 Token.tochars[TOK.TOKorass] = "|="; | |
| 796 Token.tochars[TOK.TOKidentifier] = "identifier"; | |
| 797 Token.tochars[TOK.TOKat] = "@"; | |
|
130
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
798 Token.tochars[TOK.TOKpow] = "^^"; |
| 135 | 799 Token.tochars[TOK.TOKpowass] = "^^="; |
| 178 | 800 |
| 0 | 801 // For debugging |
| 73 | 802 Token.tochars[TOKerror] = "error"; |
| 0 | 803 Token.tochars[TOK.TOKdotexp] = "dotexp"; |
| 804 Token.tochars[TOK.TOKdotti] = "dotti"; | |
| 805 Token.tochars[TOK.TOKdotvar] = "dotvar"; | |
| 806 Token.tochars[TOK.TOKdottype] = "dottype"; | |
| 807 Token.tochars[TOK.TOKsymoff] = "symoff"; | |
| 808 Token.tochars[TOK.TOKarraylength] = "arraylength"; | |
| 809 Token.tochars[TOK.TOKarrayliteral] = "arrayliteral"; | |
| 810 Token.tochars[TOK.TOKassocarrayliteral] = "assocarrayliteral"; | |
| 811 Token.tochars[TOK.TOKstructliteral] = "structliteral"; | |
| 812 Token.tochars[TOK.TOKstring] = "string"; | |
| 813 Token.tochars[TOK.TOKdsymbol] = "symbol"; | |
| 814 Token.tochars[TOK.TOKtuple] = "tuple"; | |
| 815 Token.tochars[TOK.TOKdeclaration] = "declaration"; | |
| 816 Token.tochars[TOK.TOKdottd] = "dottd"; | |
| 817 Token.tochars[TOK.TOKon_scope_exit] = "scope(exit)"; | |
| 818 Token.tochars[TOK.TOKon_scope_success] = "scope(success)"; | |
| 819 Token.tochars[TOK.TOKon_scope_failure] = "scope(failure)"; | |
| 820 } | |
| 821 | |
| 822 static Identifier idPool(string s) | |
| 823 { | |
| 178 | 824 Object* sv = stringtable.update(s); |
| 825 Identifier id = cast(Identifier) *sv; | |
| 0 | 826 if (id is null) |
| 827 { | |
| 178 | 828 id = new Identifier(s, TOK.TOKidentifier); |
| 829 *sv = id; | |
| 0 | 830 } |
| 831 | |
| 832 return id; | |
| 833 } | |
| 834 | |
| 835 static Identifier uniqueId(string s) | |
| 836 { | |
|
168
ceed63f310fb
stringtable, stringbuffer and freelist moved to Global
korDen
parents:
163
diff
changeset
|
837 return uniqueId(s, ++global.num); |
| 0 | 838 } |
| 839 | |
| 840 /********************************************* | |
| 841 * Create a unique identifier using the prefix s. | |
| 842 */ | |
| 843 static Identifier uniqueId(string s, int num) | |
| 844 { | |
| 845 char buffer[32]; | |
| 846 size_t slen = s.length; | |
| 847 | |
| 848 assert(slen + num.sizeof * 3 + 1 <= buffer.sizeof); | |
| 849 int len = sprintf(buffer.ptr, "%.*s%d", s, num); | |
| 850 | |
| 851 return idPool(buffer[0..len].idup); | |
| 852 } | |
| 853 | |
| 854 TOK nextToken() | |
| 855 { | |
| 856 Token *t; | |
| 857 | |
| 858 if (token.next) | |
| 859 { | |
| 860 t = token.next; | |
| 861 memcpy(&token, t, Token.sizeof); | |
|
168
ceed63f310fb
stringtable, stringbuffer and freelist moved to Global
korDen
parents:
163
diff
changeset
|
862 t.next = global.freelist; |
|
ceed63f310fb
stringtable, stringbuffer and freelist moved to Global
korDen
parents:
163
diff
changeset
|
863 global.freelist = t; |
| 0 | 864 } |
| 865 else | |
| 866 { | |
| 867 scan(&token); | |
| 868 } | |
| 869 | |
| 870 //token.print(); | |
| 871 return token.value; | |
| 872 } | |
| 873 | |
| 874 /*********************** | |
| 875 * Look ahead at next token's value. | |
| 876 */ | |
| 877 TOK peekNext() | |
| 878 { | |
| 879 return peek(&token).value; | |
| 880 } | |
| 881 | |
| 163 | 882 /*********************** |
| 883 * Look 2 tokens ahead at value. | |
| 884 */ | |
| 0 | 885 TOK peekNext2() |
| 886 { | |
| 163 | 887 Token* t = peek(&token); |
| 888 return peek(t).value; | |
| 0 | 889 } |
| 890 | |
| 891 void scan(Token* t) | |
| 892 { | |
| 893 uint lastLine = loc.linnum; | |
| 894 uint linnum; | |
| 895 | |
| 896 t.blockComment = null; | |
| 897 t.lineComment = null; | |
| 898 while (1) | |
| 899 { | |
| 900 t.ptr = p; | |
| 901 //printf("p = %p, *p = '%c'\n",p,*p); | |
| 902 switch (*p) | |
| 903 { | |
| 904 case 0: | |
| 905 case 0x1A: | |
| 906 t.value = TOK.TOKeof; // end of file | |
| 907 return; | |
| 908 | |
| 909 case ' ': | |
| 910 case '\t': | |
| 911 case '\v': | |
| 912 case '\f': | |
| 913 p++; | |
| 914 continue; // skip white space | |
| 915 | |
| 916 case '\r': | |
| 917 p++; | |
| 918 if (*p != '\n') // if CR stands by itself | |
| 919 loc.linnum++; | |
| 920 continue; // skip white space | |
| 921 | |
| 922 case '\n': | |
| 923 p++; | |
| 924 loc.linnum++; | |
| 925 continue; // skip white space | |
| 926 | |
| 927 case '0': case '1': case '2': case '3': case '4': | |
| 928 case '5': case '6': case '7': case '8': case '9': | |
| 929 t.value = number(t); | |
| 930 return; | |
| 931 | |
| 932 version (CSTRINGS) { | |
| 933 case '\'': | |
| 934 t.value = charConstant(t, 0); | |
| 935 return; | |
| 936 | |
| 937 case '"': | |
| 938 t.value = stringConstant(t,0); | |
| 939 return; | |
| 940 | |
| 941 case 'l': | |
| 942 case 'L': | |
| 943 if (p[1] == '\'') | |
| 944 { | |
| 945 p++; | |
| 946 t.value = charConstant(t, 1); | |
| 947 return; | |
| 948 } | |
| 949 else if (p[1] == '"') | |
| 950 { | |
| 951 p++; | |
| 952 t.value = stringConstant(t, 1); | |
| 953 return; | |
| 954 } | |
| 955 } else { | |
| 956 case '\'': | |
| 957 t.value = charConstant(t,0); | |
| 958 return; | |
| 959 | |
| 960 case 'r': | |
| 961 if (p[1] != '"') | |
| 962 goto case_ident; | |
| 963 p++; | |
| 964 case '`': | |
| 965 t.value = wysiwygStringConstant(t, *p); | |
| 966 return; | |
| 967 | |
| 968 case 'x': | |
| 969 if (p[1] != '"') | |
| 970 goto case_ident; | |
| 971 p++; | |
| 972 t.value = hexStringConstant(t); | |
| 973 return; | |
| 974 | |
| 975 version (DMDV2) { | |
| 976 case 'q': | |
| 977 if (p[1] == '"') | |
| 978 { | |
| 979 p++; | |
| 980 t.value = delimitedStringConstant(t); | |
| 981 return; | |
| 982 } | |
| 983 else if (p[1] == '{') | |
| 984 { | |
| 985 p++; | |
| 986 t.value = tokenStringConstant(t); | |
| 987 return; | |
| 988 } | |
| 989 else | |
| 990 goto case_ident; | |
| 991 } | |
| 992 | |
| 993 case '"': | |
| 994 t.value = escapeStringConstant(t,0); | |
| 995 return; | |
| 996 version (TEXTUAL_ASSEMBLY_OUT) { | |
| 997 } else { | |
| 998 case '\\': // escaped string literal | |
| 999 { uint c; | |
| 1000 ubyte* pstart = p; | |
| 1001 | |
| 1002 stringbuffer.reset(); | |
| 1003 do | |
| 1004 { | |
| 1005 p++; | |
| 1006 switch (*p) | |
| 1007 { | |
| 1008 case 'u': | |
| 1009 case 'U': | |
| 1010 case '&': | |
| 1011 c = escapeSequence(); | |
| 1012 stringbuffer.writeUTF8(c); | |
| 1013 break; | |
| 1014 | |
| 1015 default: | |
| 1016 c = escapeSequence(); | |
| 1017 stringbuffer.writeByte(c); | |
| 1018 break; | |
| 1019 } | |
| 1020 } while (*p == '\\'); | |
| 1021 t.len = stringbuffer.offset; | |
| 1022 stringbuffer.writeByte(0); | |
| 2 | 1023 char* cc = cast(char*)GC.malloc(stringbuffer.offset); |
| 0 | 1024 memcpy(cc, stringbuffer.data, stringbuffer.offset); |
| 1025 t.ustring = cc; | |
| 1026 t.postfix = 0; | |
| 1027 t.value = TOK.TOKstring; | |
| 1028 if (!global.params.useDeprecated) | |
| 1029 error("Escape String literal %.*s is deprecated, use double quoted string literal \"%.*s\" instead", p - pstart, pstart, p - pstart, pstart); | |
| 1030 return; | |
| 1031 } | |
| 1032 } | |
| 1033 case 'l': | |
| 1034 case 'L': | |
| 1035 } | |
| 1036 case 'a': case 'b': case 'c': case 'd': case 'e': | |
| 1037 case 'f': case 'g': case 'h': case 'i': case 'j': | |
| 1038 case 'k': case 'm': case 'n': case 'o': | |
| 1039 version (DMDV2) { | |
| 1040 case 'p': /*case 'q': case 'r':*/ case 's': case 't': | |
| 1041 } else { | |
| 1042 case 'p': case 'q': /*case 'r':*/ case 's': case 't': | |
| 1043 } | |
| 1044 case 'u': case 'v': case 'w': /*case 'x':*/ case 'y': | |
| 1045 case 'z': | |
| 1046 case 'A': case 'B': case 'C': case 'D': case 'E': | |
| 1047 case 'F': case 'G': case 'H': case 'I': case 'J': | |
| 1048 case 'K': case 'M': case 'N': case 'O': | |
| 1049 case 'P': case 'Q': case 'R': case 'S': case 'T': | |
| 1050 case 'U': case 'V': case 'W': case 'X': case 'Y': | |
| 1051 case 'Z': | |
| 1052 case '_': | |
| 1053 case_ident: | |
| 135 | 1054 { |
| 1055 ubyte c; | |
| 0 | 1056 |
| 135 | 1057 while (1) |
| 1058 { | |
| 1059 c = *++p; | |
| 1060 if (isidchar(c)) | |
| 1061 continue; | |
| 1062 else if (c & 0x80) | |
| 1063 { | |
| 1064 ubyte *s = p; | |
| 1065 uint u = decodeUTF(); | |
| 1066 if (isUniAlpha(u)) | |
| 1067 continue; | |
| 1068 error("char 0x%04x not allowed in identifier", u); | |
| 1069 p = s; | |
| 1070 } | |
| 1071 break; | |
| 1072 } | |
| 1073 | |
| 178 | 1074 auto s = cast(string)(t.ptr[0.. p - t.ptr]); |
| 1075 Object* sv = stringtable.update(s); | |
| 1076 Identifier id = cast(Identifier) *sv; | |
| 1077 | |
| 0 | 1078 if (id is null) |
| 178 | 1079 { |
| 1080 id = new Identifier(s, TOK.TOKidentifier); | |
| 1081 *sv = id; | |
| 0 | 1082 } |
| 1083 t.ident = id; | |
| 1084 t.value = cast(TOK) id.value; | |
| 1085 anyToken = 1; | |
| 1086 if (*t.ptr == '_') // if special identifier token | |
| 1087 { | |
| 1088 ///version (DMDV1) { | |
| 1089 /// if (mod && id == Id.FILE) | |
| 1090 /// { | |
| 1091 /// t.ustring = cast(ubyte*)(loc.filename ? loc.filename : mod.ident.toChars()); | |
| 1092 /// goto Lstr; | |
| 1093 /// } | |
| 1094 /// else if (mod && id == Id.LINE) | |
| 1095 /// { | |
| 1096 /// t.value = TOK.TOKint64v; | |
| 1097 /// t.uns64value = loc.linnum; | |
| 1098 /// } | |
| 1099 /// else | |
| 1100 ///} | |
| 1101 if (id == Id.DATE) | |
| 1102 { | |
| 169 | 1103 t.ustring = global.date.ptr; |
| 0 | 1104 goto Lstr; |
| 1105 } | |
| 1106 else if (id == Id.TIME) | |
| 1107 { | |
| 169 | 1108 t.ustring = global.time.ptr; |
| 0 | 1109 goto Lstr; |
| 1110 } | |
| 1111 else if (id == Id.VENDOR) | |
| 1112 { | |
| 1113 t.ustring = "Digital Mars D".ptr; | |
| 1114 goto Lstr; | |
| 1115 } | |
| 1116 else if (id == Id.TIMESTAMP) | |
| 1117 { | |
| 169 | 1118 t.ustring = global.timestamp.ptr; |
| 0 | 1119 Lstr: |
| 1120 t.value = TOK.TOKstring; | |
| 1121 Llen: | |
| 1122 t.postfix = 0; | |
| 1123 t.len = strlen(cast(char*)t.ustring); | |
| 1124 } | |
| 1125 else if (id == Id.VERSIONX) | |
| 1126 { | |
| 1127 uint major = 0; | |
| 1128 uint minor = 0; | |
| 1129 | |
| 1130 foreach (char cc; global.version_[1..$]) | |
| 1131 { | |
| 1132 if (isdigit(cc)) | |
| 1133 minor = minor * 10 + cc - '0'; | |
| 1134 else if (cc == '.') | |
| 1135 { | |
| 1136 major = minor; | |
| 1137 minor = 0; | |
| 1138 } | |
| 1139 else | |
| 1140 break; | |
| 1141 } | |
| 1142 t.value = TOK.TOKint64v; | |
| 1143 t.uns64value = major * 1000 + minor; | |
| 1144 } | |
| 1145 ///version (DMDV2) { | |
| 1146 else if (id == Id.EOFX) | |
| 1147 { | |
| 1148 t.value = TOK.TOKeof; | |
| 1149 // Advance scanner to end of file | |
| 1150 while (!(*p == 0 || *p == 0x1A)) | |
| 1151 p++; | |
| 1152 } | |
| 1153 ///} | |
| 1154 } | |
| 1155 //printf("t.value = %d\n",t.value); | |
| 1156 return; | |
| 1157 } | |
| 1158 | |
| 1159 case '/': | |
| 1160 p++; | |
| 1161 switch (*p) | |
| 1162 { | |
| 1163 case '=': | |
| 1164 p++; | |
| 1165 t.value = TOK.TOKdivass; | |
| 1166 return; | |
| 1167 | |
| 1168 case '*': | |
| 1169 p++; | |
| 1170 linnum = loc.linnum; | |
| 1171 while (1) | |
| 1172 { | |
| 1173 while (1) | |
| 1174 { | |
| 1175 ubyte c = *p; | |
| 1176 switch (c) | |
| 1177 { | |
| 1178 case '/': | |
| 1179 break; | |
| 1180 | |
| 1181 case '\n': | |
| 1182 loc.linnum++; | |
| 1183 p++; | |
| 1184 continue; | |
| 1185 | |
| 1186 case '\r': | |
| 1187 p++; | |
| 1188 if (*p != '\n') | |
| 1189 loc.linnum++; | |
| 1190 continue; | |
| 1191 | |
| 1192 case 0: | |
| 1193 case 0x1A: | |
| 1194 error("unterminated /* */ comment"); | |
| 1195 p = end; | |
| 1196 t.value = TOK.TOKeof; | |
| 1197 return; | |
| 1198 | |
| 1199 default: | |
| 1200 if (c & 0x80) | |
| 1201 { uint u = decodeUTF(); | |
| 1202 if (u == PS || u == LS) | |
| 1203 loc.linnum++; | |
| 1204 } | |
| 1205 p++; | |
| 1206 continue; | |
| 1207 } | |
| 1208 break; | |
| 1209 } | |
| 1210 p++; | |
| 1211 if (p[-2] == '*' && p - 3 != t.ptr) | |
| 1212 break; | |
| 1213 } | |
| 1214 if (commentToken) | |
| 1215 { | |
| 1216 t.value = TOK.TOKcomment; | |
| 1217 return; | |
| 1218 } | |
| 1219 else if (doDocComment && t.ptr[2] == '*' && p - 4 != t.ptr) | |
| 1220 { // if /** but not /**/ | |
| 1221 getDocComment(t, lastLine == linnum); | |
| 1222 } | |
| 1223 continue; | |
| 1224 | |
| 1225 case '/': // do // style comments | |
| 1226 linnum = loc.linnum; | |
| 1227 while (1) | |
| 1228 { ubyte c = *++p; | |
| 1229 switch (c) | |
| 1230 { | |
| 1231 case '\n': | |
| 1232 break; | |
| 1233 | |
| 1234 case '\r': | |
| 1235 if (p[1] == '\n') | |
| 1236 p++; | |
| 1237 break; | |
| 1238 | |
| 1239 case 0: | |
| 1240 case 0x1A: | |
| 1241 if (commentToken) | |
| 1242 { | |
| 1243 p = end; | |
| 1244 t.value = TOK.TOKcomment; | |
| 1245 return; | |
| 1246 } | |
|
157
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
1247 if (doDocComment && t.ptr[2] == '/' || t.ptr[2] == '!') // '///' or '//!' |
| 0 | 1248 getDocComment(t, lastLine == linnum); |
| 1249 p = end; | |
| 1250 t.value = TOK.TOKeof; | |
| 1251 return; | |
| 1252 | |
| 1253 default: | |
| 1254 if (c & 0x80) | |
| 1255 { uint u = decodeUTF(); | |
| 1256 if (u == PS || u == LS) | |
| 1257 break; | |
| 1258 } | |
| 1259 continue; | |
| 1260 } | |
| 1261 break; | |
| 1262 } | |
| 1263 | |
| 1264 if (commentToken) | |
| 1265 { | |
| 1266 p++; | |
| 1267 loc.linnum++; | |
| 1268 t.value = TOK.TOKcomment; | |
| 1269 return; | |
| 1270 } | |
|
157
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
1271 if (doDocComment && t.ptr[2] == '/' || t.ptr[2] == '!') // '///' or '//!' |
| 0 | 1272 getDocComment(t, lastLine == linnum); |
| 1273 | |
| 1274 p++; | |
| 1275 loc.linnum++; | |
| 1276 continue; | |
| 1277 | |
| 1278 case '+': | |
| 1279 { | |
| 1280 int nest; | |
| 1281 | |
| 1282 linnum = loc.linnum; | |
| 1283 p++; | |
| 1284 nest = 1; | |
| 1285 while (1) | |
| 1286 { ubyte c = *p; | |
| 1287 switch (c) | |
| 1288 { | |
| 1289 case '/': | |
| 1290 p++; | |
| 1291 if (*p == '+') | |
| 1292 { | |
| 1293 p++; | |
| 1294 nest++; | |
| 1295 } | |
| 1296 continue; | |
| 1297 | |
| 1298 case '+': | |
| 1299 p++; | |
| 1300 if (*p == '/') | |
| 1301 { | |
| 1302 p++; | |
| 1303 if (--nest == 0) | |
| 1304 break; | |
| 1305 } | |
| 1306 continue; | |
| 1307 | |
| 1308 case '\r': | |
| 1309 p++; | |
| 1310 if (*p != '\n') | |
| 1311 loc.linnum++; | |
| 1312 continue; | |
| 1313 | |
| 1314 case '\n': | |
| 1315 loc.linnum++; | |
| 1316 p++; | |
| 1317 continue; | |
| 1318 | |
| 1319 case 0: | |
| 1320 case 0x1A: | |
| 1321 error("unterminated /+ +/ comment"); | |
| 1322 p = end; | |
| 1323 t.value = TOK.TOKeof; | |
| 1324 return; | |
| 1325 | |
| 1326 default: | |
| 1327 if (c & 0x80) | |
| 1328 { uint u = decodeUTF(); | |
| 1329 if (u == PS || u == LS) | |
| 1330 loc.linnum++; | |
| 1331 } | |
| 1332 p++; | |
| 1333 continue; | |
| 1334 } | |
| 1335 break; | |
| 1336 } | |
| 1337 if (commentToken) | |
| 1338 { | |
| 1339 t.value = TOK.TOKcomment; | |
| 1340 return; | |
| 1341 } | |
| 1342 if (doDocComment && t.ptr[2] == '+' && p - 4 != t.ptr) | |
| 1343 { // if /++ but not /++/ | |
| 1344 getDocComment(t, lastLine == linnum); | |
| 1345 } | |
| 1346 continue; | |
| 1347 } | |
| 178 | 1348 |
| 0 | 1349 default: |
| 1350 break; /// | |
| 1351 } | |
| 1352 t.value = TOK.TOKdiv; | |
| 1353 return; | |
| 1354 | |
| 1355 case '.': | |
| 1356 p++; | |
| 1357 if (isdigit(*p)) | |
| 1358 { /* Note that we don't allow ._1 and ._ as being | |
| 1359 * valid floating point numbers. | |
| 1360 */ | |
| 1361 p--; | |
| 1362 t.value = inreal(t); | |
| 1363 } | |
| 1364 else if (p[0] == '.') | |
| 1365 { | |
| 1366 if (p[1] == '.') | |
| 1367 { p += 2; | |
| 1368 t.value = TOK.TOKdotdotdot; | |
| 1369 } | |
| 1370 else | |
| 1371 { p++; | |
| 1372 t.value = TOK.TOKslice; | |
| 1373 } | |
| 1374 } | |
| 1375 else | |
| 1376 t.value = TOK.TOKdot; | |
| 1377 return; | |
| 1378 | |
| 1379 case '&': | |
| 1380 p++; | |
| 1381 if (*p == '=') | |
| 1382 { p++; | |
| 1383 t.value = TOK.TOKandass; | |
| 1384 } | |
| 1385 else if (*p == '&') | |
| 1386 { p++; | |
| 1387 t.value = TOK.TOKandand; | |
| 1388 } | |
| 1389 else | |
| 1390 t.value = TOK.TOKand; | |
| 1391 return; | |
| 1392 | |
| 1393 case '|': | |
| 1394 p++; | |
| 1395 if (*p == '=') | |
| 1396 { p++; | |
| 1397 t.value = TOK.TOKorass; | |
| 1398 } | |
| 1399 else if (*p == '|') | |
| 1400 { p++; | |
| 1401 t.value = TOK.TOKoror; | |
| 1402 } | |
| 1403 else | |
| 1404 t.value = TOK.TOKor; | |
| 1405 return; | |
| 1406 | |
| 1407 case '-': | |
| 1408 p++; | |
| 1409 if (*p == '=') | |
| 1410 { p++; | |
| 1411 t.value = TOK.TOKminass; | |
| 1412 } | |
| 1413 /// #if 0 | |
| 1414 /// else if (*p == '>') | |
| 1415 /// { p++; | |
| 1416 /// t.value = TOK.TOKarrow; | |
| 1417 /// } | |
| 1418 /// #endif | |
| 1419 else if (*p == '-') | |
| 1420 { p++; | |
| 1421 t.value = TOK.TOKminusminus; | |
| 1422 } | |
| 1423 else | |
| 1424 t.value = TOK.TOKmin; | |
| 1425 return; | |
| 1426 | |
| 1427 case '+': | |
| 1428 p++; | |
| 1429 if (*p == '=') | |
| 1430 { p++; | |
| 1431 t.value = TOK.TOKaddass; | |
| 1432 } | |
| 1433 else if (*p == '+') | |
| 1434 { p++; | |
| 1435 t.value = TOK.TOKplusplus; | |
| 1436 } | |
| 1437 else | |
| 1438 t.value = TOK.TOKadd; | |
| 1439 return; | |
| 1440 | |
| 1441 case '<': | |
| 1442 p++; | |
| 1443 if (*p == '=') | |
| 1444 { p++; | |
| 1445 t.value = TOK.TOKle; // <= | |
| 1446 } | |
| 1447 else if (*p == '<') | |
| 1448 { p++; | |
| 1449 if (*p == '=') | |
| 1450 { p++; | |
| 1451 t.value = TOK.TOKshlass; // <<= | |
| 1452 } | |
| 1453 else | |
| 1454 t.value = TOK.TOKshl; // << | |
| 1455 } | |
| 1456 else if (*p == '>') | |
| 1457 { p++; | |
| 1458 if (*p == '=') | |
| 1459 { p++; | |
| 1460 t.value = TOK.TOKleg; // <>= | |
| 1461 } | |
| 1462 else | |
| 1463 t.value = TOK.TOKlg; // <> | |
| 1464 } | |
| 1465 else | |
| 1466 t.value = TOK.TOKlt; // < | |
| 1467 return; | |
| 1468 | |
| 1469 case '>': | |
| 1470 p++; | |
| 1471 if (*p == '=') | |
| 1472 { p++; | |
| 1473 t.value = TOK.TOKge; // >= | |
| 1474 } | |
| 1475 else if (*p == '>') | |
| 1476 { p++; | |
| 1477 if (*p == '=') | |
| 1478 { p++; | |
| 1479 t.value = TOK.TOKshrass; // >>= | |
| 1480 } | |
| 1481 else if (*p == '>') | |
| 1482 { p++; | |
| 1483 if (*p == '=') | |
| 1484 { p++; | |
| 1485 t.value = TOK.TOKushrass; // >>>= | |
| 1486 } | |
| 1487 else | |
| 1488 t.value = TOK.TOKushr; // >>> | |
| 1489 } | |
| 1490 else | |
| 1491 t.value = TOK.TOKshr; // >> | |
| 1492 } | |
| 1493 else | |
| 1494 t.value = TOK.TOKgt; // > | |
| 1495 return; | |
| 1496 | |
| 1497 case '!': | |
| 1498 p++; | |
| 1499 if (*p == '=') | |
| 1500 { p++; | |
| 1501 if (*p == '=' && global.params.Dversion == 1) | |
| 1502 { p++; | |
| 1503 t.value = TOK.TOKnotidentity; // !== | |
| 1504 } | |
| 1505 else | |
| 1506 t.value = TOK.TOKnotequal; // != | |
| 1507 } | |
| 1508 else if (*p == '<') | |
| 1509 { p++; | |
| 1510 if (*p == '>') | |
| 1511 { p++; | |
| 1512 if (*p == '=') | |
| 1513 { p++; | |
| 1514 t.value = TOK.TOKunord; // !<>= | |
| 1515 } | |
| 1516 else | |
| 1517 t.value = TOK.TOKue; // !<> | |
| 1518 } | |
| 1519 else if (*p == '=') | |
| 1520 { p++; | |
| 1521 t.value = TOK.TOKug; // !<= | |
| 1522 } | |
| 1523 else | |
| 1524 t.value = TOK.TOKuge; // !< | |
| 1525 } | |
| 1526 else if (*p == '>') | |
| 1527 { p++; | |
| 1528 if (*p == '=') | |
| 1529 { p++; | |
| 1530 t.value = TOK.TOKul; // !>= | |
| 1531 } | |
| 1532 else | |
| 1533 t.value = TOK.TOKule; // !> | |
| 1534 } | |
| 1535 else | |
| 1536 t.value = TOK.TOKnot; // ! | |
| 1537 return; | |
| 1538 | |
| 1539 case '=': | |
| 1540 p++; | |
| 1541 if (*p == '=') | |
| 1542 { p++; | |
| 1543 if (*p == '=' && global.params.Dversion == 1) | |
| 1544 { p++; | |
| 1545 t.value = TOK.TOKidentity; // === | |
| 1546 } | |
| 1547 else | |
| 1548 t.value = TOK.TOKequal; // == | |
| 1549 } | |
| 1550 else | |
| 1551 t.value = TOK.TOKassign; // = | |
| 1552 return; | |
| 1553 | |
| 1554 case '~': | |
| 1555 p++; | |
| 1556 if (*p == '=') | |
| 1557 { p++; | |
| 1558 t.value = TOK.TOKcatass; // ~= | |
| 1559 } | |
| 1560 else | |
| 1561 t.value = TOK.TOKtilde; // ~ | |
| 1562 return; | |
| 178 | 1563 |
|
130
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1564 version(DMDV2) { |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1565 case '^': |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1566 p++; |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1567 if (*p == '^') |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1568 { p++; |
| 135 | 1569 if (*p == '=') |
| 1570 { p++; | |
| 1571 t.value = TOKpowass; // ^^= | |
| 1572 } | |
| 1573 else | |
|
130
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1574 t.value = TOKpow; // ^^ |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1575 } |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1576 else if (*p == '=') |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1577 { p++; |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1578 t.value = TOKxorass; // ^= |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1579 } |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1580 else |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1581 t.value = TOKxor; // ^ |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1582 return; |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1583 } |
|
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1584 |
| 0 | 1585 /* |
| 1586 #define SINGLE(c,tok) case c: p++; t.value = tok; return; | |
| 1587 | |
| 1588 SINGLE('(', TOKlparen) | |
| 1589 SINGLE(')', TOKrparen) | |
| 1590 SINGLE('[', TOKlbracket) | |
| 1591 SINGLE(']', TOKrbracket) | |
| 1592 SINGLE('{', TOKlcurly) | |
| 1593 SINGLE('}', TOKrcurly) | |
| 1594 SINGLE('?', TOKquestion) | |
| 1595 SINGLE(',', TOKcomma) | |
| 1596 SINGLE(';', TOKsemicolon) | |
| 1597 SINGLE(':', TOKcolon) | |
| 1598 SINGLE('$', TOKdollar) | |
| 1599 SINGLE('@', TOKat) | |
| 1600 | |
| 1601 #undef SINGLE | |
| 1602 | |
| 1603 #define DOUBLE(c1,tok1,c2,tok2) \ | |
| 1604 case c1: \ | |
| 1605 p++; \ | |
| 1606 if (*p == c2) \ | |
| 1607 { p++; \ | |
| 1608 t.value = tok2; \ | |
| 1609 } \ | |
| 1610 else \ | |
| 1611 t.value = tok1; \ | |
| 1612 return; | |
| 1613 | |
| 1614 DOUBLE('*', TOKmul, '=', TOKmulass) | |
| 1615 DOUBLE('%', TOKmod, '=', TOKmodass) | |
|
130
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1616 #if DMDV1 |
| 0 | 1617 DOUBLE('^', TOKxor, '=', TOKxorass) |
|
130
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1618 #endif |
| 0 | 1619 #undef DOUBLE |
| 1620 */ | |
| 1621 | |
| 1622 case '(': p++; t.value = TOK.TOKlparen; return; | |
| 1623 case ')': p++; t.value = TOK.TOKrparen; return; | |
| 1624 case '[': p++; t.value = TOK.TOKlbracket; return; | |
| 1625 case ']': p++; t.value = TOK.TOKrbracket; return; | |
| 1626 case '{': p++; t.value = TOK.TOKlcurly; return; | |
| 1627 case '}': p++; t.value = TOK.TOKrcurly; return; | |
| 1628 case '?': p++; t.value = TOK.TOKquestion; return; | |
| 1629 case ',': p++; t.value = TOK.TOKcomma; return; | |
| 1630 case ';': p++; t.value = TOK.TOKsemicolon; return; | |
| 1631 case ':': p++; t.value = TOK.TOKcolon; return; | |
| 1632 case '$': p++; t.value = TOK.TOKdollar; return; | |
| 1633 case '@': p++; t.value = TOK.TOKat; return; | |
| 1634 | |
| 1635 case '*': | |
| 1636 p++; | |
| 1637 if (*p == '=') { | |
| 1638 p++; | |
| 1639 t.value = TOK.TOKmulass; | |
| 1640 } else { | |
| 1641 t.value = TOK.TOKmul; | |
| 1642 } | |
| 1643 return; | |
| 178 | 1644 |
| 0 | 1645 case '%': |
| 1646 p++; | |
| 1647 if (*p == '=') { | |
| 1648 p++; | |
| 1649 t.value = TOK.TOKmodass; | |
| 1650 } else { | |
| 1651 t.value = TOK.TOKmod; | |
| 1652 } | |
| 1653 return; | |
|
130
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1654 version(DMDV1) { |
| 0 | 1655 case '^': |
| 1656 p++; | |
| 1657 if (*p == '=') { | |
| 1658 p++; | |
| 1659 t.value = TOK.TOKxorass; | |
| 1660 } else { | |
| 1661 t.value = TOK.TOKxor; | |
| 1662 } | |
| 1663 return; | |
|
130
60bb0fe4563e
dmdfe 2.037 first main iteration
Eldar Insafutdinov <e.insafutdinov@gmail.com>
parents:
114
diff
changeset
|
1664 } |
| 0 | 1665 case '#': |
| 1666 p++; | |
| 1667 pragma_(); | |
| 1668 continue; | |
| 1669 | |
| 1670 default: | |
| 135 | 1671 { uint c = *p; |
| 0 | 1672 |
| 1673 if (c & 0x80) | |
| 135 | 1674 { c = decodeUTF(); |
| 0 | 1675 |
| 1676 // Check for start of unicode identifier | |
| 135 | 1677 if (isUniAlpha(c)) |
| 0 | 1678 goto case_ident; |
| 1679 | |
| 135 | 1680 if (c == PS || c == LS) |
| 0 | 1681 { |
| 1682 loc.linnum++; | |
| 1683 p++; | |
| 1684 continue; | |
| 1685 } | |
| 1686 } | |
| 135 | 1687 if (c < 0x80 && isprint(c)) |
| 0 | 1688 error("unsupported char '%c'", c); |
| 1689 else | |
| 1690 error("unsupported char 0x%02x", c); | |
| 1691 p++; | |
| 1692 continue; | |
| 1693 } | |
| 1694 } | |
| 1695 } | |
| 1696 } | |
| 1697 | |
| 1698 Token* peek(Token* ct) | |
| 1699 { | |
| 1700 Token* t; | |
| 1701 | |
| 1702 if (ct.next) | |
| 1703 t = ct.next; | |
| 1704 else | |
| 1705 { | |
| 1706 t = new Token(); | |
| 1707 scan(t); | |
| 1708 t.next = null; | |
| 1709 ct.next = t; | |
| 1710 } | |
| 1711 return t; | |
| 1712 } | |
| 1713 | |
| 1714 Token* peekPastParen(Token* tk) | |
| 1715 { | |
| 1716 //printf("peekPastParen()\n"); | |
| 1717 int parens = 1; | |
| 1718 int curlynest = 0; | |
| 1719 while (1) | |
| 1720 { | |
| 1721 tk = peek(tk); | |
| 1722 //tk.print(); | |
| 1723 switch (tk.value) | |
| 1724 { | |
| 1725 case TOK.TOKlparen: | |
| 1726 parens++; | |
| 1727 continue; | |
| 1728 | |
| 1729 case TOK.TOKrparen: | |
| 1730 --parens; | |
| 1731 if (parens) | |
| 1732 continue; | |
| 1733 tk = peek(tk); | |
| 1734 break; | |
| 1735 | |
| 1736 case TOK.TOKlcurly: | |
| 1737 curlynest++; | |
| 1738 continue; | |
| 1739 | |
| 1740 case TOK.TOKrcurly: | |
| 1741 if (--curlynest >= 0) | |
| 1742 continue; | |
| 1743 break; | |
| 1744 | |
| 1745 case TOK.TOKsemicolon: | |
| 1746 if (curlynest) | |
| 1747 continue; | |
| 1748 break; | |
| 1749 | |
| 1750 case TOK.TOKeof: | |
| 1751 break; | |
| 1752 | |
| 1753 default: | |
| 1754 continue; | |
| 1755 } | |
| 1756 return tk; | |
| 1757 } | |
| 1758 } | |
| 1759 | |
| 1760 /******************************************* | |
| 1761 * Parse escape sequence. | |
| 1762 */ | |
| 1763 uint escapeSequence() | |
| 1764 { | |
| 1765 uint c = *p; | |
| 1766 | |
| 1767 version (TEXTUAL_ASSEMBLY_OUT) { | |
| 1768 return c; | |
| 1769 } | |
| 1770 int n; | |
| 1771 int ndigits; | |
| 1772 | |
| 1773 switch (c) | |
| 1774 { | |
| 1775 case '\'': | |
| 1776 case '"': | |
| 1777 case '?': | |
| 1778 case '\\': | |
| 1779 Lconsume: | |
| 1780 p++; | |
| 1781 break; | |
| 1782 | |
| 1783 case 'a': c = 7; goto Lconsume; | |
| 1784 case 'b': c = 8; goto Lconsume; | |
| 1785 case 'f': c = 12; goto Lconsume; | |
| 1786 case 'n': c = 10; goto Lconsume; | |
| 1787 case 'r': c = 13; goto Lconsume; | |
| 1788 case 't': c = 9; goto Lconsume; | |
| 1789 case 'v': c = 11; goto Lconsume; | |
| 1790 | |
| 1791 case 'u': | |
| 1792 ndigits = 4; | |
| 1793 goto Lhex; | |
| 1794 case 'U': | |
| 1795 ndigits = 8; | |
| 1796 goto Lhex; | |
| 1797 case 'x': | |
| 1798 ndigits = 2; | |
| 1799 Lhex: | |
| 1800 p++; | |
| 1801 c = *p; | |
| 1802 if (ishex(cast(ubyte)c)) | |
| 178 | 1803 { |
| 0 | 1804 uint v; |
| 1805 | |
| 1806 n = 0; | |
| 1807 v = 0; | |
| 1808 while (1) | |
| 1809 { | |
| 1810 if (isdigit(c)) | |
| 1811 c -= '0'; | |
| 1812 else if (islower(c)) | |
| 1813 c -= 'a' - 10; | |
| 1814 else | |
| 1815 c -= 'A' - 10; | |
| 1816 v = v * 16 + c; | |
| 1817 c = *++p; | |
| 1818 if (++n == ndigits) | |
| 1819 break; | |
| 1820 if (!ishex(cast(ubyte)c)) | |
| 1821 { error("escape hex sequence has %d hex digits instead of %d", n, ndigits); | |
| 1822 break; | |
| 1823 } | |
| 1824 } | |
| 1825 if (ndigits != 2 && !utf_isValidDchar(v)) | |
| 1826 { error("invalid UTF character \\U%08x", v); | |
| 1827 v = '?'; // recover with valid UTF character | |
| 1828 } | |
| 1829 c = v; | |
| 1830 } | |
| 1831 else | |
| 1832 error("undefined escape hex sequence \\%c\n",c); | |
| 1833 break; | |
| 1834 | |
| 1835 case '&': // named character entity | |
| 1836 for (ubyte* idstart = ++p; true; p++) | |
| 1837 { | |
| 1838 switch (*p) | |
| 1839 { | |
| 1840 case ';': | |
| 1841 c = HtmlNamedEntity(idstart, p - idstart); | |
| 1842 if (c == ~0) | |
| 178 | 1843 { |
| 0 | 1844 error("unnamed character entity &%s;", idstart[0..(p - idstart)]); |
| 1845 c = ' '; | |
| 1846 } | |
| 1847 p++; | |
| 1848 break; | |
| 1849 | |
| 1850 default: | |
| 1851 if (isalpha(*p) || | |
| 1852 (p != idstart + 1 && isdigit(*p))) | |
| 1853 continue; | |
| 1854 error("unterminated named entity"); | |
| 1855 break; | |
| 1856 } | |
| 1857 break; | |
| 1858 } | |
| 1859 break; | |
| 1860 | |
| 1861 case 0: | |
| 1862 case 0x1A: // end of file | |
| 1863 c = '\\'; | |
| 1864 break; | |
| 1865 | |
| 1866 default: | |
| 1867 if (isoctal(cast(ubyte)c)) | |
| 178 | 1868 { |
| 0 | 1869 uint v; |
| 1870 | |
| 1871 n = 0; | |
| 1872 v = 0; | |
| 1873 do | |
| 1874 { | |
| 1875 v = v * 8 + (c - '0'); | |
| 1876 c = *++p; | |
| 1877 } while (++n < 3 && isoctal(cast(ubyte)c)); | |
| 1878 c = v; | |
| 1879 if (c > 0xFF) | |
| 1880 error("0%03o is larger than a byte", c); | |
| 1881 } | |
| 1882 else | |
| 1883 error("undefined escape sequence \\%c\n",c); | |
| 1884 break; | |
| 1885 } | |
| 1886 return c; | |
| 1887 } | |
| 1888 | |
| 1889 TOK wysiwygStringConstant(Token* t, int tc) | |
| 1890 { | |
|
8
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1891 uint c; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1892 Loc start = loc; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1893 |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1894 p++; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1895 stringbuffer.reset(); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1896 while (true) |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1897 { |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1898 c = *p++; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1899 switch (c) |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1900 { |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1901 case '\n': |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1902 loc.linnum++; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1903 break; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1904 |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1905 case '\r': |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1906 if (*p == '\n') |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1907 continue; // ignore |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1908 c = '\n'; // treat EndOfLine as \n character |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1909 loc.linnum++; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1910 break; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1911 |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1912 case 0: |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1913 case 0x1A: |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1914 error("unterminated string constant starting at %s", start.toChars()); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1915 t.ustring = "".ptr; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1916 t.len = 0; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1917 t.postfix = 0; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1918 return TOKstring; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1919 |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1920 case '"': |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1921 case '`': |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1922 if (c == tc) |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1923 { |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1924 t.len = stringbuffer.offset; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1925 stringbuffer.writeByte(0); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1926 char* tmp = cast(char*)GC.malloc(stringbuffer.offset); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1927 memcpy(tmp, stringbuffer.data, stringbuffer.offset); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1928 t.ustring = tmp; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1929 stringPostfix(t); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1930 return TOKstring; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1931 } |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1932 break; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1933 |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1934 default: |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1935 if (c & 0x80) |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1936 { p--; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1937 uint u = decodeUTF(); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1938 p++; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1939 if (u == PS || u == LS) |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1940 loc.linnum++; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1941 stringbuffer.writeUTF8(u); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1942 continue; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1943 } |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1944 break; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1945 } |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1946 stringbuffer.writeByte(c); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
1947 } |
| 178 | 1948 |
| 0 | 1949 assert(false); |
| 1950 } | |
| 1951 | |
| 51 | 1952 /************************************** |
| 1953 * Lex hex strings: | |
| 1954 * x"0A ae 34FE BD" | |
| 1955 */ | |
| 0 | 1956 TOK hexStringConstant(Token* t) |
| 1957 { | |
| 51 | 1958 uint c; |
| 1959 Loc start = loc; | |
| 1960 uint n = 0; | |
| 1961 uint v; | |
| 1962 | |
| 1963 p++; | |
| 1964 stringbuffer.reset(); | |
| 1965 while (1) | |
| 1966 { | |
| 1967 c = *p++; | |
| 1968 switch (c) | |
| 1969 { | |
| 1970 case ' ': | |
| 1971 case '\t': | |
| 1972 case '\v': | |
| 1973 case '\f': | |
| 1974 continue; // skip white space | |
| 1975 | |
| 1976 case '\r': | |
| 1977 if (*p == '\n') | |
| 1978 continue; // ignore | |
| 1979 // Treat isolated '\r' as if it were a '\n' | |
| 1980 case '\n': | |
| 1981 loc.linnum++; | |
| 1982 continue; | |
| 1983 | |
| 1984 case 0: | |
| 1985 case 0x1A: | |
| 1986 error("unterminated string constant starting at %s", start.toChars()); | |
| 1987 t.ustring = "".ptr; | |
| 1988 t.len = 0; | |
| 1989 t.postfix = 0; | |
| 1990 return TOKstring; | |
| 1991 | |
| 1992 case '"': | |
| 1993 if (n & 1) | |
| 178 | 1994 { |
| 51 | 1995 error("odd number (%d) of hex characters in hex string", n); |
| 1996 stringbuffer.writeByte(v); | |
| 1997 } | |
| 1998 t.len = stringbuffer.offset; | |
| 1999 stringbuffer.writeByte(0); | |
| 175 | 2000 void* mem = GC.malloc(stringbuffer.offset); |
| 51 | 2001 memcpy(mem, stringbuffer.data, stringbuffer.offset); |
| 2002 t.ustring = cast(const(char)*)mem; | |
| 2003 stringPostfix(t); | |
| 2004 return TOKstring; | |
| 2005 | |
| 2006 default: | |
| 2007 if (c >= '0' && c <= '9') | |
| 2008 c -= '0'; | |
| 2009 else if (c >= 'a' && c <= 'f') | |
| 2010 c -= 'a' - 10; | |
| 2011 else if (c >= 'A' && c <= 'F') | |
| 2012 c -= 'A' - 10; | |
| 2013 else if (c & 0x80) | |
| 2014 { p--; | |
| 2015 uint u = decodeUTF(); | |
| 2016 p++; | |
| 2017 if (u == PS || u == LS) | |
| 2018 loc.linnum++; | |
| 2019 else | |
| 135 | 2020 error("non-hex character \\u%04x", u); |
| 51 | 2021 } |
| 2022 else | |
| 2023 error("non-hex character '%c'", c); | |
| 2024 if (n & 1) | |
| 2025 { v = (v << 4) | c; | |
| 2026 stringbuffer.writeByte(v); | |
| 2027 } | |
| 2028 else | |
| 2029 v = c; | |
| 2030 n++; | |
| 2031 break; | |
| 2032 } | |
| 2033 } | |
| 0 | 2034 } |
| 2035 | |
| 2036 version (DMDV2) { | |
| 51 | 2037 /************************************** |
| 2038 * Lex delimited strings: | |
| 2039 * q"(foo(xxx))" // "foo(xxx)" | |
| 2040 * q"[foo(]" // "foo(" | |
| 2041 * q"/foo]/" // "foo]" | |
| 2042 * q"HERE | |
| 2043 * foo | |
| 2044 * HERE" // "foo\n" | |
| 2045 * Input: | |
| 2046 * p is on the " | |
| 2047 */ | |
| 0 | 2048 TOK delimitedStringConstant(Token* t) |
| 2049 { | |
| 51 | 2050 uint c; |
| 2051 Loc start = loc; | |
| 2052 uint delimleft = 0; | |
| 2053 uint delimright = 0; | |
| 2054 uint nest = 1; | |
| 2055 uint nestcount; | |
| 2056 Identifier hereid = null; | |
| 2057 uint blankrol = 0; | |
| 2058 uint startline = 0; | |
| 2059 | |
| 2060 p++; | |
| 2061 stringbuffer.reset(); | |
| 2062 while (1) | |
| 2063 { | |
| 2064 c = *p++; | |
| 2065 //printf("c = '%c'\n", c); | |
| 2066 switch (c) | |
| 2067 { | |
| 2068 case '\n': | |
| 2069 Lnextline: | |
| 2070 loc.linnum++; | |
| 2071 startline = 1; | |
| 2072 if (blankrol) | |
| 2073 { blankrol = 0; | |
| 2074 continue; | |
| 2075 } | |
| 2076 if (hereid) | |
| 2077 { | |
| 2078 stringbuffer.writeUTF8(c); | |
| 2079 continue; | |
| 2080 } | |
| 2081 break; | |
| 2082 | |
| 2083 case '\r': | |
| 2084 if (*p == '\n') | |
| 2085 continue; // ignore | |
| 2086 c = '\n'; // treat EndOfLine as \n character | |
| 2087 goto Lnextline; | |
| 2088 | |
| 2089 case 0: | |
| 2090 case 0x1A: | |
| 2091 goto Lerror; | |
| 2092 | |
| 2093 default: | |
| 2094 if (c & 0x80) | |
| 2095 { p--; | |
| 2096 c = decodeUTF(); | |
| 2097 p++; | |
| 2098 if (c == PS || c == LS) | |
| 2099 goto Lnextline; | |
| 2100 } | |
| 2101 break; | |
| 2102 } | |
| 2103 if (delimleft == 0) | |
| 178 | 2104 { |
| 51 | 2105 delimleft = c; |
| 2106 nest = 1; | |
| 2107 nestcount = 1; | |
| 2108 if (c == '(') | |
| 2109 delimright = ')'; | |
| 2110 else if (c == '{') | |
| 2111 delimright = '}'; | |
| 2112 else if (c == '[') | |
| 2113 delimright = ']'; | |
| 2114 else if (c == '<') | |
| 2115 delimright = '>'; | |
| 2116 else if (isalpha(c) || c == '_' || (c >= 0x80 && isUniAlpha(c))) | |
| 178 | 2117 { |
| 51 | 2118 // Start of identifier; must be a heredoc |
| 2119 Token t2; | |
| 2120 p--; | |
| 2121 scan(&t2); // read in heredoc identifier | |
| 2122 if (t2.value != TOKidentifier) | |
| 178 | 2123 { |
| 51 | 2124 error("identifier expected for heredoc, not %s", t2.toChars()); |
| 2125 delimright = c; | |
| 2126 } | |
| 2127 else | |
| 178 | 2128 { |
| 51 | 2129 hereid = t2.ident; |
| 2130 //printf("hereid = '%s'\n", hereid.toChars()); | |
| 2131 blankrol = 1; | |
| 2132 } | |
| 2133 nest = 0; | |
| 2134 } | |
| 2135 else | |
| 178 | 2136 { |
| 51 | 2137 delimright = c; |
| 2138 nest = 0; | |
| 2139 if (isspace(c)) | |
| 2140 error("delimiter cannot be whitespace"); | |
| 2141 } | |
| 2142 } | |
| 2143 else | |
| 2144 { | |
| 2145 if (blankrol) | |
| 178 | 2146 { |
| 51 | 2147 error("heredoc rest of line should be blank"); |
| 2148 blankrol = 0; | |
| 2149 continue; | |
| 2150 } | |
| 2151 if (nest == 1) | |
| 2152 { | |
| 2153 if (c == delimleft) | |
| 2154 nestcount++; | |
| 2155 else if (c == delimright) | |
| 2156 { nestcount--; | |
| 2157 if (nestcount == 0) | |
| 2158 goto Ldone; | |
| 2159 } | |
| 2160 } | |
| 2161 else if (c == delimright) | |
| 2162 goto Ldone; | |
| 2163 if (startline && isalpha(c) && hereid) | |
| 178 | 2164 { |
| 51 | 2165 Token t2; |
| 2166 ubyte* psave = p; | |
| 2167 p--; | |
| 2168 scan(&t2); // read in possible heredoc identifier | |
| 2169 //printf("endid = '%s'\n", t2.ident.toChars()); | |
| 2170 if (t2.value == TOKidentifier && t2.ident.equals(hereid)) | |
| 178 | 2171 { |
| 51 | 2172 /* should check that rest of line is blank |
| 2173 */ | |
| 2174 goto Ldone; | |
| 2175 } | |
| 2176 p = psave; | |
| 2177 } | |
| 2178 stringbuffer.writeUTF8(c); | |
| 2179 startline = 0; | |
| 2180 } | |
| 2181 } | |
| 2182 | |
| 2183 Ldone: | |
| 2184 if (*p == '"') | |
| 2185 p++; | |
| 2186 else | |
| 2187 error("delimited string must end in %c\"", delimright); | |
| 2188 t.len = stringbuffer.offset; | |
| 2189 stringbuffer.writeByte(0); | |
| 175 | 2190 void* mem = GC.malloc(stringbuffer.offset); |
| 51 | 2191 memcpy(mem, stringbuffer.data, stringbuffer.offset); |
| 2192 t.ustring = cast(const(char)*)mem; | |
| 2193 stringPostfix(t); | |
| 2194 return TOKstring; | |
| 2195 | |
| 2196 Lerror: | |
| 2197 error("unterminated string constant starting at %s", start.toChars()); | |
| 2198 t.ustring = "".ptr; | |
| 2199 t.len = 0; | |
| 2200 t.postfix = 0; | |
| 2201 return TOKstring; | |
| 0 | 2202 } |
| 2203 | |
|
8
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2204 /************************************** |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2205 * Lex delimited strings: |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2206 * q{ foo(xxx) } // " foo(xxx) " |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2207 * q{foo(} // "foo(" |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2208 * q{{foo}"}"} // "{foo}"}"" |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2209 * Input: |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2210 * p is on the q |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2211 */ |
| 0 | 2212 TOK tokenStringConstant(Token* t) |
| 2213 { | |
|
8
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2214 uint nest = 1; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2215 Loc start = loc; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2216 ubyte* pstart = ++p; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2217 |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2218 while (true) |
| 178 | 2219 { |
|
8
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2220 Token tok; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2221 |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2222 scan(&tok); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2223 switch (tok.value) |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2224 { |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2225 case TOKlcurly: |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2226 nest++; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2227 continue; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2228 |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2229 case TOKrcurly: |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2230 if (--nest == 0) |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2231 goto Ldone; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2232 continue; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2233 |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2234 case TOKeof: |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2235 goto Lerror; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2236 |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2237 default: |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2238 continue; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2239 } |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2240 } |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2241 |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2242 Ldone: |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2243 t.len = p - 1 - pstart; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2244 char* tmp = cast(char*)GC.malloc(t.len + 1); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2245 memcpy(tmp, pstart, t.len); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2246 tmp[t.len] = 0; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2247 t.ustring = tmp; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2248 stringPostfix(t); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2249 return TOKstring; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2250 |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2251 Lerror: |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2252 error("unterminated token string constant starting at %s", start.toChars()); |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2253 t.ustring = "".ptr; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2254 t.len = 0; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2255 t.postfix = 0; |
|
d42cd5917df4
wysiwyg strings, alias this, templates, TypeSlice implementation
dkoroskin <>
parents:
4
diff
changeset
|
2256 return TOKstring; |
| 0 | 2257 } |
| 2258 } | |
| 2259 TOK escapeStringConstant(Token* t, int wide) | |
| 2260 { | |
| 2261 uint c; | |
| 2262 Loc start = loc; | |
| 2263 | |
| 2264 p++; | |
| 2265 stringbuffer.reset(); | |
| 2266 while (true) | |
| 2267 { | |
| 2268 c = *p++; | |
| 2269 switch (c) | |
| 2270 { | |
| 2271 version (TEXTUAL_ASSEMBLY_OUT) { | |
| 2272 } else { | |
| 2273 case '\\': | |
| 2274 switch (*p) | |
| 2275 { | |
| 2276 case 'u': | |
| 2277 case 'U': | |
| 2278 case '&': | |
| 2279 c = escapeSequence(); | |
| 2280 stringbuffer.writeUTF8(c); | |
| 2281 continue; | |
| 2282 | |
| 2283 default: | |
| 2284 c = escapeSequence(); | |
| 2285 break; | |
| 2286 } | |
| 2287 break; | |
| 2288 } | |
| 2289 case '\n': | |
| 2290 loc.linnum++; | |
| 2291 break; | |
| 2292 | |
| 2293 case '\r': | |
| 2294 if (*p == '\n') | |
| 2295 continue; // ignore | |
| 2296 c = '\n'; // treat EndOfLine as \n character | |
| 2297 loc.linnum++; | |
| 2298 break; | |
| 2299 | |
| 2300 case '"': | |
| 2301 t.len = stringbuffer.offset; | |
| 2302 stringbuffer.writeByte(0); | |
| 2 | 2303 char* tmp = cast(char*)GC.malloc(stringbuffer.offset); |
| 0 | 2304 memcpy(tmp, stringbuffer.data, stringbuffer.offset); |
| 2305 t.ustring = tmp; | |
| 2306 stringPostfix(t); | |
| 2307 return TOK.TOKstring; | |
| 2308 | |
| 2309 case 0: | |
| 2310 case 0x1A: | |
| 2311 p--; | |
| 2312 error("unterminated string constant starting at %s", start.toChars()); | |
| 2313 t.ustring = "".ptr; | |
| 2314 t.len = 0; | |
| 2315 t.postfix = 0; | |
| 2316 return TOK.TOKstring; | |
| 2317 | |
| 2318 default: | |
| 2319 if (c & 0x80) | |
| 2320 { | |
| 2321 p--; | |
| 2322 c = decodeUTF(); | |
| 2323 if (c == LS || c == PS) | |
| 2324 { c = '\n'; | |
| 2325 loc.linnum++; | |
| 2326 } | |
| 2327 p++; | |
| 2328 stringbuffer.writeUTF8(c); | |
| 2329 continue; | |
| 2330 } | |
| 2331 break; | |
| 2332 } | |
| 2333 stringbuffer.writeByte(c); | |
| 2334 } | |
| 178 | 2335 |
| 0 | 2336 assert(false); |
| 2337 } | |
| 2338 | |
| 2339 TOK charConstant(Token* t, int wide) | |
| 2340 { | |
| 2341 uint c; | |
| 2342 TOK tk = TOKcharv; | |
| 2343 | |
| 2344 //printf("Lexer.charConstant\n"); | |
| 2345 p++; | |
| 2346 c = *p++; | |
| 2347 switch (c) | |
| 2348 { | |
| 2349 version (TEXTUAL_ASSEMBLY_OUT) { | |
| 2350 } else { | |
| 2351 case '\\': | |
| 2352 switch (*p) | |
| 2353 { | |
| 2354 case 'u': | |
| 2355 t.uns64value = escapeSequence(); | |
| 2356 tk = TOKwcharv; | |
| 2357 break; | |
| 2358 | |
| 2359 case 'U': | |
| 2360 case '&': | |
| 2361 t.uns64value = escapeSequence(); | |
| 2362 tk = TOKdcharv; | |
| 2363 break; | |
| 2364 | |
| 2365 default: | |
| 2366 t.uns64value = escapeSequence(); | |
| 2367 break; | |
| 2368 } | |
| 2369 break; | |
| 2370 } | |
| 2371 case '\n': | |
| 2372 L1: | |
| 2373 loc.linnum++; | |
| 2374 case '\r': | |
| 2375 case 0: | |
| 2376 case 0x1A: | |
| 2377 case '\'': | |
| 2378 error("unterminated character constant"); | |
| 2379 return tk; | |
| 2380 | |
| 2381 default: | |
| 2382 if (c & 0x80) | |
| 2383 { | |
| 2384 p--; | |
| 2385 c = decodeUTF(); | |
| 2386 p++; | |
| 2387 if (c == LS || c == PS) | |
| 2388 goto L1; | |
| 2389 if (c < 0xD800 || (c >= 0xE000 && c < 0xFFFE)) | |
| 2390 tk = TOKwcharv; | |
| 2391 else | |
| 2392 tk = TOKdcharv; | |
| 2393 } | |
| 2394 t.uns64value = c; | |
| 2395 break; | |
| 2396 } | |
| 2397 | |
| 2398 if (*p != '\'') | |
| 178 | 2399 { |
| 0 | 2400 error("unterminated character constant"); |
| 2401 return tk; | |
| 2402 } | |
| 2403 p++; | |
| 2404 return tk; | |
| 2405 } | |
| 2406 | |
| 2407 /*************************************** | |
| 2408 * Get postfix of string literal. | |
| 2409 */ | |
| 2410 void stringPostfix(Token* t) | |
| 2411 { | |
| 2412 switch (*p) | |
| 2413 { | |
| 2414 case 'c': | |
| 2415 case 'w': | |
| 2416 case 'd': | |
| 2417 t.postfix = *p; | |
| 2418 p++; | |
| 2419 break; | |
| 2420 | |
| 2421 default: | |
| 2422 t.postfix = 0; | |
| 2423 break; | |
| 2424 } | |
| 2425 } | |
| 2426 | |
| 2427 uint wchar_(uint u) | |
| 2428 { | |
| 2429 assert(false); | |
| 2430 } | |
| 178 | 2431 |
| 0 | 2432 /************************************** |
| 2433 * Read in a number. | |
| 2434 * If it's an integer, store it in tok.TKutok.Vlong. | |
| 2435 * integers can be decimal, octal or hex | |
| 2436 * Handle the suffixes U, UL, LU, L, etc. | |
| 2437 * If it's double, store it in tok.TKutok.Vdouble. | |
| 2438 * Returns: | |
| 2439 * TKnum | |
| 2440 * TKdouble,... | |
| 2441 */ | |
| 2442 | |
| 2443 TOK number(Token* t) | |
| 2444 { | |
| 2445 // We use a state machine to collect numbers | |
| 2446 enum STATE { STATE_initial, STATE_0, STATE_decimal, STATE_octal, STATE_octale, | |
| 2447 STATE_hex, STATE_binary, STATE_hex0, STATE_binary0, | |
| 2448 STATE_hexh, STATE_error }; | |
| 2449 STATE state; | |
| 2450 | |
| 2451 enum FLAGS | |
| 2452 { | |
| 2453 FLAGS_undefined = 0, | |
| 2454 FLAGS_decimal = 1, // decimal | |
| 2455 FLAGS_unsigned = 2, // u or U suffix | |
| 2456 FLAGS_long = 4, // l or L suffix | |
| 2457 }; | |
| 178 | 2458 |
| 0 | 2459 FLAGS flags = FLAGS.FLAGS_decimal; |
| 2460 | |
| 2461 int i; | |
| 2462 int base; | |
| 2463 uint c; | |
| 2464 ubyte *start; | |
| 2465 TOK result; | |
| 2466 | |
| 2467 //printf("Lexer.number()\n"); | |
| 2468 state = STATE.STATE_initial; | |
| 2469 base = 0; | |
| 2470 stringbuffer.reset(); | |
| 2471 start = p; | |
| 2472 while (1) | |
| 2473 { | |
| 2474 c = *p; | |
| 2475 switch (state) | |
| 2476 { | |
| 2477 case STATE.STATE_initial: // opening state | |
| 2478 if (c == '0') | |
| 2479 state = STATE.STATE_0; | |
| 2480 else | |
| 2481 state = STATE.STATE_decimal; | |
| 2482 break; | |
| 2483 | |
| 2484 case STATE.STATE_0: | |
| 2485 flags = (flags & ~FLAGS.FLAGS_decimal); | |
| 2486 switch (c) | |
| 2487 { | |
| 2488 version (ZEROH) { | |
| 2489 case 'H': // 0h | |
| 2490 case 'h': | |
| 2491 goto hexh; | |
| 2492 } | |
| 2493 case 'X': | |
| 2494 case 'x': | |
| 2495 state = STATE.STATE_hex0; | |
| 2496 break; | |
| 2497 | |
| 2498 case '.': | |
| 2499 if (p[1] == '.') // .. is a separate token | |
| 2500 goto done; | |
| 2501 case 'i': | |
| 2502 case 'f': | |
| 2503 case 'F': | |
| 2504 goto real_; | |
| 2505 version (ZEROH) { | |
| 2506 case 'E': | |
| 2507 case 'e': | |
| 2508 goto case_hex; | |
| 2509 } | |
| 2510 case 'B': | |
| 2511 case 'b': | |
| 2512 state = STATE.STATE_binary0; | |
| 2513 break; | |
| 2514 | |
| 2515 case '0': case '1': case '2': case '3': | |
| 2516 case '4': case '5': case '6': case '7': | |
| 2517 state = STATE.STATE_octal; | |
| 2518 break; | |
| 2519 | |
| 2520 version (ZEROH) { | |
| 2521 case '8': case '9': case 'A': | |
| 2522 case 'C': case 'D': case 'F': | |
| 2523 case 'a': case 'c': case 'd': case 'f': | |
| 2524 case_hex: | |
| 2525 state = STATE.STATE_hexh; | |
| 2526 break; | |
| 2527 } | |
| 2528 case '_': | |
| 2529 state = STATE.STATE_octal; | |
| 2530 p++; | |
| 2531 continue; | |
| 2532 | |
| 2533 case 'L': | |
| 2534 if (p[1] == 'i') | |
| 2535 goto real_; | |
| 2536 goto done; | |
| 2537 | |
| 2538 default: | |
| 2539 goto done; | |
| 2540 } | |
| 2541 break; | |
| 2542 | |
| 2543 case STATE.STATE_decimal: // reading decimal number | |
| 2544 if (!isdigit(c)) | |
| 2545 { | |
| 2546 version (ZEROH) { | |
| 2547 if (ishex(c) | |
| 2548 || c == 'H' || c == 'h' | |
| 2549 ) | |
| 2550 goto hexh; | |
| 2551 } | |
| 2552 if (c == '_') // ignore embedded _ | |
| 2553 { p++; | |
| 2554 continue; | |
| 2555 } | |
| 2556 if (c == '.' && p[1] != '.') | |
| 2557 goto real_; | |
| 2558 else if (c == 'i' || c == 'f' || c == 'F' || | |
| 2559 c == 'e' || c == 'E') | |
| 2560 { | |
| 2561 real_: // It's a real number. Back up and rescan as a real | |
| 2562 p = start; | |
| 2563 return inreal(t); | |
| 2564 } | |
| 2565 else if (c == 'L' && p[1] == 'i') | |
| 2566 goto real_; | |
| 2567 goto done; | |
| 2568 } | |
| 2569 break; | |
| 2570 | |
| 2571 case STATE.STATE_hex0: // reading hex number | |
| 2572 case STATE.STATE_hex: | |
| 2573 if (! ishex(cast(ubyte)c)) | |
| 2574 { | |
| 2575 if (c == '_') // ignore embedded _ | |
| 2576 { p++; | |
| 2577 continue; | |
| 2578 } | |
| 2579 if (c == '.' && p[1] != '.') | |
| 2580 goto real_; | |
| 2581 if (c == 'P' || c == 'p' || c == 'i') | |
| 2582 goto real_; | |
| 2583 if (state == STATE.STATE_hex0) | |
| 2584 error("Hex digit expected, not '%c'", c); | |
| 2585 goto done; | |
| 2586 } | |
| 2587 state = STATE.STATE_hex; | |
| 2588 break; | |
| 2589 | |
| 2590 version (ZEROH) { | |
| 2591 hexh: | |
| 2592 state = STATE.STATE_hexh; | |
| 2593 case STATE.STATE_hexh: // parse numbers like 0FFh | |
| 2594 if (!ishex(c)) | |
| 2595 { | |
| 2596 if (c == 'H' || c == 'h') | |
| 2597 { | |
| 2598 p++; | |
| 2599 base = 16; | |
| 2600 goto done; | |
| 2601 } | |
| 2602 else | |
| 2603 { | |
| 2604 // Check for something like 1E3 or 0E24 | |
| 2605 if (memchr(cast(char*)stringbuffer.data, 'E', stringbuffer.offset) || | |
| 2606 memchr(cast(char*)stringbuffer.data, 'e', stringbuffer.offset)) | |
| 2607 goto real_; | |
| 2608 error("Hex digit expected, not '%c'", c); | |
| 2609 goto done; | |
| 2610 } | |
| 2611 } | |
| 2612 break; | |
| 2613 } | |
| 2614 | |
| 2615 case STATE.STATE_octal: // reading octal number | |
| 2616 case STATE.STATE_octale: // reading octal number with non-octal digits | |
| 2617 if (!isoctal(cast(ubyte)c)) | |
| 2618 { | |
| 2619 version (ZEROH) { | |
| 2620 if (ishex(c) | |
| 2621 || c == 'H' || c == 'h' | |
| 2622 ) | |
| 2623 goto hexh; | |
| 2624 } | |
| 2625 if (c == '_') // ignore embedded _ | |
| 2626 { p++; | |
| 2627 continue; | |
| 2628 } | |
| 2629 if (c == '.' && p[1] != '.') | |
| 2630 goto real_; | |
| 2631 if (c == 'i') | |
| 2632 goto real_; | |
| 2633 if (isdigit(c)) | |
| 2634 { | |
| 2635 state = STATE.STATE_octale; | |
| 2636 } | |
| 2637 else | |
| 2638 goto done; | |
| 2639 } | |
| 2640 break; | |
| 2641 | |
| 2642 case STATE.STATE_binary0: // starting binary number | |
| 2643 case STATE.STATE_binary: // reading binary number | |
| 2644 if (c != '0' && c != '1') | |
| 2645 { | |
| 2646 version (ZEROH) { | |
| 2647 if (ishex(c) | |
| 2648 || c == 'H' || c == 'h' | |
| 2649 ) | |
| 2650 goto hexh; | |
| 2651 } | |
| 2652 if (c == '_') // ignore embedded _ | |
| 2653 { p++; | |
| 2654 continue; | |
| 2655 } | |
| 2656 if (state == STATE.STATE_binary0) | |
| 2657 { error("binary digit expected"); | |
| 2658 state = STATE.STATE_error; | |
| 2659 break; | |
| 2660 } | |
| 2661 else | |
| 2662 goto done; | |
| 2663 } | |
| 2664 state = STATE.STATE_binary; | |
| 2665 break; | |
| 2666 | |
| 2667 case STATE.STATE_error: // for error recovery | |
| 2668 if (!isdigit(c)) // scan until non-digit | |
| 2669 goto done; | |
| 2670 break; | |
| 2671 | |
| 2672 default: | |
| 2673 assert(0); | |
| 2674 } | |
| 2675 stringbuffer.writeByte(c); | |
| 2676 p++; | |
| 2677 } | |
| 2678 done: | |
| 2679 stringbuffer.writeByte(0); // terminate string | |
| 2680 if (state == STATE.STATE_octale) | |
| 2681 error("Octal digit expected"); | |
| 2682 | |
| 2683 ulong n; // unsigned >=64 bit integer type | |
| 2684 | |
| 2685 if (stringbuffer.offset == 2 && (state == STATE.STATE_decimal || state == STATE.STATE_0)) | |
| 2686 n = stringbuffer.data[0] - '0'; | |
| 2687 else | |
| 2688 { | |
| 2689 // Convert string to integer | |
| 2690 version (__DMC__) { | |
| 2691 errno = 0; | |
| 2692 n = strtoull(cast(char*)stringbuffer.data,null,base); | |
| 2693 if (errno == ERANGE) | |
| 2694 error("integer overflow"); | |
| 2695 } else { | |
| 2696 // Not everybody implements strtoull() | |
| 2697 char* p = cast(char*)stringbuffer.data; | |
| 2698 int r = 10, d; | |
| 2699 | |
| 2700 if (*p == '0') | |
| 2701 { | |
| 2702 if (p[1] == 'x' || p[1] == 'X') | |
| 2703 p += 2, r = 16; | |
| 2704 else if (p[1] == 'b' || p[1] == 'B') | |
| 2705 p += 2, r = 2; | |
| 2706 else if (isdigit(p[1])) | |
| 2707 p += 1, r = 8; | |
| 2708 } | |
| 2709 | |
| 2710 n = 0; | |
| 2711 while (1) | |
| 2712 { | |
| 2713 if (*p >= '0' && *p <= '9') | |
| 2714 d = *p - '0'; | |
| 2715 else if (*p >= 'a' && *p <= 'z') | |
| 2716 d = *p - 'a' + 10; | |
| 2717 else if (*p >= 'A' && *p <= 'Z') | |
| 2718 d = *p - 'A' + 10; | |
| 2719 else | |
| 2720 break; | |
| 2721 if (d >= r) | |
| 2722 break; | |
| 2723 ulong n2 = n * r; | |
| 2724 //printf("n2 / r = %llx, n = %llx\n", n2/r, n); | |
| 2725 if (n2 / r != n || n2 + d < n) | |
| 2726 { | |
| 2727 error ("integer overflow"); | |
| 2728 break; | |
| 2729 } | |
| 2730 | |
| 2731 n = n2 + d; | |
| 2732 p++; | |
| 2733 } | |
| 2734 } | |
| 2735 if (n.sizeof > 8 && | |
| 2736 n > 0xFFFFFFFFFFFFFFFF) // if n needs more than 64 bits | |
| 2737 error("integer overflow"); | |
| 2738 } | |
| 2739 | |
| 2740 // Parse trailing 'u', 'U', 'l' or 'L' in any combination | |
| 2741 while (1) | |
| 2742 { FLAGS f; | |
| 2743 | |
| 2744 switch (*p) | |
| 2745 { case 'U': | |
| 2746 case 'u': | |
| 2747 f = FLAGS.FLAGS_unsigned; | |
| 2748 goto L1; | |
| 2749 | |
| 2750 case 'l': | |
| 2751 if (1 || !global.params.useDeprecated) | |
| 2752 error("'l' suffix is deprecated, use 'L' instead"); | |
| 2753 case 'L': | |
| 2754 f = FLAGS.FLAGS_long; | |
| 2755 L1: | |
| 2756 p++; | |
| 2757 if (flags & f) | |
| 2758 error("unrecognized token"); | |
| 2759 flags = (flags | f); | |
| 2760 continue; | |
| 2761 default: | |
| 2762 break; | |
| 2763 } | |
| 2764 break; | |
| 2765 } | |
| 2766 | |
| 2767 switch (flags) | |
| 2768 { | |
| 2769 case FLAGS.FLAGS_undefined: | |
| 2770 /* Octal or Hexadecimal constant. | |
| 2771 * First that fits: int, uint, long, ulong | |
| 2772 */ | |
| 2773 if (n & 0x8000000000000000) | |
| 2774 result = TOK.TOKuns64v; | |
| 2775 else if (n & 0xFFFFFFFF00000000) | |
| 2776 result = TOK.TOKint64v; | |
| 2777 else if (n & 0x80000000) | |
| 2778 result = TOK.TOKuns32v; | |
| 2779 else | |
| 2780 result = TOK.TOKint32v; | |
| 2781 break; | |
| 2782 | |
| 2783 case FLAGS.FLAGS_decimal: | |
| 2784 /* First that fits: int, long, long long | |
| 2785 */ | |
| 2786 if (n & 0x8000000000000000) | |
| 2787 { error("signed integer overflow"); | |
| 2788 result = TOK.TOKuns64v; | |
| 2789 } | |
| 2790 else if (n & 0xFFFFFFFF80000000) | |
| 2791 result = TOK.TOKint64v; | |
| 2792 else | |
| 2793 result = TOK.TOKint32v; | |
| 2794 break; | |
| 2795 | |
| 2796 case FLAGS.FLAGS_unsigned: | |
| 2797 case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned: | |
| 2798 /* First that fits: uint, ulong | |
| 2799 */ | |
| 2800 if (n & 0xFFFFFFFF00000000) | |
| 2801 result = TOK.TOKuns64v; | |
| 2802 else | |
| 2803 result = TOK.TOKuns32v; | |
| 2804 break; | |
| 2805 | |
| 2806 case FLAGS.FLAGS_decimal | FLAGS.FLAGS_long: | |
| 2807 if (n & 0x8000000000000000) | |
| 2808 { error("signed integer overflow"); | |
| 2809 result = TOK.TOKuns64v; | |
| 2810 } | |
| 2811 else | |
| 2812 result = TOK.TOKint64v; | |
| 2813 break; | |
| 2814 | |
| 2815 case FLAGS.FLAGS_long: | |
| 2816 if (n & 0x8000000000000000) | |
| 2817 result = TOK.TOKuns64v; | |
| 2818 else | |
| 2819 result = TOK.TOKint64v; | |
| 2820 break; | |
| 2821 | |
| 2822 case FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long: | |
| 2823 case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long: | |
| 2824 result = TOK.TOKuns64v; | |
| 2825 break; | |
| 2826 | |
| 2827 default: | |
| 2828 debug { | |
| 2829 printf("%x\n",flags); | |
| 2830 } | |
| 2831 assert(0); | |
| 2832 } | |
| 2833 t.uns64value = n; | |
| 2834 return result; | |
| 2835 } | |
| 2836 | |
| 2837 /************************************** | |
| 2838 * Read in characters, converting them to real. | |
| 2839 * Bugs: | |
| 2840 * Exponent overflow not detected. | |
| 2841 * Too much requested precision is not detected. | |
| 2842 */ | |
| 2843 TOK inreal(Token* t) | |
| 2844 in | |
| 2845 { | |
| 2846 assert(*p == '.' || isdigit(*p)); | |
| 2847 } | |
| 2848 out (result) | |
| 2849 { | |
| 2850 switch (result) | |
| 2851 { | |
| 2852 case TOKfloat32v: | |
| 2853 case TOKfloat64v: | |
| 2854 case TOKfloat80v: | |
| 2855 case TOKimaginary32v: | |
| 2856 case TOKimaginary64v: | |
| 2857 case TOKimaginary80v: | |
| 2858 break; | |
| 2859 | |
| 2860 default: | |
| 2861 assert(0); | |
| 2862 } | |
| 2863 } | |
| 2864 body | |
| 2865 { | |
| 2866 int dblstate; | |
| 2867 uint c; | |
| 2868 char hex; // is this a hexadecimal-floating-constant? | |
| 2869 TOK result; | |
| 2870 | |
| 2871 //printf("Lexer.inreal()\n"); | |
| 2872 stringbuffer.reset(); | |
| 2873 dblstate = 0; | |
| 2874 hex = 0; | |
| 2875 Lnext: | |
| 2876 while (true) | |
| 2877 { | |
| 2878 // Get next char from input | |
| 2879 c = *p++; | |
| 2880 //printf("dblstate = %d, c = '%c'\n", dblstate, c); | |
| 2881 while (true) | |
| 2882 { | |
| 2883 switch (dblstate) | |
| 2884 { | |
| 2885 case 0: // opening state | |
| 2886 if (c == '0') | |
| 2887 dblstate = 9; | |
| 2888 else if (c == '.') | |
| 2889 dblstate = 3; | |
| 2890 else | |
| 2891 dblstate = 1; | |
| 2892 break; | |
| 2893 | |
| 2894 case 9: | |
| 2895 dblstate = 1; | |
| 2896 if (c == 'X' || c == 'x') | |
| 178 | 2897 { |
| 0 | 2898 hex++; |
| 2899 break; | |
| 2900 } | |
| 2901 case 1: // digits to left of . | |
| 2902 case 3: // digits to right of . | |
| 2903 case 7: // continuing exponent digits | |
| 2904 if (!isdigit(c) && !(hex && isxdigit(c))) | |
| 2905 { | |
| 2906 if (c == '_') | |
| 2907 goto Lnext; // ignore embedded '_' | |
| 2908 dblstate++; | |
| 2909 continue; | |
| 2910 } | |
| 2911 break; | |
| 2912 | |
| 2913 case 2: // no more digits to left of . | |
| 2914 if (c == '.') | |
| 178 | 2915 { |
| 0 | 2916 dblstate++; |
| 2917 break; | |
| 2918 } | |
| 2919 case 4: // no more digits to right of . | |
| 2920 if ((c == 'E' || c == 'e') || | |
| 2921 hex && (c == 'P' || c == 'p')) | |
| 178 | 2922 { |
| 0 | 2923 dblstate = 5; |
| 2924 hex = 0; // exponent is always decimal | |
| 2925 break; | |
| 2926 } | |
| 2927 if (hex) | |
| 2928 error("binary-exponent-part required"); | |
| 2929 goto done; | |
| 2930 | |
| 2931 case 5: // looking immediately to right of E | |
| 2932 dblstate++; | |
| 2933 if (c == '-' || c == '+') | |
| 2934 break; | |
| 2935 case 6: // 1st exponent digit expected | |
| 2936 if (!isdigit(c)) | |
| 2937 error("exponent expected"); | |
| 2938 dblstate++; | |
| 2939 break; | |
| 2940 | |
| 2941 case 8: // past end of exponent digits | |
| 2942 goto done; | |
| 79 | 2943 |
| 2944 default: | |
| 2945 assert(0, "inreal.dblstate has unexpected value"); | |
| 0 | 2946 } |
| 2947 break; | |
| 2948 } | |
| 2949 stringbuffer.writeByte(c); | |
| 2950 } | |
| 2951 done: | |
| 2952 p--; | |
| 2953 | |
| 2954 stringbuffer.writeByte(0); | |
| 2955 | |
| 114 | 2956 version (Windows) { /// && __DMC__ |
| 0 | 2957 char* save = __locale_decpoint; |
| 2958 __locale_decpoint = cast(char*)".".ptr; | |
| 2959 } | |
| 2960 t.float80value = strtold(cast(char*)stringbuffer.data, null); | |
| 2961 | |
| 2962 errno = 0; | |
| 2963 switch (*p) | |
| 2964 { | |
| 2965 case 'F': | |
| 2966 case 'f': | |
| 2967 strtof(cast(char*)stringbuffer.data, null); | |
| 2968 result = TOKfloat32v; | |
| 2969 p++; | |
| 2970 break; | |
| 2971 | |
| 2972 default: | |
| 2973 strtod(cast(char*)stringbuffer.data, null); | |
| 2974 result = TOKfloat64v; | |
| 2975 break; | |
| 2976 | |
| 2977 case 'l': | |
| 2978 if (!global.params.useDeprecated) | |
| 2979 error("'l' suffix is deprecated, use 'L' instead"); | |
| 2980 case 'L': | |
| 2981 result = TOKfloat80v; | |
| 2982 p++; | |
| 2983 break; | |
| 2984 } | |
| 2985 if (*p == 'i' || *p == 'I') | |
| 2986 { | |
| 2987 if (!global.params.useDeprecated && *p == 'I') | |
| 2988 error("'I' suffix is deprecated, use 'i' instead"); | |
| 2989 p++; | |
| 2990 switch (result) | |
| 2991 { | |
| 2992 case TOKfloat32v: | |
| 2993 result = TOKimaginary32v; | |
| 2994 break; | |
| 2995 case TOKfloat64v: | |
| 2996 result = TOKimaginary64v; | |
| 2997 break; | |
| 2998 case TOKfloat80v: | |
| 2999 result = TOKimaginary80v; | |
| 3000 break; | |
|
157
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3001 default: |
| 0 | 3002 } |
| 3003 } | |
| 178 | 3004 |
| 114 | 3005 version (Windows) { ///&& __DMC__ |
| 0 | 3006 __locale_decpoint = save; |
| 3007 } | |
| 3008 if (errno == ERANGE) | |
| 3009 error("number is not representable"); | |
| 3010 | |
| 3011 return result; | |
| 3012 } | |
| 3013 | |
| 3014 void error(T...)(string format, T t) | |
| 3015 { | |
| 3016 error(this.loc, format, t); | |
| 3017 } | |
| 3018 | |
| 3019 void error(T...)(Loc loc, string format, T t) | |
| 3020 { | |
| 3021 if (mod && !global.gag) | |
| 3022 { | |
| 3023 string p = loc.toChars(); | |
| 3024 if (p.length != 0) | |
| 3025 writef("%s: ", p); | |
| 3026 | |
| 3027 writefln(format, t); | |
| 3028 | |
| 3029 if (global.errors >= 20) // moderate blizzard of cascading messages | |
| 3030 fatal(); | |
| 3031 } | |
| 3032 | |
| 3033 global.errors++; | |
| 3034 } | |
| 178 | 3035 |
| 162 | 3036 /********************************************* |
| 3037 * Do pragma. | |
| 3038 * Currently, the only pragma supported is: | |
| 3039 * #line linnum [filespec] | |
| 3040 */ | |
| 0 | 3041 void pragma_() |
| 3042 { | |
| 162 | 3043 Token tok; |
| 3044 int linnum; | |
| 3045 string filespec = null; | |
| 3046 Loc loc = this.loc; | |
| 3047 | |
| 3048 scan(&tok); | |
| 3049 if (tok.value != TOKidentifier || tok.ident != Id.line) | |
| 3050 goto Lerr; | |
| 3051 | |
| 3052 scan(&tok); | |
| 3053 if (tok.value == TOKint32v || tok.value == TOKint64v) | |
| 3054 linnum = cast(int)(tok.uns64value - 1); /// | |
| 3055 else | |
| 3056 goto Lerr; | |
| 3057 | |
| 3058 while (1) | |
| 3059 { | |
| 3060 switch (*p) | |
| 3061 { | |
| 3062 case 0: | |
| 3063 case 0x1A: | |
| 3064 case '\n': | |
| 3065 Lnewline: | |
| 3066 this.loc.linnum = linnum; | |
| 3067 if (filespec != null) | |
| 3068 this.loc.filename = filespec; | |
| 3069 return; | |
| 3070 | |
| 3071 case '\r': | |
| 3072 p++; | |
| 3073 if (*p != '\n') | |
| 3074 { p--; | |
| 3075 goto Lnewline; | |
| 3076 } | |
| 3077 continue; | |
| 3078 | |
| 3079 case ' ': | |
| 3080 case '\t': | |
| 3081 case '\v': | |
| 3082 case '\f': | |
| 3083 p++; | |
| 3084 continue; // skip white space | |
| 3085 | |
| 3086 case '_': | |
| 3087 if (mod && memcmp(p, "__FILE__".ptr, 8) == 0) | |
| 3088 { | |
| 3089 p += 8; | |
| 3090 filespec = (loc.filename ? loc.filename : mod.ident.toChars()); | |
| 3091 } | |
| 3092 continue; | |
| 3093 | |
| 3094 case '"': | |
| 3095 if (filespec) | |
| 3096 goto Lerr; | |
| 3097 stringbuffer.reset(); | |
| 3098 p++; | |
| 3099 while (1) | |
| 178 | 3100 { |
| 162 | 3101 uint c; |
| 3102 | |
| 3103 c = *p; | |
| 3104 switch (c) | |
| 3105 { | |
| 3106 case '\n': | |
| 3107 case '\r': | |
| 3108 case 0: | |
| 3109 case 0x1A: | |
| 3110 goto Lerr; | |
| 3111 | |
| 3112 case '"': | |
| 3113 stringbuffer.writeByte(0); | |
| 3114 filespec = stringbuffer.extractString(); /// | |
| 3115 p++; | |
| 3116 break; | |
| 3117 | |
| 3118 default: | |
| 3119 if (c & 0x80) | |
| 178 | 3120 { |
| 162 | 3121 uint u = decodeUTF(); |
| 3122 if (u == PS || u == LS) | |
| 3123 goto Lerr; | |
| 3124 } | |
| 3125 stringbuffer.writeByte(c); | |
| 3126 p++; | |
| 3127 continue; | |
| 3128 } | |
| 3129 break; | |
| 3130 } | |
| 3131 continue; | |
| 3132 | |
| 3133 default: | |
| 3134 if (*p & 0x80) | |
| 178 | 3135 { |
| 162 | 3136 uint u = decodeUTF(); |
| 3137 if (u == PS || u == LS) | |
| 3138 goto Lnewline; | |
| 3139 } | |
| 3140 goto Lerr; | |
| 3141 } | |
| 3142 } | |
| 3143 | |
| 3144 Lerr: | |
| 3145 error(loc, "#line integer [\"filespec\"]\\n expected"); | |
| 0 | 3146 } |
| 3147 | |
| 49 | 3148 /******************************************** |
| 3149 * Decode UTF character. | |
| 3150 * Issue error messages for invalid sequences. | |
| 3151 * Return decoded character, advance p to last character in UTF sequence. | |
| 3152 */ | |
| 0 | 3153 uint decodeUTF() |
| 3154 { | |
| 49 | 3155 dchar u; |
| 3156 ubyte c; | |
| 3157 ubyte* s = p; | |
| 3158 size_t len; | |
| 3159 size_t idx; | |
| 3160 string msg; | |
| 3161 | |
| 3162 c = *s; | |
| 3163 assert(c & 0x80); | |
| 3164 | |
| 3165 // Check length of remaining string up to 6 UTF-8 characters | |
| 3166 for (len = 1; len < 6 && s[len]; len++) { | |
| 3167 ; | |
| 3168 } | |
| 3169 | |
| 3170 idx = 0; | |
| 3171 msg = utf_decodeChar(cast(string)s[0..len], &idx, &u); | |
| 3172 p += idx - 1; | |
| 3173 if (msg) | |
| 3174 { | |
| 3175 error("%s", msg); | |
| 3176 } | |
| 3177 return u; | |
| 0 | 3178 } |
| 3179 | |
|
157
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3180 /*************************************************** |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3181 * Parse doc comment embedded between t.ptr and p. |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3182 * Remove trailing blanks and tabs from lines. |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3183 * Replace all newlines with \n. |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3184 * Remove leading comment character from each line. |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3185 * Decide if it's a lineComment or a blockComment. |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3186 * Append to previous one for this token. |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3187 */ |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3188 void getDocComment(Token* t, uint lineComment) |
| 0 | 3189 { |
|
157
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3190 /* ct tells us which kind of comment it is: '!', '/', '*', or '+' |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3191 */ |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3192 ubyte ct = t.ptr[2]; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3193 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3194 /* Start of comment text skips over / * *, / + +, or / / / |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3195 */ |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3196 ubyte* q = t.ptr + 3; // start of comment text |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3197 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3198 ubyte* qend = p; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3199 if (ct == '*' || ct == '+') |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3200 qend -= 2; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3201 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3202 /* Scan over initial row of ****'s or ++++'s or ////'s |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3203 */ |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3204 for (; q < qend; q++) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3205 { |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3206 if (*q != ct) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3207 break; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3208 } |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3209 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3210 /* Remove trailing row of ****'s or ++++'s |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3211 */ |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3212 if (ct != '/' && ct != '!') |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3213 { |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3214 for (; q < qend; qend--) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3215 { |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3216 if (qend[-1] != ct) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3217 break; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3218 } |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3219 } |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3220 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3221 /* Comment is now [q .. qend]. |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3222 * Canonicalize it into buf[]. |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3223 */ |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3224 OutBuffer buf = new OutBuffer; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3225 int linestart = 0; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3226 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3227 for (; q < qend; q++) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3228 { |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3229 ubyte c = *q; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3230 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3231 switch (c) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3232 { |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3233 case '*': |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3234 case '+': |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3235 if (linestart && c == ct) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3236 { linestart = 0; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3237 /* Trim preceding whitespace up to preceding \n |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3238 */ |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3239 while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3240 buf.offset--; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3241 continue; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3242 } |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3243 break; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3244 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3245 case ' ': |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3246 case '\t': |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3247 break; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3248 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3249 case '\r': |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3250 if (q[1] == '\n') |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3251 continue; // skip the \r |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3252 goto Lnewline; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3253 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3254 default: |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3255 if (c == 226) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3256 { |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3257 // If LS or PS |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3258 if (q[1] == 128 && |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3259 (q[2] == 168 || q[2] == 169)) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3260 { |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3261 q += 2; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3262 goto Lnewline; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3263 } |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3264 } |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3265 linestart = 0; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3266 break; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3267 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3268 Lnewline: |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3269 c = '\n'; // replace all newlines with \n |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3270 case '\n': |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3271 linestart = 1; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3272 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3273 /* Trim trailing whitespace |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3274 */ |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3275 while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3276 buf.offset--; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3277 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3278 break; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3279 } |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3280 buf.writeByte(c); |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3281 } |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3282 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3283 // Always end with a newline |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3284 if (!buf.offset || buf.data[buf.offset - 1] != '\n') |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3285 buf.writeByte('\n'); |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3286 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3287 buf.writeByte(0); |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3288 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3289 // It's a line comment if the start of the doc comment comes |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3290 // after other non-whitespace on the same line. |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3291 string* dc = (lineComment && anyToken) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3292 ? &t.lineComment |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3293 : &t.blockComment; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3294 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3295 // Combine with previous doc comment, if any |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3296 if (*dc) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3297 *dc = combineComments(*dc, cast(string) buf.data[0 .. buf.size]); // TODO: utf decode etc? |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3298 else |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3299 { |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3300 auto bufsize = buf.size; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3301 *dc = cast(string) buf.extractData()[0..bufsize]; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3302 } |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3303 } |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3304 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3305 /******************************************** |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3306 * Combine two document comments into one, |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3307 * separated by a newline. |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3308 */ |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3309 static string combineComments(string c1, string c2) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3310 { |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3311 //printf("Lexer::combineComments('%s', '%s')\n", c1, c2); |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3312 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3313 string c = c2; |
| 178 | 3314 |
|
157
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3315 if (c1) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3316 { |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3317 c = c1; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3318 if (c2) |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3319 { |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3320 size_t len1 = c1.length; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3321 size_t len2 = c2.length; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3322 |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3323 c = c1.idup; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3324 if (len1 && c1[$-1] != '\n') |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3325 c ~= '\n'; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3326 c ~= c2; |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3327 } |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3328 } |
|
b7b61140701d
* added all missing default cases in switch statements
trass3r
parents:
135
diff
changeset
|
3329 return c; |
| 0 | 3330 } |
| 3331 | |
| 3332 static bool isValidIdentifier(string p) | |
| 3333 { | |
| 3334 if (p.length == 0) { | |
| 3335 return false; | |
| 3336 } | |
| 3337 | |
| 3338 if (p[0] >= '0' && p[0] <= '9') { // beware of isdigit() on signed chars | |
| 3339 return false; | |
| 3340 } | |
| 3341 | |
| 3342 size_t idx = 0; | |
| 3343 while (idx < p.length) | |
| 3344 { | |
| 3345 dchar dc; | |
| 3346 | |
| 3347 if (utf_decodeChar(p, &idx, &dc) !is null) { | |
| 3348 return false; | |
| 3349 } | |
| 3350 | |
| 3351 if (!((dc >= 0x80 && isUniAlpha(dc)) || isalnum(dc) || dc == '_')) { | |
| 3352 return false; | |
| 3353 } | |
| 3354 } | |
| 3355 | |
| 3356 return true; | |
| 3357 } | |
| 3358 | |
| 79 | 3359 /// TODO: use normal string append when GC works |
| 3360 static string combineComments(const(char)[] c1, const(char)[] c2) | |
| 0 | 3361 { |
| 79 | 3362 //writef("Lexer.combineComments('%s', '%s')\n", c1, c2); |
| 0 | 3363 |
| 79 | 3364 char[] c = cast(char[]) c2; |
| 178 | 3365 |
| 79 | 3366 if (c1 !is null) |
| 0 | 3367 { |
| 79 | 3368 c = cast(char[]) c1; |
| 3369 if (c2 !is null) | |
| 0 | 3370 { |
| 79 | 3371 c = cast(char[]) (GC.malloc(c1.length + 1 + c2.length)[0 .. c1.length + 1 + c2.length]); |
| 3372 size_t len1 = c1.length; | |
| 3373 c[0..len1] = c1[]; | |
| 3374 c[len1++] = '\n'; | |
| 3375 c[len1 .. len1 + c2.length] = c2[]; | |
| 0 | 3376 } |
| 3377 } | |
| 178 | 3378 |
| 79 | 3379 return cast(string)c; |
| 0 | 3380 } |
| 178 | 3381 } |