Mercurial > projects > ldc
annotate dmd/lexer.c @ 650:aa6a0b7968f7
Added test case for bug #100
Removed dubious check for not emitting static private global in other modules without access. This should be handled properly somewhere else, it's causing unresolved global errors for stuff that should work (in MiniD)
| author | Tomas Lindquist Olsen <tomas.l.olsen@gmail.com> |
|---|---|
| date | Sun, 05 Oct 2008 17:28:15 +0200 |
| parents | 1d3026702f65 |
| children | 50383e476c7e |
| rev | line source |
|---|---|
| 159 | 1 |
| 2 // Compiler implementation of the D programming language | |
| 3 // Copyright (c) 1999-2008 by Digital Mars | |
| 4 // All Rights Reserved | |
| 5 // written by Walter Bright | |
| 6 // http://www.digitalmars.com | |
| 7 // License for redistribution is by either the Artistic License | |
| 8 // in artistic.txt, or the GNU General Public License in gnu.txt. | |
| 9 // See the included readme.txt for details. | |
| 10 | |
| 11 /* Lexical Analyzer */ | |
| 12 | |
| 13 #include <stdio.h> | |
| 14 #include <string.h> | |
| 15 #include <ctype.h> | |
| 16 #include <stdarg.h> | |
| 17 #include <errno.h> | |
| 18 #include <wchar.h> | |
| 19 #include <stdlib.h> | |
| 20 #include <assert.h> | |
| 21 #include <sys/time.h> | |
| 22 | |
| 23 #ifdef IN_GCC | |
| 24 | |
| 25 #include <time.h> | |
| 26 #include "mem.h" | |
| 27 | |
| 28 #else | |
| 29 | |
| 30 #if __GNUC__ | |
| 31 #include <time.h> | |
| 32 #endif | |
| 33 | |
| 34 #if IN_LLVM | |
| 35 #include "mem.h" | |
| 36 #elif _WIN32 | |
| 37 #include "..\root\mem.h" | |
| 38 #else | |
| 39 #include "../root/mem.h" | |
| 40 #endif | |
| 41 #endif | |
| 42 | |
| 43 #include "stringtable.h" | |
| 44 | |
| 45 #include "lexer.h" | |
| 46 #include "utf.h" | |
| 47 #include "identifier.h" | |
| 48 #include "id.h" | |
| 49 #include "module.h" | |
| 50 | |
| 51 #if _WIN32 && __DMC__ | |
| 52 // from \dm\src\include\setlocal.h | |
| 53 extern "C" char * __cdecl __locale_decpoint; | |
| 54 #endif | |
| 55 | |
| 56 extern int HtmlNamedEntity(unsigned char *p, int length); | |
| 57 | |
| 58 #define LS 0x2028 // UTF line separator | |
| 59 #define PS 0x2029 // UTF paragraph separator | |
| 60 | |
| 61 /******************************************** | |
| 62 * Do our own char maps | |
| 63 */ | |
| 64 | |
| 65 static unsigned char cmtable[256]; | |
| 66 | |
| 67 const int CMoctal = 0x1; | |
| 68 const int CMhex = 0x2; | |
| 69 const int CMidchar = 0x4; | |
| 70 | |
| 71 inline unsigned char isoctal (unsigned char c) { return cmtable[c] & CMoctal; } | |
| 72 inline unsigned char ishex (unsigned char c) { return cmtable[c] & CMhex; } | |
| 73 inline unsigned char isidchar(unsigned char c) { return cmtable[c] & CMidchar; } | |
| 74 | |
| 75 static void cmtable_init() | |
| 76 { | |
| 77 for (unsigned c = 0; c < sizeof(cmtable) / sizeof(cmtable[0]); c++) | |
| 78 { | |
| 79 if ('0' <= c && c <= '7') | |
| 80 cmtable[c] |= CMoctal; | |
| 81 if (isdigit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F')) | |
| 82 cmtable[c] |= CMhex; | |
| 83 if (isalnum(c) || c == '_') | |
| 84 cmtable[c] |= CMidchar; | |
| 85 } | |
| 86 } | |
| 87 | |
| 88 | |
| 89 /************************* Token **********************************************/ | |
| 90 | |
| 91 char *Token::tochars[TOKMAX]; | |
| 92 | |
| 93 void *Token::operator new(size_t size) | |
| 94 { Token *t; | |
| 95 | |
| 96 if (Lexer::freelist) | |
| 97 { | |
| 98 t = Lexer::freelist; | |
| 99 Lexer::freelist = t->next; | |
| 100 return t; | |
| 101 } | |
| 102 | |
| 103 return ::operator new(size); | |
| 104 } | |
| 105 | |
| 106 #ifdef DEBUG | |
| 107 void Token::print() | |
| 108 { | |
| 109 fprintf(stdmsg, "%s\n", toChars()); | |
| 110 } | |
| 111 #endif | |
| 112 | |
| 113 char *Token::toChars() | |
| 114 { char *p; | |
| 115 static char buffer[3 + 3 * sizeof(value) + 1]; | |
| 116 | |
| 117 p = buffer; | |
| 118 switch (value) | |
| 119 { | |
| 120 case TOKint32v: | |
| 121 #if IN_GCC | |
| 122 sprintf(buffer,"%d",(d_int32)int64value); | |
| 123 #else | |
| 124 sprintf(buffer,"%d",int32value); | |
| 125 #endif | |
| 126 break; | |
| 127 | |
| 128 case TOKuns32v: | |
| 129 case TOKcharv: | |
| 130 case TOKwcharv: | |
| 131 case TOKdcharv: | |
| 132 #if IN_GCC | |
| 133 sprintf(buffer,"%uU",(d_uns32)uns64value); | |
| 134 #else | |
| 135 sprintf(buffer,"%uU",uns32value); | |
| 136 #endif | |
| 137 break; | |
| 138 | |
| 139 case TOKint64v: | |
|
305
2b72433d5c8c
[svn r326] Fixed a bunch of issues with printf's that MinGW32 did not support.
lindquist
parents:
159
diff
changeset
|
140 sprintf(buffer,"%lldL",int64value); |
| 159 | 141 break; |
| 142 | |
| 143 case TOKuns64v: | |
|
305
2b72433d5c8c
[svn r326] Fixed a bunch of issues with printf's that MinGW32 did not support.
lindquist
parents:
159
diff
changeset
|
144 sprintf(buffer,"%lluUL",uns64value); |
| 159 | 145 break; |
| 146 | |
| 147 #if IN_GCC | |
| 148 case TOKfloat32v: | |
| 149 case TOKfloat64v: | |
| 150 case TOKfloat80v: | |
| 151 float80value.format(buffer, sizeof(buffer)); | |
| 152 break; | |
| 153 case TOKimaginary32v: | |
| 154 case TOKimaginary64v: | |
| 155 case TOKimaginary80v: | |
| 156 float80value.format(buffer, sizeof(buffer)); | |
| 157 // %% buffer | |
| 158 strcat(buffer, "i"); | |
| 159 break; | |
| 160 #else | |
| 161 case TOKfloat32v: | |
| 162 sprintf(buffer,"%Lgf", float80value); | |
| 163 break; | |
| 164 | |
| 165 case TOKfloat64v: | |
| 166 sprintf(buffer,"%Lg", float80value); | |
| 167 break; | |
| 168 | |
| 169 case TOKfloat80v: | |
| 170 sprintf(buffer,"%LgL", float80value); | |
| 171 break; | |
| 172 | |
| 173 case TOKimaginary32v: | |
| 174 sprintf(buffer,"%Lgfi", float80value); | |
| 175 break; | |
| 176 | |
| 177 case TOKimaginary64v: | |
| 178 sprintf(buffer,"%Lgi", float80value); | |
| 179 break; | |
| 180 | |
| 181 case TOKimaginary80v: | |
| 182 sprintf(buffer,"%LgLi", float80value); | |
| 183 break; | |
| 184 #endif | |
| 185 | |
| 186 case TOKstring: | |
| 187 #if CSTRINGS | |
| 188 p = string; | |
| 189 #else | |
| 190 { OutBuffer buf; | |
| 191 | |
| 192 buf.writeByte('"'); | |
| 193 for (size_t i = 0; i < len; ) | |
| 194 { unsigned c; | |
| 195 | |
| 196 utf_decodeChar((unsigned char *)ustring, len, &i, &c); | |
| 197 switch (c) | |
| 198 { | |
| 199 case 0: | |
| 200 break; | |
| 201 | |
| 202 case '"': | |
| 203 case '\\': | |
| 204 buf.writeByte('\\'); | |
| 205 default: | |
| 206 if (isprint(c)) | |
| 207 buf.writeByte(c); | |
| 208 else if (c <= 0x7F) | |
| 209 buf.printf("\\x%02x", c); | |
| 210 else if (c <= 0xFFFF) | |
| 211 buf.printf("\\u%04x", c); | |
| 212 else | |
| 213 buf.printf("\\U%08x", c); | |
| 214 continue; | |
| 215 } | |
| 216 break; | |
| 217 } | |
| 218 buf.writeByte('"'); | |
| 219 if (postfix) | |
| 220 buf.writeByte('"'); | |
| 221 buf.writeByte(0); | |
| 222 p = (char *)buf.extractData(); | |
| 223 } | |
| 224 #endif | |
| 225 break; | |
| 226 | |
| 227 case TOKidentifier: | |
| 228 case TOKenum: | |
| 229 case TOKstruct: | |
| 230 case TOKimport: | |
| 231 CASE_BASIC_TYPES: | |
| 232 p = ident->toChars(); | |
| 233 break; | |
| 234 | |
| 235 default: | |
| 236 p = toChars(value); | |
| 237 break; | |
| 238 } | |
| 239 return p; | |
| 240 } | |
| 241 | |
| 242 char *Token::toChars(enum TOK value) | |
| 243 { char *p; | |
| 244 static char buffer[3 + 3 * sizeof(value) + 1]; | |
| 245 | |
| 246 p = tochars[value]; | |
| 247 if (!p) | |
| 248 { sprintf(buffer,"TOK%d",value); | |
| 249 p = buffer; | |
| 250 } | |
| 251 return p; | |
| 252 } | |
| 253 | |
| 254 /*************************** Lexer ********************************************/ | |
| 255 | |
| 256 Token *Lexer::freelist = NULL; | |
| 257 StringTable Lexer::stringtable; | |
| 258 OutBuffer Lexer::stringbuffer; | |
| 259 | |
| 260 Lexer::Lexer(Module *mod, | |
| 261 unsigned char *base, unsigned begoffset, unsigned endoffset, | |
| 262 int doDocComment, int commentToken) | |
| 263 : loc(mod, 1) | |
| 264 { | |
| 265 //printf("Lexer::Lexer(%p,%d)\n",base,length); | |
| 266 //printf("lexer.mod = %p, %p\n", mod, this->loc.mod); | |
| 267 memset(&token,0,sizeof(token)); | |
| 268 this->base = base; | |
| 269 this->end = base + endoffset; | |
| 270 p = base + begoffset; | |
| 271 this->mod = mod; | |
| 272 this->doDocComment = doDocComment; | |
| 273 this->anyToken = 0; | |
| 274 this->commentToken = commentToken; | |
| 275 //initKeywords(); | |
| 276 | |
| 277 /* If first line starts with '#!', ignore the line | |
| 278 */ | |
| 279 | |
| 280 if (p[0] == '#' && p[1] =='!') | |
| 281 { | |
| 282 p += 2; | |
| 283 while (1) | |
| 284 { unsigned char c = *p; | |
| 285 switch (c) | |
| 286 { | |
| 287 case '\n': | |
| 288 p++; | |
| 289 break; | |
| 290 | |
| 291 case '\r': | |
| 292 p++; | |
| 293 if (*p == '\n') | |
| 294 p++; | |
| 295 break; | |
| 296 | |
| 297 case 0: | |
| 298 case 0x1A: | |
| 299 break; | |
| 300 | |
| 301 default: | |
| 302 if (c & 0x80) | |
| 303 { unsigned u = decodeUTF(); | |
| 304 if (u == PS || u == LS) | |
| 305 break; | |
| 306 } | |
| 307 p++; | |
| 308 continue; | |
| 309 } | |
| 310 break; | |
| 311 } | |
| 312 loc.linnum = 2; | |
| 313 } | |
| 314 } | |
| 315 | |
| 316 | |
| 317 void Lexer::error(const char *format, ...) | |
| 318 { | |
| 319 if (mod && !global.gag) | |
| 320 { | |
| 321 char *p = loc.toChars(); | |
| 322 if (*p) | |
| 323 fprintf(stdmsg, "%s: ", p); | |
| 324 mem.free(p); | |
| 325 | |
| 326 va_list ap; | |
| 327 va_start(ap, format); | |
| 328 vfprintf(stdmsg, format, ap); | |
| 329 va_end(ap); | |
| 330 | |
| 331 fprintf(stdmsg, "\n"); | |
| 332 fflush(stdmsg); | |
| 333 | |
| 334 if (global.errors >= 20) // moderate blizzard of cascading messages | |
| 335 fatal(); | |
| 336 } | |
| 337 global.errors++; | |
| 338 } | |
| 339 | |
| 340 void Lexer::error(Loc loc, const char *format, ...) | |
| 341 { | |
| 342 if (mod && !global.gag) | |
| 343 { | |
| 344 char *p = loc.toChars(); | |
| 345 if (*p) | |
| 346 fprintf(stdmsg, "%s: ", p); | |
| 347 mem.free(p); | |
| 348 | |
| 349 va_list ap; | |
| 350 va_start(ap, format); | |
| 351 vfprintf(stdmsg, format, ap); | |
| 352 va_end(ap); | |
| 353 | |
| 354 fprintf(stdmsg, "\n"); | |
| 355 fflush(stdmsg); | |
| 356 | |
| 357 if (global.errors >= 20) // moderate blizzard of cascading messages | |
| 358 fatal(); | |
| 359 } | |
| 360 global.errors++; | |
| 361 } | |
| 362 | |
| 363 TOK Lexer::nextToken() | |
| 364 { Token *t; | |
| 365 | |
| 366 if (token.next) | |
| 367 { | |
| 368 t = token.next; | |
| 369 memcpy(&token,t,sizeof(Token)); | |
| 370 t->next = freelist; | |
| 371 freelist = t; | |
| 372 } | |
| 373 else | |
| 374 { | |
| 375 scan(&token); | |
| 376 } | |
| 377 //token.print(); | |
| 378 return token.value; | |
| 379 } | |
| 380 | |
| 381 Token *Lexer::peek(Token *ct) | |
| 382 { Token *t; | |
| 383 | |
| 384 if (ct->next) | |
| 385 t = ct->next; | |
| 386 else | |
| 387 { | |
| 388 t = new Token(); | |
| 389 scan(t); | |
| 390 t->next = NULL; | |
| 391 ct->next = t; | |
| 392 } | |
| 393 return t; | |
| 394 } | |
| 395 | |
| 396 /********************************* | |
| 397 * tk is on the opening (. | |
| 398 * Look ahead and return token that is past the closing ). | |
| 399 */ | |
| 400 | |
| 401 Token *Lexer::peekPastParen(Token *tk) | |
| 402 { | |
| 403 //printf("peekPastParen()\n"); | |
| 404 int parens = 1; | |
| 405 int curlynest = 0; | |
| 406 while (1) | |
| 407 { | |
| 408 tk = peek(tk); | |
| 409 //tk->print(); | |
| 410 switch (tk->value) | |
| 411 { | |
| 412 case TOKlparen: | |
| 413 parens++; | |
| 414 continue; | |
| 415 | |
| 416 case TOKrparen: | |
| 417 --parens; | |
| 418 if (parens) | |
| 419 continue; | |
| 420 tk = peek(tk); | |
| 421 break; | |
| 422 | |
| 423 case TOKlcurly: | |
| 424 curlynest++; | |
| 425 continue; | |
| 426 | |
| 427 case TOKrcurly: | |
| 428 if (--curlynest >= 0) | |
| 429 continue; | |
| 430 break; | |
| 431 | |
| 432 case TOKsemicolon: | |
| 433 if (curlynest) | |
| 434 continue; | |
| 435 break; | |
| 436 | |
| 437 case TOKeof: | |
| 438 break; | |
| 439 | |
| 440 default: | |
| 441 continue; | |
| 442 } | |
| 443 return tk; | |
| 444 } | |
| 445 } | |
| 446 | |
| 447 /********************************** | |
| 448 * Determine if string is a valid Identifier. | |
| 449 * Placed here because of commonality with Lexer functionality. | |
| 450 * Returns: | |
| 451 * 0 invalid | |
| 452 */ | |
| 453 | |
| 454 int Lexer::isValidIdentifier(char *p) | |
| 455 { | |
| 456 size_t len; | |
| 457 size_t idx; | |
| 458 | |
| 459 if (!p || !*p) | |
| 460 goto Linvalid; | |
| 461 | |
| 462 if (*p >= '0' && *p <= '9') // beware of isdigit() on signed chars | |
| 463 goto Linvalid; | |
| 464 | |
| 465 len = strlen(p); | |
| 466 idx = 0; | |
| 467 while (p[idx]) | |
| 468 { dchar_t dc; | |
| 469 | |
| 470 char *q = utf_decodeChar((unsigned char *)p, len, &idx, &dc); | |
| 471 if (q) | |
| 472 goto Linvalid; | |
| 473 | |
| 474 if (!((dc >= 0x80 && isUniAlpha(dc)) || isalnum(dc) || dc == '_')) | |
| 475 goto Linvalid; | |
| 476 } | |
| 477 return 1; | |
| 478 | |
| 479 Linvalid: | |
| 480 return 0; | |
| 481 } | |
| 482 | |
| 483 /**************************** | |
| 484 * Turn next token in buffer into a token. | |
| 485 */ | |
| 486 | |
| 487 void Lexer::scan(Token *t) | |
| 488 { | |
| 489 unsigned lastLine = loc.linnum; | |
| 490 unsigned linnum; | |
| 491 | |
| 492 t->blockComment = NULL; | |
| 493 t->lineComment = NULL; | |
| 494 while (1) | |
| 495 { | |
| 496 t->ptr = p; | |
| 497 //printf("p = %p, *p = '%c'\n",p,*p); | |
| 498 switch (*p) | |
| 499 { | |
| 500 case 0: | |
| 501 case 0x1A: | |
| 502 t->value = TOKeof; // end of file | |
| 503 return; | |
| 504 | |
| 505 case ' ': | |
| 506 case '\t': | |
| 507 case '\v': | |
| 508 case '\f': | |
| 509 p++; | |
| 510 continue; // skip white space | |
| 511 | |
| 512 case '\r': | |
| 513 p++; | |
| 514 if (*p != '\n') // if CR stands by itself | |
| 515 loc.linnum++; | |
| 516 continue; // skip white space | |
| 517 | |
| 518 case '\n': | |
| 519 p++; | |
| 520 loc.linnum++; | |
| 521 continue; // skip white space | |
| 522 | |
| 523 case '0': case '1': case '2': case '3': case '4': | |
| 524 case '5': case '6': case '7': case '8': case '9': | |
| 525 t->value = number(t); | |
| 526 return; | |
| 527 | |
| 528 #if CSTRINGS | |
| 529 case '\'': | |
| 530 t->value = charConstant(t, 0); | |
| 531 return; | |
| 532 | |
| 533 case '"': | |
| 534 t->value = stringConstant(t,0); | |
| 535 return; | |
| 536 | |
| 537 case 'l': | |
| 538 case 'L': | |
| 539 if (p[1] == '\'') | |
| 540 { | |
| 541 p++; | |
| 542 t->value = charConstant(t, 1); | |
| 543 return; | |
| 544 } | |
| 545 else if (p[1] == '"') | |
| 546 { | |
| 547 p++; | |
| 548 t->value = stringConstant(t, 1); | |
| 549 return; | |
| 550 } | |
| 551 #else | |
| 552 case '\'': | |
| 553 t->value = charConstant(t,0); | |
| 554 return; | |
| 555 | |
| 556 case 'r': | |
| 557 if (p[1] != '"') | |
| 558 goto case_ident; | |
| 559 p++; | |
| 560 case '`': | |
| 561 t->value = wysiwygStringConstant(t, *p); | |
| 562 return; | |
| 563 | |
| 564 case 'x': | |
| 565 if (p[1] != '"') | |
| 566 goto case_ident; | |
| 567 p++; | |
| 568 t->value = hexStringConstant(t); | |
| 569 return; | |
| 570 | |
| 336 | 571 #if DMDV2 |
| 159 | 572 case 'q': |
| 573 if (p[1] == '"') | |
| 574 { | |
| 575 p++; | |
| 576 t->value = delimitedStringConstant(t); | |
| 577 return; | |
| 578 } | |
| 579 else if (p[1] == '{') | |
| 580 { | |
| 581 p++; | |
| 582 t->value = tokenStringConstant(t); | |
| 583 return; | |
| 584 } | |
| 585 else | |
| 586 goto case_ident; | |
| 587 #endif | |
| 588 | |
| 589 case '"': | |
| 590 t->value = escapeStringConstant(t,0); | |
| 591 return; | |
| 592 | |
| 593 case '\\': // escaped string literal | |
| 594 { unsigned c; | |
| 595 | |
| 596 stringbuffer.reset(); | |
| 597 do | |
| 598 { | |
| 599 p++; | |
| 600 switch (*p) | |
| 601 { | |
| 602 case 'u': | |
| 603 case 'U': | |
| 604 case '&': | |
| 605 c = escapeSequence(); | |
| 606 stringbuffer.writeUTF8(c); | |
| 607 break; | |
| 608 | |
| 609 default: | |
| 610 c = escapeSequence(); | |
| 611 stringbuffer.writeByte(c); | |
| 612 break; | |
| 613 } | |
| 614 } while (*p == '\\'); | |
| 615 t->len = stringbuffer.offset; | |
| 616 stringbuffer.writeByte(0); | |
| 617 t->ustring = (unsigned char *)mem.malloc(stringbuffer.offset); | |
| 618 memcpy(t->ustring, stringbuffer.data, stringbuffer.offset); | |
| 619 t->postfix = 0; | |
| 620 t->value = TOKstring; | |
| 621 return; | |
| 622 } | |
| 623 | |
| 624 case 'l': | |
| 625 case 'L': | |
| 626 #endif | |
| 627 case 'a': case 'b': case 'c': case 'd': case 'e': | |
| 628 case 'f': case 'g': case 'h': case 'i': case 'j': | |
| 629 case 'k': case 'm': case 'n': case 'o': | |
| 336 | 630 #if DMDV2 |
| 159 | 631 case 'p': /*case 'q': case 'r':*/ case 's': case 't': |
| 632 #else | |
| 633 case 'p': case 'q': /*case 'r':*/ case 's': case 't': | |
| 634 #endif | |
| 635 case 'u': case 'v': case 'w': /*case 'x':*/ case 'y': | |
| 636 case 'z': | |
| 637 case 'A': case 'B': case 'C': case 'D': case 'E': | |
| 638 case 'F': case 'G': case 'H': case 'I': case 'J': | |
| 639 case 'K': case 'M': case 'N': case 'O': | |
| 640 case 'P': case 'Q': case 'R': case 'S': case 'T': | |
| 641 case 'U': case 'V': case 'W': case 'X': case 'Y': | |
| 642 case 'Z': | |
| 643 case '_': | |
| 644 case_ident: | |
| 645 { unsigned char c; | |
| 646 StringValue *sv; | |
| 647 Identifier *id; | |
| 648 | |
| 649 do | |
| 650 { | |
| 651 c = *++p; | |
| 652 } while (isidchar(c) || (c & 0x80 && isUniAlpha(decodeUTF()))); | |
| 653 sv = stringtable.update((char *)t->ptr, p - t->ptr); | |
| 654 id = (Identifier *) sv->ptrvalue; | |
| 655 if (!id) | |
| 656 { id = new Identifier(sv->lstring.string,TOKidentifier); | |
| 657 sv->ptrvalue = id; | |
| 658 } | |
| 659 t->ident = id; | |
| 660 t->value = (enum TOK) id->value; | |
| 661 anyToken = 1; | |
| 662 if (*t->ptr == '_') // if special identifier token | |
| 663 { | |
| 664 static char date[11+1]; | |
| 665 static char time[8+1]; | |
| 666 static char timestamp[24+1]; | |
| 667 | |
| 668 if (!date[0]) // lazy evaluation | |
| 669 { time_t t; | |
| 670 char *p; | |
| 671 | |
| 672 ::time(&t); | |
| 673 p = ctime(&t); | |
| 674 assert(p); | |
| 675 sprintf(date, "%.6s %.4s", p + 4, p + 20); | |
| 676 sprintf(time, "%.8s", p + 11); | |
| 677 sprintf(timestamp, "%.24s", p); | |
| 678 } | |
| 679 | |
| 336 | 680 #if DMDV1 |
| 159 | 681 if (mod && id == Id::FILE) |
| 682 { | |
| 683 t->ustring = (unsigned char *)(loc.filename ? loc.filename : mod->ident->toChars()); | |
| 684 goto Lstring; | |
| 685 } | |
| 686 else if (mod && id == Id::LINE) | |
| 687 { | |
| 688 t->value = TOKint64v; | |
| 689 t->uns64value = loc.linnum; | |
| 690 } | |
| 336 | 691 else |
| 692 #endif | |
| 693 if (id == Id::DATE) | |
| 159 | 694 { |
| 695 t->ustring = (unsigned char *)date; | |
| 696 goto Lstring; | |
| 697 } | |
| 698 else if (id == Id::TIME) | |
| 699 { | |
| 700 t->ustring = (unsigned char *)time; | |
| 701 goto Lstring; | |
| 702 } | |
| 703 else if (id == Id::VENDOR) | |
| 704 { | |
| 366 | 705 t->ustring = (unsigned char *)"LLVMDC"; |
| 159 | 706 goto Lstring; |
| 707 } | |
| 708 else if (id == Id::TIMESTAMP) | |
| 709 { | |
| 710 t->ustring = (unsigned char *)timestamp; | |
| 711 Lstring: | |
| 712 t->value = TOKstring; | |
| 713 Llen: | |
| 714 t->postfix = 0; | |
| 715 t->len = strlen((char *)t->ustring); | |
| 716 } | |
| 717 else if (id == Id::VERSIONX) | |
| 718 { unsigned major = 0; | |
| 719 unsigned minor = 0; | |
| 720 | |
| 721 for (char *p = global.version + 1; 1; p++) | |
| 722 { | |
| 723 char c = *p; | |
| 724 if (isdigit(c)) | |
| 725 minor = minor * 10 + c - '0'; | |
| 726 else if (c == '.') | |
| 727 { major = minor; | |
| 728 minor = 0; | |
| 729 } | |
| 730 else | |
| 731 break; | |
| 732 } | |
| 733 t->value = TOKint64v; | |
| 734 t->uns64value = major * 1000 + minor; | |
| 735 } | |
| 336 | 736 #if DMDV2 |
| 159 | 737 else if (id == Id::EOFX) |
| 738 { | |
| 739 t->value = TOKeof; | |
| 740 // Advance scanner to end of file | |
| 741 while (!(*p == 0 || *p == 0x1A)) | |
| 742 p++; | |
| 743 } | |
| 744 #endif | |
| 745 } | |
| 746 //printf("t->value = %d\n",t->value); | |
| 747 return; | |
| 748 } | |
| 749 | |
| 750 case '/': | |
| 751 p++; | |
| 752 switch (*p) | |
| 753 { | |
| 754 case '=': | |
| 755 p++; | |
| 756 t->value = TOKdivass; | |
| 757 return; | |
| 758 | |
| 759 case '*': | |
| 760 p++; | |
| 761 linnum = loc.linnum; | |
| 762 while (1) | |
| 763 { | |
| 764 while (1) | |
| 765 { unsigned char c = *p; | |
| 766 switch (c) | |
| 767 { | |
| 768 case '/': | |
| 769 break; | |
| 770 | |
| 771 case '\n': | |
| 772 loc.linnum++; | |
| 773 p++; | |
| 774 continue; | |
| 775 | |
| 776 case '\r': | |
| 777 p++; | |
| 778 if (*p != '\n') | |
| 779 loc.linnum++; | |
| 780 continue; | |
| 781 | |
| 782 case 0: | |
| 783 case 0x1A: | |
| 784 error("unterminated /* */ comment"); | |
| 785 p = end; | |
| 786 t->value = TOKeof; | |
| 787 return; | |
| 788 | |
| 789 default: | |
| 790 if (c & 0x80) | |
| 791 { unsigned u = decodeUTF(); | |
| 792 if (u == PS || u == LS) | |
| 793 loc.linnum++; | |
| 794 } | |
| 795 p++; | |
| 796 continue; | |
| 797 } | |
| 798 break; | |
| 799 } | |
| 800 p++; | |
| 801 if (p[-2] == '*' && p - 3 != t->ptr) | |
| 802 break; | |
| 803 } | |
| 804 if (commentToken) | |
| 805 { | |
| 806 t->value = TOKcomment; | |
| 807 return; | |
| 808 } | |
| 809 else if (doDocComment && t->ptr[2] == '*' && p - 4 != t->ptr) | |
| 810 { // if /** but not /**/ | |
| 811 getDocComment(t, lastLine == linnum); | |
| 812 } | |
| 813 continue; | |
| 814 | |
| 815 case '/': // do // style comments | |
| 816 linnum = loc.linnum; | |
| 817 while (1) | |
| 818 { unsigned char c = *++p; | |
| 819 switch (c) | |
| 820 { | |
| 821 case '\n': | |
| 822 break; | |
| 823 | |
| 824 case '\r': | |
| 825 if (p[1] == '\n') | |
| 826 p++; | |
| 827 break; | |
| 828 | |
| 829 case 0: | |
| 830 case 0x1A: | |
| 831 if (commentToken) | |
| 832 { | |
| 833 p = end; | |
| 834 t->value = TOKcomment; | |
| 835 return; | |
| 836 } | |
| 837 if (doDocComment && t->ptr[2] == '/') | |
| 838 getDocComment(t, lastLine == linnum); | |
| 839 p = end; | |
| 840 t->value = TOKeof; | |
| 841 return; | |
| 842 | |
| 843 default: | |
| 844 if (c & 0x80) | |
| 845 { unsigned u = decodeUTF(); | |
| 846 if (u == PS || u == LS) | |
| 847 break; | |
| 848 } | |
| 849 continue; | |
| 850 } | |
| 851 break; | |
| 852 } | |
| 853 | |
| 854 if (commentToken) | |
| 855 { | |
| 856 p++; | |
| 857 loc.linnum++; | |
| 858 t->value = TOKcomment; | |
| 859 return; | |
| 860 } | |
| 861 if (doDocComment && t->ptr[2] == '/') | |
| 862 getDocComment(t, lastLine == linnum); | |
| 863 | |
| 864 p++; | |
| 865 loc.linnum++; | |
| 866 continue; | |
| 867 | |
| 868 case '+': | |
| 869 { int nest; | |
| 870 | |
| 871 linnum = loc.linnum; | |
| 872 p++; | |
| 873 nest = 1; | |
| 874 while (1) | |
| 875 { unsigned char c = *p; | |
| 876 switch (c) | |
| 877 { | |
| 878 case '/': | |
| 879 p++; | |
| 880 if (*p == '+') | |
| 881 { | |
| 882 p++; | |
| 883 nest++; | |
| 884 } | |
| 885 continue; | |
| 886 | |
| 887 case '+': | |
| 888 p++; | |
| 889 if (*p == '/') | |
| 890 { | |
| 891 p++; | |
| 892 if (--nest == 0) | |
| 893 break; | |
| 894 } | |
| 895 continue; | |
| 896 | |
| 897 case '\r': | |
| 898 p++; | |
| 899 if (*p != '\n') | |
| 900 loc.linnum++; | |
| 901 continue; | |
| 902 | |
| 903 case '\n': | |
| 904 loc.linnum++; | |
| 905 p++; | |
| 906 continue; | |
| 907 | |
| 908 case 0: | |
| 909 case 0x1A: | |
| 910 error("unterminated /+ +/ comment"); | |
| 911 p = end; | |
| 912 t->value = TOKeof; | |
| 913 return; | |
| 914 | |
| 915 default: | |
| 916 if (c & 0x80) | |
| 917 { unsigned u = decodeUTF(); | |
| 918 if (u == PS || u == LS) | |
| 919 loc.linnum++; | |
| 920 } | |
| 921 p++; | |
| 922 continue; | |
| 923 } | |
| 924 break; | |
| 925 } | |
| 926 if (commentToken) | |
| 927 { | |
| 928 t->value = TOKcomment; | |
| 929 return; | |
| 930 } | |
| 931 if (doDocComment && t->ptr[2] == '+' && p - 4 != t->ptr) | |
| 932 { // if /++ but not /++/ | |
| 933 getDocComment(t, lastLine == linnum); | |
| 934 } | |
| 935 continue; | |
| 936 } | |
| 937 } | |
| 938 t->value = TOKdiv; | |
| 939 return; | |
| 940 | |
| 941 case '.': | |
| 942 p++; | |
| 943 if (isdigit(*p)) | |
| 944 { /* Note that we don't allow ._1 and ._ as being | |
| 945 * valid floating point numbers. | |
| 946 */ | |
| 947 p--; | |
| 948 t->value = inreal(t); | |
| 949 } | |
| 950 else if (p[0] == '.') | |
| 951 { | |
| 952 if (p[1] == '.') | |
| 953 { p += 2; | |
| 954 t->value = TOKdotdotdot; | |
| 955 } | |
| 956 else | |
| 957 { p++; | |
| 958 t->value = TOKslice; | |
| 959 } | |
| 960 } | |
| 961 else | |
| 962 t->value = TOKdot; | |
| 963 return; | |
| 964 | |
| 965 case '&': | |
| 966 p++; | |
| 967 if (*p == '=') | |
| 968 { p++; | |
| 969 t->value = TOKandass; | |
| 970 } | |
| 971 else if (*p == '&') | |
| 972 { p++; | |
| 973 t->value = TOKandand; | |
| 974 } | |
| 975 else | |
| 976 t->value = TOKand; | |
| 977 return; | |
| 978 | |
| 979 case '|': | |
| 980 p++; | |
| 981 if (*p == '=') | |
| 982 { p++; | |
| 983 t->value = TOKorass; | |
| 984 } | |
| 985 else if (*p == '|') | |
| 986 { p++; | |
| 987 t->value = TOKoror; | |
| 988 } | |
| 989 else | |
| 990 t->value = TOKor; | |
| 991 return; | |
| 992 | |
| 993 case '-': | |
| 994 p++; | |
| 995 if (*p == '=') | |
| 996 { p++; | |
| 997 t->value = TOKminass; | |
| 998 } | |
| 999 #if 0 | |
| 1000 else if (*p == '>') | |
| 1001 { p++; | |
| 1002 t->value = TOKarrow; | |
| 1003 } | |
| 1004 #endif | |
| 1005 else if (*p == '-') | |
| 1006 { p++; | |
| 1007 t->value = TOKminusminus; | |
| 1008 } | |
| 1009 else | |
| 1010 t->value = TOKmin; | |
| 1011 return; | |
| 1012 | |
| 1013 case '+': | |
| 1014 p++; | |
| 1015 if (*p == '=') | |
| 1016 { p++; | |
| 1017 t->value = TOKaddass; | |
| 1018 } | |
| 1019 else if (*p == '+') | |
| 1020 { p++; | |
| 1021 t->value = TOKplusplus; | |
| 1022 } | |
| 1023 else | |
| 1024 t->value = TOKadd; | |
| 1025 return; | |
| 1026 | |
| 1027 case '<': | |
| 1028 p++; | |
| 1029 if (*p == '=') | |
| 1030 { p++; | |
| 1031 t->value = TOKle; // <= | |
| 1032 } | |
| 1033 else if (*p == '<') | |
| 1034 { p++; | |
| 1035 if (*p == '=') | |
| 1036 { p++; | |
| 1037 t->value = TOKshlass; // <<= | |
| 1038 } | |
| 1039 else | |
| 1040 t->value = TOKshl; // << | |
| 1041 } | |
| 1042 else if (*p == '>') | |
| 1043 { p++; | |
| 1044 if (*p == '=') | |
| 1045 { p++; | |
| 1046 t->value = TOKleg; // <>= | |
| 1047 } | |
| 1048 else | |
| 1049 t->value = TOKlg; // <> | |
| 1050 } | |
| 1051 else | |
| 1052 t->value = TOKlt; // < | |
| 1053 return; | |
| 1054 | |
| 1055 case '>': | |
| 1056 p++; | |
| 1057 if (*p == '=') | |
| 1058 { p++; | |
| 1059 t->value = TOKge; // >= | |
| 1060 } | |
| 1061 else if (*p == '>') | |
| 1062 { p++; | |
| 1063 if (*p == '=') | |
| 1064 { p++; | |
| 1065 t->value = TOKshrass; // >>= | |
| 1066 } | |
| 1067 else if (*p == '>') | |
| 1068 { p++; | |
| 1069 if (*p == '=') | |
| 1070 { p++; | |
| 1071 t->value = TOKushrass; // >>>= | |
| 1072 } | |
| 1073 else | |
| 1074 t->value = TOKushr; // >>> | |
| 1075 } | |
| 1076 else | |
| 1077 t->value = TOKshr; // >> | |
| 1078 } | |
| 1079 else | |
| 1080 t->value = TOKgt; // > | |
| 1081 return; | |
| 1082 | |
| 1083 case '!': | |
| 1084 p++; | |
| 1085 if (*p == '=') | |
| 1086 { p++; | |
| 1087 if (*p == '=' && global.params.Dversion == 1) | |
| 1088 { p++; | |
| 1089 t->value = TOKnotidentity; // !== | |
| 1090 } | |
| 1091 else | |
| 1092 t->value = TOKnotequal; // != | |
| 1093 } | |
| 1094 else if (*p == '<') | |
| 1095 { p++; | |
| 1096 if (*p == '>') | |
| 1097 { p++; | |
| 1098 if (*p == '=') | |
| 1099 { p++; | |
| 1100 t->value = TOKunord; // !<>= | |
| 1101 } | |
| 1102 else | |
| 1103 t->value = TOKue; // !<> | |
| 1104 } | |
| 1105 else if (*p == '=') | |
| 1106 { p++; | |
| 1107 t->value = TOKug; // !<= | |
| 1108 } | |
| 1109 else | |
| 1110 t->value = TOKuge; // !< | |
| 1111 } | |
| 1112 else if (*p == '>') | |
| 1113 { p++; | |
| 1114 if (*p == '=') | |
| 1115 { p++; | |
| 1116 t->value = TOKul; // !>= | |
| 1117 } | |
| 1118 else | |
| 1119 t->value = TOKule; // !> | |
| 1120 } | |
| 1121 else | |
| 1122 t->value = TOKnot; // ! | |
| 1123 return; | |
| 1124 | |
| 1125 case '=': | |
| 1126 p++; | |
| 1127 if (*p == '=') | |
| 1128 { p++; | |
| 1129 if (*p == '=' && global.params.Dversion == 1) | |
| 1130 { p++; | |
| 1131 t->value = TOKidentity; // === | |
| 1132 } | |
| 1133 else | |
| 1134 t->value = TOKequal; // == | |
| 1135 } | |
| 1136 else | |
| 1137 t->value = TOKassign; // = | |
| 1138 return; | |
| 1139 | |
| 1140 case '~': | |
| 1141 p++; | |
| 1142 if (*p == '=') | |
| 1143 { p++; | |
| 1144 t->value = TOKcatass; // ~= | |
| 1145 } | |
| 1146 else | |
| 1147 t->value = TOKtilde; // ~ | |
| 1148 return; | |
| 1149 | |
| 1150 #define SINGLE(c,tok) case c: p++; t->value = tok; return; | |
| 1151 | |
| 1152 SINGLE('(', TOKlparen) | |
| 1153 SINGLE(')', TOKrparen) | |
| 1154 SINGLE('[', TOKlbracket) | |
| 1155 SINGLE(']', TOKrbracket) | |
| 1156 SINGLE('{', TOKlcurly) | |
| 1157 SINGLE('}', TOKrcurly) | |
| 1158 SINGLE('?', TOKquestion) | |
| 1159 SINGLE(',', TOKcomma) | |
| 1160 SINGLE(';', TOKsemicolon) | |
| 1161 SINGLE(':', TOKcolon) | |
| 1162 SINGLE('$', TOKdollar) | |
| 1163 | |
| 1164 #undef SINGLE | |
| 1165 | |
| 1166 #define DOUBLE(c1,tok1,c2,tok2) \ | |
| 1167 case c1: \ | |
| 1168 p++; \ | |
| 1169 if (*p == c2) \ | |
| 1170 { p++; \ | |
| 1171 t->value = tok2; \ | |
| 1172 } \ | |
| 1173 else \ | |
| 1174 t->value = tok1; \ | |
| 1175 return; | |
| 1176 | |
| 1177 DOUBLE('*', TOKmul, '=', TOKmulass) | |
| 1178 DOUBLE('%', TOKmod, '=', TOKmodass) | |
| 1179 DOUBLE('^', TOKxor, '=', TOKxorass) | |
| 1180 | |
| 1181 #undef DOUBLE | |
| 1182 | |
| 1183 case '#': | |
| 1184 p++; | |
| 1185 pragma(); | |
| 1186 continue; | |
| 1187 | |
| 1188 default: | |
| 1189 { unsigned char c = *p; | |
| 1190 | |
| 1191 if (c & 0x80) | |
| 1192 { unsigned u = decodeUTF(); | |
| 1193 | |
| 1194 // Check for start of unicode identifier | |
| 1195 if (isUniAlpha(u)) | |
| 1196 goto case_ident; | |
| 1197 | |
| 1198 if (u == PS || u == LS) | |
| 1199 { | |
| 1200 loc.linnum++; | |
| 1201 p++; | |
| 1202 continue; | |
| 1203 } | |
| 1204 } | |
| 1205 if (isprint(c)) | |
| 1206 error("unsupported char '%c'", c); | |
| 1207 else | |
| 1208 error("unsupported char 0x%02x", c); | |
| 1209 p++; | |
| 1210 continue; | |
| 1211 } | |
| 1212 } | |
| 1213 } | |
| 1214 } | |
| 1215 | |
| 1216 /******************************************* | |
| 1217 * Parse escape sequence. | |
| 1218 */ | |
| 1219 | |
| 1220 unsigned Lexer::escapeSequence() | |
| 1221 { unsigned c; | |
| 1222 int n; | |
| 1223 int ndigits; | |
| 1224 | |
| 1225 c = *p; | |
| 1226 switch (c) | |
| 1227 { | |
| 1228 case '\'': | |
| 1229 case '"': | |
| 1230 case '?': | |
| 1231 case '\\': | |
| 1232 Lconsume: | |
| 1233 p++; | |
| 1234 break; | |
| 1235 | |
| 1236 case 'a': c = 7; goto Lconsume; | |
| 1237 case 'b': c = 8; goto Lconsume; | |
| 1238 case 'f': c = 12; goto Lconsume; | |
| 1239 case 'n': c = 10; goto Lconsume; | |
| 1240 case 'r': c = 13; goto Lconsume; | |
| 1241 case 't': c = 9; goto Lconsume; | |
| 1242 case 'v': c = 11; goto Lconsume; | |
| 1243 | |
| 1244 case 'u': | |
| 1245 ndigits = 4; | |
| 1246 goto Lhex; | |
| 1247 case 'U': | |
| 1248 ndigits = 8; | |
| 1249 goto Lhex; | |
| 1250 case 'x': | |
| 1251 ndigits = 2; | |
| 1252 Lhex: | |
| 1253 p++; | |
| 1254 c = *p; | |
| 1255 if (ishex(c)) | |
| 1256 { unsigned v; | |
| 1257 | |
| 1258 n = 0; | |
| 1259 v = 0; | |
| 1260 while (1) | |
| 1261 { | |
| 1262 if (isdigit(c)) | |
| 1263 c -= '0'; | |
| 1264 else if (islower(c)) | |
| 1265 c -= 'a' - 10; | |
| 1266 else | |
| 1267 c -= 'A' - 10; | |
| 1268 v = v * 16 + c; | |
| 1269 c = *++p; | |
| 1270 if (++n == ndigits) | |
| 1271 break; | |
| 1272 if (!ishex(c)) | |
| 1273 { error("escape hex sequence has %d hex digits instead of %d", n, ndigits); | |
| 1274 break; | |
| 1275 } | |
| 1276 } | |
| 1277 if (ndigits != 2 && !utf_isValidDchar(v)) | |
| 1278 error("invalid UTF character \\U%08x", v); | |
| 1279 c = v; | |
| 1280 } | |
| 1281 else | |
| 1282 error("undefined escape hex sequence \\%c\n",c); | |
| 1283 break; | |
| 1284 | |
| 1285 case '&': // named character entity | |
| 1286 for (unsigned char *idstart = ++p; 1; p++) | |
| 1287 { | |
| 1288 switch (*p) | |
| 1289 { | |
| 1290 case ';': | |
| 1291 c = HtmlNamedEntity(idstart, p - idstart); | |
| 1292 if (c == ~0) | |
| 1293 { error("unnamed character entity &%.*s;", (int)(p - idstart), idstart); | |
| 1294 c = ' '; | |
| 1295 } | |
| 1296 p++; | |
| 1297 break; | |
| 1298 | |
| 1299 default: | |
| 1300 if (isalpha(*p) || | |
| 1301 (p != idstart + 1 && isdigit(*p))) | |
| 1302 continue; | |
| 1303 error("unterminated named entity"); | |
| 1304 break; | |
| 1305 } | |
| 1306 break; | |
| 1307 } | |
| 1308 break; | |
| 1309 | |
| 1310 case 0: | |
| 1311 case 0x1A: // end of file | |
| 1312 c = '\\'; | |
| 1313 break; | |
| 1314 | |
| 1315 default: | |
| 1316 if (isoctal(c)) | |
| 1317 { unsigned v; | |
| 1318 | |
| 1319 n = 0; | |
| 1320 v = 0; | |
| 1321 do | |
| 1322 { | |
| 1323 v = v * 8 + (c - '0'); | |
| 1324 c = *++p; | |
| 1325 } while (++n < 3 && isoctal(c)); | |
| 1326 c = v; | |
| 1327 if (c > 0xFF) | |
| 1328 error("0%03o is larger than a byte", c); | |
| 1329 } | |
| 1330 else | |
| 1331 error("undefined escape sequence \\%c\n",c); | |
| 1332 break; | |
| 1333 } | |
| 1334 return c; | |
| 1335 } | |
| 1336 | |
| 1337 /************************************** | |
| 1338 */ | |
| 1339 | |
| 1340 TOK Lexer::wysiwygStringConstant(Token *t, int tc) | |
| 1341 { unsigned c; | |
| 1342 Loc start = loc; | |
| 1343 | |
| 1344 p++; | |
| 1345 stringbuffer.reset(); | |
| 1346 while (1) | |
| 1347 { | |
| 1348 c = *p++; | |
| 1349 switch (c) | |
| 1350 { | |
| 1351 case '\n': | |
| 1352 loc.linnum++; | |
| 1353 break; | |
| 1354 | |
| 1355 case '\r': | |
| 1356 if (*p == '\n') | |
| 1357 continue; // ignore | |
| 1358 c = '\n'; // treat EndOfLine as \n character | |
| 1359 loc.linnum++; | |
| 1360 break; | |
| 1361 | |
| 1362 case 0: | |
| 1363 case 0x1A: | |
| 1364 error("unterminated string constant starting at %s", start.toChars()); | |
| 1365 t->ustring = (unsigned char *)""; | |
| 1366 t->len = 0; | |
| 1367 t->postfix = 0; | |
| 1368 return TOKstring; | |
| 1369 | |
| 1370 case '"': | |
| 1371 case '`': | |
| 1372 if (c == tc) | |
| 1373 { | |
| 1374 t->len = stringbuffer.offset; | |
| 1375 stringbuffer.writeByte(0); | |
| 1376 t->ustring = (unsigned char *)mem.malloc(stringbuffer.offset); | |
| 1377 memcpy(t->ustring, stringbuffer.data, stringbuffer.offset); | |
| 1378 stringPostfix(t); | |
| 1379 return TOKstring; | |
| 1380 } | |
| 1381 break; | |
| 1382 | |
| 1383 default: | |
| 1384 if (c & 0x80) | |
| 1385 { p--; | |
| 1386 unsigned u = decodeUTF(); | |
| 1387 p++; | |
| 1388 if (u == PS || u == LS) | |
| 1389 loc.linnum++; | |
| 1390 stringbuffer.writeUTF8(u); | |
| 1391 continue; | |
| 1392 } | |
| 1393 break; | |
| 1394 } | |
| 1395 stringbuffer.writeByte(c); | |
| 1396 } | |
| 1397 } | |
| 1398 | |
| 1399 /************************************** | |
| 1400 * Lex hex strings: | |
| 1401 * x"0A ae 34FE BD" | |
| 1402 */ | |
| 1403 | |
| 1404 TOK Lexer::hexStringConstant(Token *t) | |
| 1405 { unsigned c; | |
| 1406 Loc start = loc; | |
| 1407 unsigned n = 0; | |
| 1408 unsigned v; | |
| 1409 | |
| 1410 p++; | |
| 1411 stringbuffer.reset(); | |
| 1412 while (1) | |
| 1413 { | |
| 1414 c = *p++; | |
| 1415 switch (c) | |
| 1416 { | |
| 1417 case ' ': | |
| 1418 case '\t': | |
| 1419 case '\v': | |
| 1420 case '\f': | |
| 1421 continue; // skip white space | |
| 1422 | |
| 1423 case '\r': | |
| 1424 if (*p == '\n') | |
| 1425 continue; // ignore | |
| 1426 // Treat isolated '\r' as if it were a '\n' | |
| 1427 case '\n': | |
| 1428 loc.linnum++; | |
| 1429 continue; | |
| 1430 | |
| 1431 case 0: | |
| 1432 case 0x1A: | |
| 1433 error("unterminated string constant starting at %s", start.toChars()); | |
| 1434 t->ustring = (unsigned char *)""; | |
| 1435 t->len = 0; | |
| 1436 t->postfix = 0; | |
| 1437 return TOKstring; | |
| 1438 | |
| 1439 case '"': | |
| 1440 if (n & 1) | |
| 1441 { error("odd number (%d) of hex characters in hex string", n); | |
| 1442 stringbuffer.writeByte(v); | |
| 1443 } | |
| 1444 t->len = stringbuffer.offset; | |
| 1445 stringbuffer.writeByte(0); | |
| 1446 t->ustring = (unsigned char *)mem.malloc(stringbuffer.offset); | |
| 1447 memcpy(t->ustring, stringbuffer.data, stringbuffer.offset); | |
| 1448 stringPostfix(t); | |
| 1449 return TOKstring; | |
| 1450 | |
| 1451 default: | |
| 1452 if (c >= '0' && c <= '9') | |
| 1453 c -= '0'; | |
| 1454 else if (c >= 'a' && c <= 'f') | |
| 1455 c -= 'a' - 10; | |
| 1456 else if (c >= 'A' && c <= 'F') | |
| 1457 c -= 'A' - 10; | |
| 1458 else if (c & 0x80) | |
| 1459 { p--; | |
| 1460 unsigned u = decodeUTF(); | |
| 1461 p++; | |
| 1462 if (u == PS || u == LS) | |
| 1463 loc.linnum++; | |
| 1464 else | |
| 1465 error("non-hex character \\u%x", u); | |
| 1466 } | |
| 1467 else | |
| 1468 error("non-hex character '%c'", c); | |
| 1469 if (n & 1) | |
| 1470 { v = (v << 4) | c; | |
| 1471 stringbuffer.writeByte(v); | |
| 1472 } | |
| 1473 else | |
| 1474 v = c; | |
| 1475 n++; | |
| 1476 break; | |
| 1477 } | |
| 1478 } | |
| 1479 } | |
| 1480 | |
| 1481 | |
| 336 | 1482 #if DMDV2 |
| 159 | 1483 /************************************** |
| 1484 * Lex delimited strings: | |
| 1485 * q"(foo(xxx))" // "foo(xxx)" | |
| 1486 * q"[foo(]" // "foo(" | |
| 1487 * q"/foo]/" // "foo]" | |
| 1488 * q"HERE | |
| 1489 * foo | |
| 1490 * HERE" // "foo\n" | |
| 1491 * Input: | |
| 1492 * p is on the " | |
| 1493 */ | |
| 1494 | |
| 1495 TOK Lexer::delimitedStringConstant(Token *t) | |
| 1496 { unsigned c; | |
| 1497 Loc start = loc; | |
| 1498 unsigned delimleft = 0; | |
| 1499 unsigned delimright = 0; | |
| 1500 unsigned nest = 1; | |
| 1501 unsigned nestcount; | |
| 1502 Identifier *hereid = NULL; | |
| 1503 unsigned blankrol = 0; | |
| 1504 unsigned startline = 0; | |
| 1505 | |
| 1506 p++; | |
| 1507 stringbuffer.reset(); | |
| 1508 while (1) | |
| 1509 { | |
| 1510 c = *p++; | |
| 1511 //printf("c = '%c'\n", c); | |
| 1512 switch (c) | |
| 1513 { | |
| 1514 case '\n': | |
| 1515 Lnextline: | |
| 1516 loc.linnum++; | |
| 1517 startline = 1; | |
| 1518 if (blankrol) | |
| 1519 { blankrol = 0; | |
| 1520 continue; | |
| 1521 } | |
| 1522 if (hereid) | |
| 1523 { | |
| 1524 stringbuffer.writeUTF8(c); | |
| 1525 continue; | |
| 1526 } | |
| 1527 break; | |
| 1528 | |
| 1529 case '\r': | |
| 1530 if (*p == '\n') | |
| 1531 continue; // ignore | |
| 1532 c = '\n'; // treat EndOfLine as \n character | |
| 1533 goto Lnextline; | |
| 1534 | |
| 1535 case 0: | |
| 1536 case 0x1A: | |
| 1537 goto Lerror; | |
| 1538 | |
| 1539 default: | |
| 1540 if (c & 0x80) | |
| 1541 { p--; | |
| 1542 c = decodeUTF(); | |
| 1543 p++; | |
| 1544 if (c == PS || c == LS) | |
| 1545 goto Lnextline; | |
| 1546 } | |
| 1547 break; | |
| 1548 } | |
| 1549 if (delimleft == 0) | |
| 1550 { delimleft = c; | |
| 1551 nest = 1; | |
| 1552 nestcount = 1; | |
| 1553 if (c == '(') | |
| 1554 delimright = ')'; | |
| 1555 else if (c == '{') | |
| 1556 delimright = '}'; | |
| 1557 else if (c == '[') | |
| 1558 delimright = ']'; | |
| 1559 else if (c == '<') | |
| 1560 delimright = '>'; | |
| 1561 else if (isalpha(c) || c == '_' || (c >= 0x80 && isUniAlpha(c))) | |
| 1562 { // Start of identifier; must be a heredoc | |
| 1563 Token t; | |
| 1564 p--; | |
| 1565 scan(&t); // read in heredoc identifier | |
| 1566 if (t.value != TOKidentifier) | |
| 1567 { error("identifier expected for heredoc, not %s", t.toChars()); | |
| 1568 delimright = c; | |
| 1569 } | |
| 1570 else | |
| 1571 { hereid = t.ident; | |
| 1572 //printf("hereid = '%s'\n", hereid->toChars()); | |
| 1573 blankrol = 1; | |
| 1574 } | |
| 1575 nest = 0; | |
| 1576 } | |
| 1577 else | |
| 1578 { delimright = c; | |
| 1579 nest = 0; | |
| 1580 } | |
| 1581 } | |
| 1582 else | |
| 1583 { | |
| 1584 if (blankrol) | |
| 1585 { error("heredoc rest of line should be blank"); | |
| 1586 blankrol = 0; | |
| 1587 continue; | |
| 1588 } | |
| 1589 if (nest == 1) | |
| 1590 { | |
| 1591 if (c == delimleft) | |
| 1592 nestcount++; | |
| 1593 else if (c == delimright) | |
| 1594 { nestcount--; | |
| 1595 if (nestcount == 0) | |
| 1596 goto Ldone; | |
| 1597 } | |
| 1598 } | |
| 1599 else if (c == delimright) | |
| 1600 goto Ldone; | |
| 1601 if (startline && isalpha(c)) | |
| 1602 { Token t; | |
| 1603 unsigned char *psave = p; | |
| 1604 p--; | |
| 1605 scan(&t); // read in possible heredoc identifier | |
| 1606 //printf("endid = '%s'\n", t.ident->toChars()); | |
| 1607 if (t.value == TOKidentifier && t.ident->equals(hereid)) | |
| 1608 { /* should check that rest of line is blank | |
| 1609 */ | |
| 1610 goto Ldone; | |
| 1611 } | |
| 1612 p = psave; | |
| 1613 } | |
| 1614 stringbuffer.writeUTF8(c); | |
| 1615 startline = 0; | |
| 1616 } | |
| 1617 } | |
| 1618 | |
| 1619 Ldone: | |
| 1620 if (*p == '"') | |
| 1621 p++; | |
| 1622 else | |
| 1623 error("delimited string must end in %c\"", delimright); | |
| 1624 t->len = stringbuffer.offset; | |
| 1625 stringbuffer.writeByte(0); | |
| 1626 t->ustring = (unsigned char *)mem.malloc(stringbuffer.offset); | |
| 1627 memcpy(t->ustring, stringbuffer.data, stringbuffer.offset); | |
| 1628 stringPostfix(t); | |
| 1629 return TOKstring; | |
| 1630 | |
| 1631 Lerror: | |
| 1632 error("unterminated string constant starting at %s", start.toChars()); | |
| 1633 t->ustring = (unsigned char *)""; | |
| 1634 t->len = 0; | |
| 1635 t->postfix = 0; | |
| 1636 return TOKstring; | |
| 1637 } | |
| 1638 | |
| 1639 /************************************** | |
| 1640 * Lex delimited strings: | |
| 1641 * q{ foo(xxx) } // " foo(xxx) " | |
| 1642 * q{foo(} // "foo(" | |
| 1643 * q{{foo}"}"} // "{foo}"}"" | |
| 1644 * Input: | |
| 1645 * p is on the q | |
| 1646 */ | |
| 1647 | |
| 1648 TOK Lexer::tokenStringConstant(Token *t) | |
| 1649 { | |
| 1650 unsigned nest = 1; | |
| 1651 Loc start = loc; | |
| 1652 unsigned char *pstart = ++p; | |
| 1653 | |
| 1654 while (1) | |
| 1655 { Token tok; | |
| 1656 | |
| 1657 scan(&tok); | |
| 1658 switch (tok.value) | |
| 1659 { | |
| 1660 case TOKlcurly: | |
| 1661 nest++; | |
| 1662 continue; | |
| 1663 | |
| 1664 case TOKrcurly: | |
| 1665 if (--nest == 0) | |
| 1666 goto Ldone; | |
| 1667 continue; | |
| 1668 | |
| 1669 case TOKeof: | |
| 1670 goto Lerror; | |
| 1671 | |
| 1672 default: | |
| 1673 continue; | |
| 1674 } | |
| 1675 } | |
| 1676 | |
| 1677 Ldone: | |
| 1678 t->len = p - 1 - pstart; | |
| 1679 t->ustring = (unsigned char *)mem.malloc(t->len + 1); | |
| 1680 memcpy(t->ustring, pstart, t->len); | |
| 1681 t->ustring[t->len] = 0; | |
| 1682 stringPostfix(t); | |
| 1683 return TOKstring; | |
| 1684 | |
| 1685 Lerror: | |
| 1686 error("unterminated token string constant starting at %s", start.toChars()); | |
| 1687 t->ustring = (unsigned char *)""; | |
| 1688 t->len = 0; | |
| 1689 t->postfix = 0; | |
| 1690 return TOKstring; | |
| 1691 } | |
| 1692 | |
| 1693 #endif | |
| 1694 | |
| 1695 | |
| 1696 /************************************** | |
| 1697 */ | |
| 1698 | |
| 1699 TOK Lexer::escapeStringConstant(Token *t, int wide) | |
| 1700 { unsigned c; | |
| 1701 Loc start = loc; | |
| 1702 | |
| 1703 p++; | |
| 1704 stringbuffer.reset(); | |
| 1705 while (1) | |
| 1706 { | |
| 1707 c = *p++; | |
| 1708 switch (c) | |
| 1709 { | |
| 1710 case '\\': | |
| 1711 switch (*p) | |
| 1712 { | |
| 1713 case 'u': | |
| 1714 case 'U': | |
| 1715 case '&': | |
| 1716 c = escapeSequence(); | |
| 1717 stringbuffer.writeUTF8(c); | |
| 1718 continue; | |
| 1719 | |
| 1720 default: | |
| 1721 c = escapeSequence(); | |
| 1722 break; | |
| 1723 } | |
| 1724 break; | |
| 1725 | |
| 1726 case '\n': | |
| 1727 loc.linnum++; | |
| 1728 break; | |
| 1729 | |
| 1730 case '\r': | |
| 1731 if (*p == '\n') | |
| 1732 continue; // ignore | |
| 1733 c = '\n'; // treat EndOfLine as \n character | |
| 1734 loc.linnum++; | |
| 1735 break; | |
| 1736 | |
| 1737 case '"': | |
| 1738 t->len = stringbuffer.offset; | |
| 1739 stringbuffer.writeByte(0); | |
| 1740 t->ustring = (unsigned char *)mem.malloc(stringbuffer.offset); | |
| 1741 memcpy(t->ustring, stringbuffer.data, stringbuffer.offset); | |
| 1742 stringPostfix(t); | |
| 1743 return TOKstring; | |
| 1744 | |
| 1745 case 0: | |
| 1746 case 0x1A: | |
| 1747 p--; | |
| 1748 error("unterminated string constant starting at %s", start.toChars()); | |
| 1749 t->ustring = (unsigned char *)""; | |
| 1750 t->len = 0; | |
| 1751 t->postfix = 0; | |
| 1752 return TOKstring; | |
| 1753 | |
| 1754 default: | |
| 1755 if (c & 0x80) | |
| 1756 { | |
| 1757 p--; | |
| 1758 c = decodeUTF(); | |
| 1759 if (c == LS || c == PS) | |
| 1760 { c = '\n'; | |
| 1761 loc.linnum++; | |
| 1762 } | |
| 1763 p++; | |
| 1764 stringbuffer.writeUTF8(c); | |
| 1765 continue; | |
| 1766 } | |
| 1767 break; | |
| 1768 } | |
| 1769 stringbuffer.writeByte(c); | |
| 1770 } | |
| 1771 } | |
| 1772 | |
| 1773 /************************************** | |
| 1774 */ | |
| 1775 | |
| 1776 TOK Lexer::charConstant(Token *t, int wide) | |
| 1777 { | |
| 1778 unsigned c; | |
| 1779 TOK tk = TOKcharv; | |
| 1780 | |
| 1781 //printf("Lexer::charConstant\n"); | |
| 1782 p++; | |
| 1783 c = *p++; | |
| 1784 switch (c) | |
| 1785 { | |
| 1786 case '\\': | |
| 1787 switch (*p) | |
| 1788 { | |
| 1789 case 'u': | |
| 1790 t->uns64value = escapeSequence(); | |
| 1791 tk = TOKwcharv; | |
| 1792 break; | |
| 1793 | |
| 1794 case 'U': | |
| 1795 case '&': | |
| 1796 t->uns64value = escapeSequence(); | |
| 1797 tk = TOKdcharv; | |
| 1798 break; | |
| 1799 | |
| 1800 default: | |
| 1801 t->uns64value = escapeSequence(); | |
| 1802 break; | |
| 1803 } | |
| 1804 break; | |
| 1805 | |
| 1806 case '\n': | |
| 1807 L1: | |
| 1808 loc.linnum++; | |
| 1809 case '\r': | |
| 1810 case 0: | |
| 1811 case 0x1A: | |
| 1812 case '\'': | |
| 1813 error("unterminated character constant"); | |
| 1814 return tk; | |
| 1815 | |
| 1816 default: | |
| 1817 if (c & 0x80) | |
| 1818 { | |
| 1819 p--; | |
| 1820 c = decodeUTF(); | |
| 1821 p++; | |
| 1822 if (c == LS || c == PS) | |
| 1823 goto L1; | |
| 1824 if (c < 0xD800 || (c >= 0xE000 && c < 0xFFFE)) | |
| 1825 tk = TOKwcharv; | |
| 1826 else | |
| 1827 tk = TOKdcharv; | |
| 1828 } | |
| 1829 t->uns64value = c; | |
| 1830 break; | |
| 1831 } | |
| 1832 | |
| 1833 if (*p != '\'') | |
| 1834 { error("unterminated character constant"); | |
| 1835 return tk; | |
| 1836 } | |
| 1837 p++; | |
| 1838 return tk; | |
| 1839 } | |
| 1840 | |
| 1841 /*************************************** | |
| 1842 * Get postfix of string literal. | |
| 1843 */ | |
| 1844 | |
| 1845 void Lexer::stringPostfix(Token *t) | |
| 1846 { | |
| 1847 switch (*p) | |
| 1848 { | |
| 1849 case 'c': | |
| 1850 case 'w': | |
| 1851 case 'd': | |
| 1852 t->postfix = *p; | |
| 1853 p++; | |
| 1854 break; | |
| 1855 | |
| 1856 default: | |
| 1857 t->postfix = 0; | |
| 1858 break; | |
| 1859 } | |
| 1860 } | |
| 1861 | |
| 1862 /*************************************** | |
| 1863 * Read \u or \U unicode sequence | |
| 1864 * Input: | |
| 1865 * u 'u' or 'U' | |
| 1866 */ | |
| 1867 | |
| 1868 #if 0 | |
| 1869 unsigned Lexer::wchar(unsigned u) | |
| 1870 { | |
| 1871 unsigned value; | |
| 1872 unsigned n; | |
| 1873 unsigned char c; | |
| 1874 unsigned nchars; | |
| 1875 | |
| 1876 nchars = (u == 'U') ? 8 : 4; | |
| 1877 value = 0; | |
| 1878 for (n = 0; 1; n++) | |
| 1879 { | |
| 1880 ++p; | |
| 1881 if (n == nchars) | |
| 1882 break; | |
| 1883 c = *p; | |
| 1884 if (!ishex(c)) | |
| 1885 { error("\\%c sequence must be followed by %d hex characters", u, nchars); | |
| 1886 break; | |
| 1887 } | |
| 1888 if (isdigit(c)) | |
| 1889 c -= '0'; | |
| 1890 else if (islower(c)) | |
| 1891 c -= 'a' - 10; | |
| 1892 else | |
| 1893 c -= 'A' - 10; | |
| 1894 value <<= 4; | |
| 1895 value |= c; | |
| 1896 } | |
| 1897 return value; | |
| 1898 } | |
| 1899 #endif | |
| 1900 | |
| 1901 /************************************** | |
| 1902 * Read in a number. | |
| 1903 * If it's an integer, store it in tok.TKutok.Vlong. | |
| 1904 * integers can be decimal, octal or hex | |
| 1905 * Handle the suffixes U, UL, LU, L, etc. | |
| 1906 * If it's double, store it in tok.TKutok.Vdouble. | |
| 1907 * Returns: | |
| 1908 * TKnum | |
| 1909 * TKdouble,... | |
| 1910 */ | |
| 1911 | |
| 1912 TOK Lexer::number(Token *t) | |
| 1913 { | |
| 1914 // We use a state machine to collect numbers | |
| 1915 enum STATE { STATE_initial, STATE_0, STATE_decimal, STATE_octal, STATE_octale, | |
| 1916 STATE_hex, STATE_binary, STATE_hex0, STATE_binary0, | |
| 1917 STATE_hexh, STATE_error }; | |
| 1918 enum STATE state; | |
| 1919 | |
| 1920 enum FLAGS | |
| 1921 { FLAGS_decimal = 1, // decimal | |
| 1922 FLAGS_unsigned = 2, // u or U suffix | |
| 1923 FLAGS_long = 4, // l or L suffix | |
| 1924 }; | |
| 1925 enum FLAGS flags = FLAGS_decimal; | |
| 1926 | |
| 1927 int i; | |
| 1928 int base; | |
| 1929 unsigned c; | |
| 1930 unsigned char *start; | |
| 1931 TOK result; | |
| 1932 | |
| 1933 //printf("Lexer::number()\n"); | |
| 1934 state = STATE_initial; | |
| 1935 base = 0; | |
| 1936 stringbuffer.reset(); | |
| 1937 start = p; | |
| 1938 while (1) | |
| 1939 { | |
| 1940 c = *p; | |
| 1941 switch (state) | |
| 1942 { | |
| 1943 case STATE_initial: // opening state | |
| 1944 if (c == '0') | |
| 1945 state = STATE_0; | |
| 1946 else | |
| 1947 state = STATE_decimal; | |
| 1948 break; | |
| 1949 | |
| 1950 case STATE_0: | |
| 1951 flags = (FLAGS) (flags & ~FLAGS_decimal); | |
| 1952 switch (c) | |
| 1953 { | |
| 1954 #if ZEROH | |
| 1955 case 'H': // 0h | |
| 1956 case 'h': | |
| 1957 goto hexh; | |
| 1958 #endif | |
| 1959 case 'X': | |
| 1960 case 'x': | |
| 1961 state = STATE_hex0; | |
| 1962 break; | |
| 1963 | |
| 1964 case '.': | |
| 1965 if (p[1] == '.') // .. is a separate token | |
| 1966 goto done; | |
| 1967 case 'i': | |
| 1968 case 'f': | |
| 1969 case 'F': | |
| 1970 goto real; | |
| 1971 #if ZEROH | |
| 1972 case 'E': | |
| 1973 case 'e': | |
| 1974 goto case_hex; | |
| 1975 #endif | |
| 1976 case 'B': | |
| 1977 case 'b': | |
| 1978 state = STATE_binary0; | |
| 1979 break; | |
| 1980 | |
| 1981 case '0': case '1': case '2': case '3': | |
| 1982 case '4': case '5': case '6': case '7': | |
| 1983 state = STATE_octal; | |
| 1984 break; | |
| 1985 | |
| 1986 #if ZEROH | |
| 1987 case '8': case '9': case 'A': | |
| 1988 case 'C': case 'D': case 'F': | |
| 1989 case 'a': case 'c': case 'd': case 'f': | |
| 1990 case_hex: | |
| 1991 state = STATE_hexh; | |
| 1992 break; | |
| 1993 #endif | |
| 1994 case '_': | |
| 1995 state = STATE_octal; | |
| 1996 p++; | |
| 1997 continue; | |
| 1998 | |
| 1999 case 'L': | |
| 2000 if (p[1] == 'i') | |
| 2001 goto real; | |
| 2002 goto done; | |
| 2003 | |
| 2004 default: | |
| 2005 goto done; | |
| 2006 } | |
| 2007 break; | |
| 2008 | |
| 2009 case STATE_decimal: // reading decimal number | |
| 2010 if (!isdigit(c)) | |
| 2011 { | |
| 2012 #if ZEROH | |
| 2013 if (ishex(c) | |
| 2014 || c == 'H' || c == 'h' | |
| 2015 ) | |
| 2016 goto hexh; | |
| 2017 #endif | |
| 2018 if (c == '_') // ignore embedded _ | |
| 2019 { p++; | |
| 2020 continue; | |
| 2021 } | |
| 2022 if (c == '.' && p[1] != '.') | |
| 2023 goto real; | |
| 2024 else if (c == 'i' || c == 'f' || c == 'F' || | |
| 2025 c == 'e' || c == 'E') | |
| 2026 { | |
| 2027 real: // It's a real number. Back up and rescan as a real | |
| 2028 p = start; | |
| 2029 return inreal(t); | |
| 2030 } | |
| 2031 else if (c == 'L' && p[1] == 'i') | |
| 2032 goto real; | |
| 2033 goto done; | |
| 2034 } | |
| 2035 break; | |
| 2036 | |
| 2037 case STATE_hex0: // reading hex number | |
| 2038 case STATE_hex: | |
| 2039 if (!ishex(c)) | |
| 2040 { | |
| 2041 if (c == '_') // ignore embedded _ | |
| 2042 { p++; | |
| 2043 continue; | |
| 2044 } | |
| 2045 if (c == '.' && p[1] != '.') | |
| 2046 goto real; | |
| 2047 if (c == 'P' || c == 'p' || c == 'i') | |
| 2048 goto real; | |
| 2049 if (state == STATE_hex0) | |
| 2050 error("Hex digit expected, not '%c'", c); | |
| 2051 goto done; | |
| 2052 } | |
| 2053 state = STATE_hex; | |
| 2054 break; | |
| 2055 | |
| 2056 #if ZEROH | |
| 2057 hexh: | |
| 2058 state = STATE_hexh; | |
| 2059 case STATE_hexh: // parse numbers like 0FFh | |
| 2060 if (!ishex(c)) | |
| 2061 { | |
| 2062 if (c == 'H' || c == 'h') | |
| 2063 { | |
| 2064 p++; | |
| 2065 base = 16; | |
| 2066 goto done; | |
| 2067 } | |
| 2068 else | |
| 2069 { | |
| 2070 // Check for something like 1E3 or 0E24 | |
| 2071 if (memchr((char *)stringbuffer.data, 'E', stringbuffer.offset) || | |
| 2072 memchr((char *)stringbuffer.data, 'e', stringbuffer.offset)) | |
| 2073 goto real; | |
| 2074 error("Hex digit expected, not '%c'", c); | |
| 2075 goto done; | |
| 2076 } | |
| 2077 } | |
| 2078 break; | |
| 2079 #endif | |
| 2080 | |
| 2081 case STATE_octal: // reading octal number | |
| 2082 case STATE_octale: // reading octal number with non-octal digits | |
| 2083 if (!isoctal(c)) | |
| 2084 { | |
| 2085 #if ZEROH | |
| 2086 if (ishex(c) | |
| 2087 || c == 'H' || c == 'h' | |
| 2088 ) | |
| 2089 goto hexh; | |
| 2090 #endif | |
| 2091 if (c == '_') // ignore embedded _ | |
| 2092 { p++; | |
| 2093 continue; | |
| 2094 } | |
| 2095 if (c == '.' && p[1] != '.') | |
| 2096 goto real; | |
| 2097 if (c == 'i') | |
| 2098 goto real; | |
| 2099 if (isdigit(c)) | |
| 2100 { | |
| 2101 state = STATE_octale; | |
| 2102 } | |
| 2103 else | |
| 2104 goto done; | |
| 2105 } | |
| 2106 break; | |
| 2107 | |
| 2108 case STATE_binary0: // starting binary number | |
| 2109 case STATE_binary: // reading binary number | |
| 2110 if (c != '0' && c != '1') | |
| 2111 { | |
| 2112 #if ZEROH | |
| 2113 if (ishex(c) | |
| 2114 || c == 'H' || c == 'h' | |
| 2115 ) | |
| 2116 goto hexh; | |
| 2117 #endif | |
| 2118 if (c == '_') // ignore embedded _ | |
| 2119 { p++; | |
| 2120 continue; | |
| 2121 } | |
| 2122 if (state == STATE_binary0) | |
| 2123 { error("binary digit expected"); | |
| 2124 state = STATE_error; | |
| 2125 break; | |
| 2126 } | |
| 2127 else | |
| 2128 goto done; | |
| 2129 } | |
| 2130 state = STATE_binary; | |
| 2131 break; | |
| 2132 | |
| 2133 case STATE_error: // for error recovery | |
| 2134 if (!isdigit(c)) // scan until non-digit | |
| 2135 goto done; | |
| 2136 break; | |
| 2137 | |
| 2138 default: | |
| 2139 assert(0); | |
| 2140 } | |
| 2141 stringbuffer.writeByte(c); | |
| 2142 p++; | |
| 2143 } | |
| 2144 done: | |
| 2145 stringbuffer.writeByte(0); // terminate string | |
| 2146 if (state == STATE_octale) | |
| 2147 error("Octal digit expected"); | |
| 2148 | |
| 2149 uinteger_t n; // unsigned >=64 bit integer type | |
| 2150 | |
| 2151 if (stringbuffer.offset == 2 && (state == STATE_decimal || state == STATE_0)) | |
| 2152 n = stringbuffer.data[0] - '0'; | |
| 2153 else | |
| 2154 { | |
| 2155 // Convert string to integer | |
| 2156 #if __DMC__ | |
| 2157 errno = 0; | |
| 2158 n = strtoull((char *)stringbuffer.data,NULL,base); | |
| 2159 if (errno == ERANGE) | |
| 2160 error("integer overflow"); | |
| 2161 #else | |
| 2162 // Not everybody implements strtoull() | |
| 2163 char *p = (char *)stringbuffer.data; | |
| 2164 int r = 10, d; | |
| 2165 | |
| 2166 if (*p == '0') | |
| 2167 { | |
| 2168 if (p[1] == 'x' || p[1] == 'X') | |
| 2169 p += 2, r = 16; | |
| 2170 else if (p[1] == 'b' || p[1] == 'B') | |
| 2171 p += 2, r = 2; | |
| 2172 else if (isdigit(p[1])) | |
| 2173 p += 1, r = 8; | |
| 2174 } | |
| 2175 | |
| 2176 n = 0; | |
| 2177 while (1) | |
| 2178 { | |
| 2179 if (*p >= '0' && *p <= '9') | |
| 2180 d = *p - '0'; | |
| 2181 else if (*p >= 'a' && *p <= 'z') | |
| 2182 d = *p - 'a' + 10; | |
| 2183 else if (*p >= 'A' && *p <= 'Z') | |
| 2184 d = *p - 'A' + 10; | |
| 2185 else | |
| 2186 break; | |
| 2187 if (d >= r) | |
| 2188 break; | |
| 2189 if (n && n * r + d <= n) | |
| 2190 { | |
| 2191 error ("integer overflow"); | |
| 2192 break; | |
| 2193 } | |
| 2194 | |
| 2195 n = n * r + d; | |
| 2196 p++; | |
| 2197 } | |
| 2198 #endif | |
| 2199 if (sizeof(n) > 8 && | |
| 2200 n > 0xFFFFFFFFFFFFFFFFULL) // if n needs more than 64 bits | |
| 2201 error("integer overflow"); | |
| 2202 } | |
| 2203 | |
| 2204 // Parse trailing 'u', 'U', 'l' or 'L' in any combination | |
| 2205 while (1) | |
| 2206 { unsigned char f; | |
| 2207 | |
| 2208 switch (*p) | |
| 2209 { case 'U': | |
| 2210 case 'u': | |
| 2211 f = FLAGS_unsigned; | |
| 2212 goto L1; | |
| 2213 | |
| 2214 case 'l': | |
| 2215 if (1 || !global.params.useDeprecated) | |
| 2216 error("'l' suffix is deprecated, use 'L' instead"); | |
| 2217 case 'L': | |
| 2218 f = FLAGS_long; | |
| 2219 L1: | |
| 2220 p++; | |
| 2221 if (flags & f) | |
| 2222 error("unrecognized token"); | |
| 2223 flags = (FLAGS) (flags | f); | |
| 2224 continue; | |
| 2225 default: | |
| 2226 break; | |
| 2227 } | |
| 2228 break; | |
| 2229 } | |
| 2230 | |
| 2231 switch (flags) | |
| 2232 { | |
| 2233 case 0: | |
| 2234 /* Octal or Hexadecimal constant. | |
| 2235 * First that fits: int, uint, long, ulong | |
| 2236 */ | |
| 2237 if (n & 0x8000000000000000LL) | |
| 2238 result = TOKuns64v; | |
| 2239 else if (n & 0xFFFFFFFF00000000LL) | |
| 2240 result = TOKint64v; | |
| 2241 else if (n & 0x80000000) | |
| 2242 result = TOKuns32v; | |
| 2243 else | |
| 2244 result = TOKint32v; | |
| 2245 break; | |
| 2246 | |
| 2247 case FLAGS_decimal: | |
| 2248 /* First that fits: int, long, long long | |
| 2249 */ | |
| 2250 if (n & 0x8000000000000000LL) | |
| 2251 { error("signed integer overflow"); | |
| 2252 result = TOKuns64v; | |
| 2253 } | |
| 2254 else if (n & 0xFFFFFFFF80000000LL) | |
| 2255 result = TOKint64v; | |
| 2256 else | |
| 2257 result = TOKint32v; | |
| 2258 break; | |
| 2259 | |
| 2260 case FLAGS_unsigned: | |
| 2261 case FLAGS_decimal | FLAGS_unsigned: | |
| 2262 /* First that fits: uint, ulong | |
| 2263 */ | |
| 2264 if (n & 0xFFFFFFFF00000000LL) | |
| 2265 result = TOKuns64v; | |
| 2266 else | |
| 2267 result = TOKuns32v; | |
| 2268 break; | |
| 2269 | |
| 2270 case FLAGS_decimal | FLAGS_long: | |
| 2271 if (n & 0x8000000000000000LL) | |
| 2272 { error("signed integer overflow"); | |
| 2273 result = TOKuns64v; | |
| 2274 } | |
| 2275 else | |
| 2276 result = TOKint64v; | |
| 2277 break; | |
| 2278 | |
| 2279 case FLAGS_long: | |
| 2280 if (n & 0x8000000000000000LL) | |
| 2281 result = TOKuns64v; | |
| 2282 else | |
| 2283 result = TOKint64v; | |
| 2284 break; | |
| 2285 | |
| 2286 case FLAGS_unsigned | FLAGS_long: | |
| 2287 case FLAGS_decimal | FLAGS_unsigned | FLAGS_long: | |
| 2288 result = TOKuns64v; | |
| 2289 break; | |
| 2290 | |
| 2291 default: | |
| 2292 #ifdef DEBUG | |
| 2293 printf("%x\n",flags); | |
| 2294 #endif | |
| 2295 assert(0); | |
| 2296 } | |
| 2297 t->uns64value = n; | |
| 2298 return result; | |
| 2299 } | |
| 2300 | |
| 2301 /************************************** | |
| 2302 * Read in characters, converting them to real. | |
| 2303 * Bugs: | |
| 2304 * Exponent overflow not detected. | |
| 2305 * Too much requested precision is not detected. | |
| 2306 */ | |
| 2307 | |
| 2308 TOK Lexer::inreal(Token *t) | |
| 2309 #ifdef __DMC__ | |
| 2310 __in | |
| 2311 { | |
| 2312 assert(*p == '.' || isdigit(*p)); | |
| 2313 } | |
| 2314 __out (result) | |
| 2315 { | |
| 2316 switch (result) | |
| 2317 { | |
| 2318 case TOKfloat32v: | |
| 2319 case TOKfloat64v: | |
| 2320 case TOKfloat80v: | |
| 2321 case TOKimaginary32v: | |
| 2322 case TOKimaginary64v: | |
| 2323 case TOKimaginary80v: | |
| 2324 break; | |
| 2325 | |
| 2326 default: | |
| 2327 assert(0); | |
| 2328 } | |
| 2329 } | |
| 2330 __body | |
| 2331 #endif /* __DMC__ */ | |
| 2332 { int dblstate; | |
| 2333 unsigned c; | |
| 2334 char hex; // is this a hexadecimal-floating-constant? | |
| 2335 TOK result; | |
| 2336 | |
| 2337 //printf("Lexer::inreal()\n"); | |
| 2338 stringbuffer.reset(); | |
| 2339 dblstate = 0; | |
| 2340 hex = 0; | |
| 2341 Lnext: | |
| 2342 while (1) | |
| 2343 { | |
| 2344 // Get next char from input | |
| 2345 c = *p++; | |
| 2346 //printf("dblstate = %d, c = '%c'\n", dblstate, c); | |
| 2347 while (1) | |
| 2348 { | |
| 2349 switch (dblstate) | |
| 2350 { | |
| 2351 case 0: // opening state | |
| 2352 if (c == '0') | |
| 2353 dblstate = 9; | |
| 2354 else if (c == '.') | |
| 2355 dblstate = 3; | |
| 2356 else | |
| 2357 dblstate = 1; | |
| 2358 break; | |
| 2359 | |
| 2360 case 9: | |
| 2361 dblstate = 1; | |
| 2362 if (c == 'X' || c == 'x') | |
| 2363 { hex++; | |
| 2364 break; | |
| 2365 } | |
| 2366 case 1: // digits to left of . | |
| 2367 case 3: // digits to right of . | |
| 2368 case 7: // continuing exponent digits | |
| 2369 if (!isdigit(c) && !(hex && isxdigit(c))) | |
| 2370 { | |
| 2371 if (c == '_') | |
| 2372 goto Lnext; // ignore embedded '_' | |
| 2373 dblstate++; | |
| 2374 continue; | |
| 2375 } | |
| 2376 break; | |
| 2377 | |
| 2378 case 2: // no more digits to left of . | |
| 2379 if (c == '.') | |
| 2380 { dblstate++; | |
| 2381 break; | |
| 2382 } | |
| 2383 case 4: // no more digits to right of . | |
| 2384 if ((c == 'E' || c == 'e') || | |
| 2385 hex && (c == 'P' || c == 'p')) | |
| 2386 { dblstate = 5; | |
| 2387 hex = 0; // exponent is always decimal | |
| 2388 break; | |
| 2389 } | |
| 2390 if (hex) | |
| 2391 error("binary-exponent-part required"); | |
| 2392 goto done; | |
| 2393 | |
| 2394 case 5: // looking immediately to right of E | |
| 2395 dblstate++; | |
| 2396 if (c == '-' || c == '+') | |
| 2397 break; | |
| 2398 case 6: // 1st exponent digit expected | |
| 2399 if (!isdigit(c)) | |
| 2400 error("exponent expected"); | |
| 2401 dblstate++; | |
| 2402 break; | |
| 2403 | |
| 2404 case 8: // past end of exponent digits | |
| 2405 goto done; | |
| 2406 } | |
| 2407 break; | |
| 2408 } | |
| 2409 stringbuffer.writeByte(c); | |
| 2410 } | |
| 2411 done: | |
| 2412 p--; | |
| 2413 | |
| 2414 stringbuffer.writeByte(0); | |
| 2415 | |
| 2416 #if _WIN32 && __DMC__ | |
| 2417 char *save = __locale_decpoint; | |
| 2418 __locale_decpoint = "."; | |
| 2419 #endif | |
| 2420 #ifdef IN_GCC | |
| 2421 t->float80value = real_t::parse((char *)stringbuffer.data, real_t::LongDouble); | |
| 2422 #else | |
| 2423 t->float80value = strtold((char *)stringbuffer.data, NULL); | |
| 2424 #endif | |
| 2425 errno = 0; | |
| 2426 switch (*p) | |
| 2427 { | |
| 2428 case 'F': | |
| 2429 case 'f': | |
| 2430 #ifdef IN_GCC | |
| 2431 real_t::parse((char *)stringbuffer.data, real_t::Float); | |
| 2432 #else | |
| 2433 strtof((char *)stringbuffer.data, NULL); | |
| 2434 #endif | |
| 2435 result = TOKfloat32v; | |
| 2436 p++; | |
| 2437 break; | |
| 2438 | |
| 2439 default: | |
| 2440 #ifdef IN_GCC | |
| 2441 real_t::parse((char *)stringbuffer.data, real_t::Double); | |
| 2442 #else | |
| 2443 strtod((char *)stringbuffer.data, NULL); | |
| 2444 #endif | |
| 2445 result = TOKfloat64v; | |
| 2446 break; | |
| 2447 | |
| 2448 case 'l': | |
| 2449 if (!global.params.useDeprecated) | |
| 2450 error("'l' suffix is deprecated, use 'L' instead"); | |
| 2451 case 'L': | |
| 2452 result = TOKfloat80v; | |
| 2453 p++; | |
| 2454 break; | |
| 2455 } | |
| 2456 if (*p == 'i' || *p == 'I') | |
| 2457 { | |
| 2458 if (!global.params.useDeprecated && *p == 'I') | |
| 2459 error("'I' suffix is deprecated, use 'i' instead"); | |
| 2460 p++; | |
| 2461 switch (result) | |
| 2462 { | |
| 2463 case TOKfloat32v: | |
| 2464 result = TOKimaginary32v; | |
| 2465 break; | |
| 2466 case TOKfloat64v: | |
| 2467 result = TOKimaginary64v; | |
| 2468 break; | |
| 2469 case TOKfloat80v: | |
| 2470 result = TOKimaginary80v; | |
| 2471 break; | |
| 2472 } | |
| 2473 } | |
| 2474 #if _WIN32 && __DMC__ | |
| 2475 __locale_decpoint = save; | |
| 2476 #endif | |
| 2477 if (errno == ERANGE) | |
| 2478 error("number is not representable"); | |
| 2479 return result; | |
| 2480 } | |
| 2481 | |
| 2482 /********************************************* | |
| 2483 * Do pragma. | |
| 2484 * Currently, the only pragma supported is: | |
| 2485 * #line linnum [filespec] | |
| 2486 */ | |
| 2487 | |
| 2488 void Lexer::pragma() | |
| 2489 { | |
| 2490 Token tok; | |
| 2491 int linnum; | |
| 2492 char *filespec = NULL; | |
| 2493 Loc loc = this->loc; | |
| 2494 | |
| 2495 scan(&tok); | |
| 2496 if (tok.value != TOKidentifier || tok.ident != Id::line) | |
| 2497 goto Lerr; | |
| 2498 | |
| 2499 scan(&tok); | |
| 2500 if (tok.value == TOKint32v || tok.value == TOKint64v) | |
| 2501 linnum = tok.uns64value - 1; | |
| 2502 else | |
| 2503 goto Lerr; | |
| 2504 | |
| 2505 while (1) | |
| 2506 { | |
| 2507 switch (*p) | |
| 2508 { | |
| 2509 case 0: | |
| 2510 case 0x1A: | |
| 2511 case '\n': | |
| 2512 Lnewline: | |
| 2513 this->loc.linnum = linnum; | |
| 2514 if (filespec) | |
| 2515 this->loc.filename = filespec; | |
| 2516 return; | |
| 2517 | |
| 2518 case '\r': | |
| 2519 p++; | |
| 2520 if (*p != '\n') | |
| 2521 { p--; | |
| 2522 goto Lnewline; | |
| 2523 } | |
| 2524 continue; | |
| 2525 | |
| 2526 case ' ': | |
| 2527 case '\t': | |
| 2528 case '\v': | |
| 2529 case '\f': | |
| 2530 p++; | |
| 2531 continue; // skip white space | |
| 2532 | |
| 2533 case '_': | |
| 2534 if (mod && memcmp(p, "__FILE__", 8) == 0) | |
| 2535 { | |
| 2536 p += 8; | |
| 2537 filespec = mem.strdup(loc.filename ? loc.filename : mod->ident->toChars()); | |
| 2538 } | |
| 2539 continue; | |
| 2540 | |
| 2541 case '"': | |
| 2542 if (filespec) | |
| 2543 goto Lerr; | |
| 2544 stringbuffer.reset(); | |
| 2545 p++; | |
| 2546 while (1) | |
| 2547 { unsigned c; | |
| 2548 | |
| 2549 c = *p; | |
| 2550 switch (c) | |
| 2551 { | |
| 2552 case '\n': | |
| 2553 case '\r': | |
| 2554 case 0: | |
| 2555 case 0x1A: | |
| 2556 goto Lerr; | |
| 2557 | |
| 2558 case '"': | |
| 2559 stringbuffer.writeByte(0); | |
| 2560 filespec = mem.strdup((char *)stringbuffer.data); | |
| 2561 p++; | |
| 2562 break; | |
| 2563 | |
| 2564 default: | |
| 2565 if (c & 0x80) | |
| 2566 { unsigned u = decodeUTF(); | |
| 2567 if (u == PS || u == LS) | |
| 2568 goto Lerr; | |
| 2569 } | |
| 2570 stringbuffer.writeByte(c); | |
| 2571 p++; | |
| 2572 continue; | |
| 2573 } | |
| 2574 break; | |
| 2575 } | |
| 2576 continue; | |
| 2577 | |
| 2578 default: | |
| 2579 if (*p & 0x80) | |
| 2580 { unsigned u = decodeUTF(); | |
| 2581 if (u == PS || u == LS) | |
| 2582 goto Lnewline; | |
| 2583 } | |
| 2584 goto Lerr; | |
| 2585 } | |
| 2586 } | |
| 2587 | |
| 2588 Lerr: | |
| 2589 error(loc, "#line integer [\"filespec\"]\\n expected"); | |
| 2590 } | |
| 2591 | |
| 2592 | |
| 2593 /******************************************** | |
| 2594 * Decode UTF character. | |
| 2595 * Issue error messages for invalid sequences. | |
| 2596 * Return decoded character, advance p to last character in UTF sequence. | |
| 2597 */ | |
| 2598 | |
| 2599 unsigned Lexer::decodeUTF() | |
| 2600 { | |
| 2601 dchar_t u; | |
| 2602 unsigned char c; | |
| 2603 unsigned char *s = p; | |
| 2604 size_t len; | |
| 2605 size_t idx; | |
| 2606 char *msg; | |
| 2607 | |
| 2608 c = *s; | |
| 2609 assert(c & 0x80); | |
| 2610 | |
| 2611 // Check length of remaining string up to 6 UTF-8 characters | |
| 2612 for (len = 1; len < 6 && s[len]; len++) | |
| 2613 ; | |
| 2614 | |
| 2615 idx = 0; | |
| 2616 msg = utf_decodeChar(s, len, &idx, &u); | |
| 2617 p += idx - 1; | |
| 2618 if (msg) | |
| 2619 { | |
| 2620 error("%s", msg); | |
| 2621 } | |
| 2622 return u; | |
| 2623 } | |
| 2624 | |
| 2625 | |
| 2626 /*************************************************** | |
| 2627 * Parse doc comment embedded between t->ptr and p. | |
| 2628 * Remove trailing blanks and tabs from lines. | |
| 2629 * Replace all newlines with \n. | |
| 2630 * Remove leading comment character from each line. | |
| 2631 * Decide if it's a lineComment or a blockComment. | |
| 2632 * Append to previous one for this token. | |
| 2633 */ | |
| 2634 | |
| 2635 void Lexer::getDocComment(Token *t, unsigned lineComment) | |
| 2636 { | |
| 2637 OutBuffer buf; | |
| 2638 unsigned char ct = t->ptr[2]; | |
| 2639 unsigned char *q = t->ptr + 3; // start of comment text | |
| 2640 int linestart = 0; | |
| 2641 | |
| 2642 unsigned char *qend = p; | |
| 2643 if (ct == '*' || ct == '+') | |
| 2644 qend -= 2; | |
| 2645 | |
| 2646 /* Scan over initial row of ****'s or ++++'s or ////'s | |
| 2647 */ | |
| 2648 for (; q < qend; q++) | |
| 2649 { | |
| 2650 if (*q != ct) | |
| 2651 break; | |
| 2652 } | |
| 2653 | |
| 2654 /* Remove trailing row of ****'s or ++++'s | |
| 2655 */ | |
| 2656 if (ct != '/') | |
| 2657 { | |
| 2658 for (; q < qend; qend--) | |
| 2659 { | |
| 2660 if (qend[-1] != ct) | |
| 2661 break; | |
| 2662 } | |
| 2663 } | |
| 2664 | |
| 2665 for (; q < qend; q++) | |
| 2666 { | |
| 2667 unsigned char c = *q; | |
| 2668 | |
| 2669 switch (c) | |
| 2670 { | |
| 2671 case '*': | |
| 2672 case '+': | |
| 2673 if (linestart && c == ct) | |
| 2674 { linestart = 0; | |
| 2675 /* Trim preceding whitespace up to preceding \n | |
| 2676 */ | |
| 2677 while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) | |
| 2678 buf.offset--; | |
| 2679 continue; | |
| 2680 } | |
| 2681 break; | |
| 2682 | |
| 2683 case ' ': | |
| 2684 case '\t': | |
| 2685 break; | |
| 2686 | |
| 2687 case '\r': | |
| 2688 if (q[1] == '\n') | |
| 2689 continue; // skip the \r | |
| 2690 goto Lnewline; | |
| 2691 | |
| 2692 default: | |
| 2693 if (c == 226) | |
| 2694 { | |
| 2695 // If LS or PS | |
| 2696 if (q[1] == 128 && | |
| 2697 (q[2] == 168 || q[2] == 169)) | |
| 2698 { | |
| 2699 q += 2; | |
| 2700 goto Lnewline; | |
| 2701 } | |
| 2702 } | |
| 2703 linestart = 0; | |
| 2704 break; | |
| 2705 | |
| 2706 Lnewline: | |
| 2707 c = '\n'; // replace all newlines with \n | |
| 2708 case '\n': | |
| 2709 linestart = 1; | |
| 2710 | |
| 2711 /* Trim trailing whitespace | |
| 2712 */ | |
| 2713 while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) | |
| 2714 buf.offset--; | |
| 2715 | |
| 2716 break; | |
| 2717 } | |
| 2718 buf.writeByte(c); | |
| 2719 } | |
| 2720 | |
| 2721 // Always end with a newline | |
| 2722 if (!buf.offset || buf.data[buf.offset - 1] != '\n') | |
| 2723 buf.writeByte('\n'); | |
| 2724 | |
| 2725 buf.writeByte(0); | |
| 2726 | |
| 2727 // It's a line comment if the start of the doc comment comes | |
| 2728 // after other non-whitespace on the same line. | |
| 2729 unsigned char** dc = (lineComment && anyToken) | |
| 2730 ? &t->lineComment | |
| 2731 : &t->blockComment; | |
| 2732 | |
| 2733 // Combine with previous doc comment, if any | |
| 2734 if (*dc) | |
| 2735 *dc = combineComments(*dc, (unsigned char *)buf.data); | |
| 2736 else | |
| 2737 *dc = (unsigned char *)buf.extractData(); | |
| 2738 } | |
| 2739 | |
| 2740 /******************************************** | |
| 2741 * Combine two document comments into one. | |
| 2742 */ | |
| 2743 | |
| 2744 unsigned char *Lexer::combineComments(unsigned char *c1, unsigned char *c2) | |
| 2745 { | |
| 2746 unsigned char *c = c2; | |
| 2747 | |
| 2748 if (c1) | |
| 2749 { c = c1; | |
| 2750 if (c2) | |
| 2751 { size_t len1 = strlen((char *)c1); | |
| 2752 size_t len2 = strlen((char *)c2); | |
| 2753 | |
| 2754 c = (unsigned char *)mem.malloc(len1 + 1 + len2 + 1); | |
| 2755 memcpy(c, c1, len1); | |
| 2756 c[len1] = '\n'; | |
| 2757 memcpy(c + len1 + 1, c2, len2); | |
| 2758 c[len1 + 1 + len2] = 0; | |
| 2759 } | |
| 2760 } | |
| 2761 return c; | |
| 2762 } | |
| 2763 | |
| 2764 /******************************************** | |
| 2765 * Create an identifier in the string table. | |
| 2766 */ | |
| 2767 | |
| 2768 Identifier *Lexer::idPool(const char *s) | |
| 2769 { | |
| 2770 size_t len = strlen(s); | |
| 2771 StringValue *sv = stringtable.update(s, len); | |
| 2772 Identifier *id = (Identifier *) sv->ptrvalue; | |
| 2773 if (!id) | |
| 2774 { | |
| 2775 id = new Identifier(sv->lstring.string, TOKidentifier); | |
| 2776 sv->ptrvalue = id; | |
| 2777 } | |
| 2778 return id; | |
| 2779 } | |
| 2780 | |
| 2781 /********************************************* | |
| 2782 * Create a unique identifier using the prefix s. | |
| 2783 */ | |
| 2784 | |
| 2785 Identifier *Lexer::uniqueId(const char *s, int num) | |
| 2786 { char buffer[32]; | |
| 2787 size_t slen = strlen(s); | |
| 2788 | |
| 2789 assert(slen + sizeof(num) * 3 + 1 <= sizeof(buffer)); | |
| 2790 sprintf(buffer, "%s%d", s, num); | |
| 2791 return idPool(buffer); | |
| 2792 } | |
| 2793 | |
| 2794 Identifier *Lexer::uniqueId(const char *s) | |
| 2795 { | |
| 2796 static int num; | |
| 2797 return uniqueId(s, ++num); | |
| 2798 } | |
| 2799 | |
| 2800 /**************************************** | |
| 2801 */ | |
| 2802 | |
| 2803 struct Keyword | |
| 2804 { char *name; | |
| 2805 enum TOK value; | |
| 2806 }; | |
| 2807 | |
| 2808 static Keyword keywords[] = | |
| 2809 { | |
| 2810 // { "", TOK }, | |
| 2811 | |
| 2812 { "this", TOKthis }, | |
| 2813 { "super", TOKsuper }, | |
| 2814 { "assert", TOKassert }, | |
| 2815 { "null", TOKnull }, | |
| 2816 { "true", TOKtrue }, | |
| 2817 { "false", TOKfalse }, | |
| 2818 { "cast", TOKcast }, | |
| 2819 { "new", TOKnew }, | |
| 2820 { "delete", TOKdelete }, | |
| 2821 { "throw", TOKthrow }, | |
| 2822 { "module", TOKmodule }, | |
| 2823 { "pragma", TOKpragma }, | |
| 2824 { "typeof", TOKtypeof }, | |
| 2825 { "typeid", TOKtypeid }, | |
| 2826 | |
| 2827 { "template", TOKtemplate }, | |
| 2828 | |
| 2829 { "void", TOKvoid }, | |
| 2830 { "byte", TOKint8 }, | |
| 2831 { "ubyte", TOKuns8 }, | |
| 2832 { "short", TOKint16 }, | |
| 2833 { "ushort", TOKuns16 }, | |
| 2834 { "int", TOKint32 }, | |
| 2835 { "uint", TOKuns32 }, | |
| 2836 { "long", TOKint64 }, | |
| 2837 { "ulong", TOKuns64 }, | |
| 2838 { "cent", TOKcent, }, | |
| 2839 { "ucent", TOKucent, }, | |
| 2840 { "float", TOKfloat32 }, | |
| 2841 { "double", TOKfloat64 }, | |
| 2842 { "real", TOKfloat80 }, | |
| 2843 | |
| 2844 { "bool", TOKbool }, | |
| 2845 { "char", TOKchar }, | |
| 2846 { "wchar", TOKwchar }, | |
| 2847 { "dchar", TOKdchar }, | |
| 2848 | |
| 2849 { "ifloat", TOKimaginary32 }, | |
| 2850 { "idouble", TOKimaginary64 }, | |
| 2851 { "ireal", TOKimaginary80 }, | |
| 2852 | |
| 2853 { "cfloat", TOKcomplex32 }, | |
| 2854 { "cdouble", TOKcomplex64 }, | |
| 2855 { "creal", TOKcomplex80 }, | |
| 2856 | |
| 2857 { "delegate", TOKdelegate }, | |
| 2858 { "function", TOKfunction }, | |
| 2859 | |
| 2860 { "is", TOKis }, | |
| 2861 { "if", TOKif }, | |
| 2862 { "else", TOKelse }, | |
| 2863 { "while", TOKwhile }, | |
| 2864 { "for", TOKfor }, | |
| 2865 { "do", TOKdo }, | |
| 2866 { "switch", TOKswitch }, | |
| 2867 { "case", TOKcase }, | |
| 2868 { "default", TOKdefault }, | |
| 2869 { "break", TOKbreak }, | |
| 2870 { "continue", TOKcontinue }, | |
| 2871 { "synchronized", TOKsynchronized }, | |
| 2872 { "return", TOKreturn }, | |
| 2873 { "goto", TOKgoto }, | |
| 2874 { "try", TOKtry }, | |
| 2875 { "catch", TOKcatch }, | |
| 2876 { "finally", TOKfinally }, | |
| 2877 { "with", TOKwith }, | |
| 2878 { "asm", TOKasm }, | |
| 2879 { "foreach", TOKforeach }, | |
| 2880 { "foreach_reverse", TOKforeach_reverse }, | |
| 2881 { "scope", TOKscope }, | |
| 2882 | |
| 2883 { "struct", TOKstruct }, | |
| 2884 { "class", TOKclass }, | |
| 2885 { "interface", TOKinterface }, | |
| 2886 { "union", TOKunion }, | |
| 2887 { "enum", TOKenum }, | |
| 2888 { "import", TOKimport }, | |
| 2889 { "mixin", TOKmixin }, | |
| 2890 { "static", TOKstatic }, | |
| 2891 { "final", TOKfinal }, | |
| 2892 { "const", TOKconst }, | |
| 2893 { "typedef", TOKtypedef }, | |
| 2894 { "alias", TOKalias }, | |
| 2895 { "override", TOKoverride }, | |
| 2896 { "abstract", TOKabstract }, | |
| 2897 { "volatile", TOKvolatile }, | |
| 2898 { "debug", TOKdebug }, | |
| 2899 { "deprecated", TOKdeprecated }, | |
| 2900 { "in", TOKin }, | |
| 2901 { "out", TOKout }, | |
| 2902 { "inout", TOKinout }, | |
| 2903 { "lazy", TOKlazy }, | |
| 2904 { "auto", TOKauto }, | |
| 2905 | |
| 2906 { "align", TOKalign }, | |
| 2907 { "extern", TOKextern }, | |
| 2908 { "private", TOKprivate }, | |
| 2909 { "package", TOKpackage }, | |
| 2910 { "protected", TOKprotected }, | |
| 2911 { "public", TOKpublic }, | |
| 2912 { "export", TOKexport }, | |
| 2913 | |
| 2914 { "body", TOKbody }, | |
| 2915 { "invariant", TOKinvariant }, | |
| 2916 { "unittest", TOKunittest }, | |
| 2917 { "version", TOKversion }, | |
| 2918 //{ "manifest", TOKmanifest }, | |
| 2919 | |
| 2920 // Added after 1.0 | |
| 2921 { "ref", TOKref }, | |
| 2922 { "macro", TOKmacro }, | |
| 336 | 2923 #if DMDV2 |
| 159 | 2924 { "pure", TOKpure }, |
| 2925 { "nothrow", TOKnothrow }, | |
| 336 | 2926 { "__thread", TOKtls }, |
| 159 | 2927 { "__traits", TOKtraits }, |
| 2928 { "__overloadset", TOKoverloadset }, | |
| 336 | 2929 { "__FILE__", TOKfile }, |
| 2930 { "__LINE__", TOKline }, | |
| 159 | 2931 #endif |
| 2932 }; | |
| 2933 | |
| 2934 int Token::isKeyword() | |
| 2935 { | |
| 2936 for (unsigned u = 0; u < sizeof(keywords) / sizeof(keywords[0]); u++) | |
| 2937 { | |
| 2938 if (keywords[u].value == value) | |
| 2939 return 1; | |
| 2940 } | |
| 2941 return 0; | |
| 2942 } | |
| 2943 | |
| 2944 void Lexer::initKeywords() | |
| 2945 { StringValue *sv; | |
| 2946 unsigned u; | |
| 2947 enum TOK v; | |
| 2948 unsigned nkeywords = sizeof(keywords) / sizeof(keywords[0]); | |
| 2949 | |
| 2950 if (global.params.Dversion == 1) | |
| 2951 nkeywords -= 2; | |
| 2952 | |
| 2953 cmtable_init(); | |
| 2954 | |
| 2955 for (u = 0; u < nkeywords; u++) | |
| 2956 { char *s; | |
| 2957 | |
| 2958 //printf("keyword[%d] = '%s'\n",u, keywords[u].name); | |
| 2959 s = keywords[u].name; | |
| 2960 v = keywords[u].value; | |
| 2961 sv = stringtable.insert(s, strlen(s)); | |
| 2962 sv->ptrvalue = (void *) new Identifier(sv->lstring.string,v); | |
| 2963 | |
| 2964 //printf("tochars[%d] = '%s'\n",v, s); | |
| 2965 Token::tochars[v] = s; | |
| 2966 } | |
| 2967 | |
| 2968 Token::tochars[TOKeof] = "EOF"; | |
| 2969 Token::tochars[TOKlcurly] = "{"; | |
| 2970 Token::tochars[TOKrcurly] = "}"; | |
| 2971 Token::tochars[TOKlparen] = "("; | |
| 2972 Token::tochars[TOKrparen] = ")"; | |
| 2973 Token::tochars[TOKlbracket] = "["; | |
| 2974 Token::tochars[TOKrbracket] = "]"; | |
| 2975 Token::tochars[TOKsemicolon] = ";"; | |
| 2976 Token::tochars[TOKcolon] = ":"; | |
| 2977 Token::tochars[TOKcomma] = ","; | |
| 2978 Token::tochars[TOKdot] = "."; | |
| 2979 Token::tochars[TOKxor] = "^"; | |
| 2980 Token::tochars[TOKxorass] = "^="; | |
| 2981 Token::tochars[TOKassign] = "="; | |
| 2982 Token::tochars[TOKconstruct] = "="; | |
| 336 | 2983 #if DMDV2 |
| 159 | 2984 Token::tochars[TOKblit] = "="; |
| 2985 #endif | |
| 2986 Token::tochars[TOKlt] = "<"; | |
| 2987 Token::tochars[TOKgt] = ">"; | |
| 2988 Token::tochars[TOKle] = "<="; | |
| 2989 Token::tochars[TOKge] = ">="; | |
| 2990 Token::tochars[TOKequal] = "=="; | |
| 2991 Token::tochars[TOKnotequal] = "!="; | |
| 2992 Token::tochars[TOKnotidentity] = "!is"; | |
| 2993 Token::tochars[TOKtobool] = "!!"; | |
| 2994 | |
| 2995 Token::tochars[TOKunord] = "!<>="; | |
| 2996 Token::tochars[TOKue] = "!<>"; | |
| 2997 Token::tochars[TOKlg] = "<>"; | |
| 2998 Token::tochars[TOKleg] = "<>="; | |
| 2999 Token::tochars[TOKule] = "!>"; | |
| 3000 Token::tochars[TOKul] = "!>="; | |
| 3001 Token::tochars[TOKuge] = "!<"; | |
| 3002 Token::tochars[TOKug] = "!<="; | |
| 3003 | |
| 3004 Token::tochars[TOKnot] = "!"; | |
| 3005 Token::tochars[TOKtobool] = "!!"; | |
| 3006 Token::tochars[TOKshl] = "<<"; | |
| 3007 Token::tochars[TOKshr] = ">>"; | |
| 3008 Token::tochars[TOKushr] = ">>>"; | |
| 3009 Token::tochars[TOKadd] = "+"; | |
| 3010 Token::tochars[TOKmin] = "-"; | |
| 3011 Token::tochars[TOKmul] = "*"; | |
| 3012 Token::tochars[TOKdiv] = "/"; | |
| 3013 Token::tochars[TOKmod] = "%"; | |
| 3014 Token::tochars[TOKslice] = ".."; | |
| 3015 Token::tochars[TOKdotdotdot] = "..."; | |
| 3016 Token::tochars[TOKand] = "&"; | |
| 3017 Token::tochars[TOKandand] = "&&"; | |
| 3018 Token::tochars[TOKor] = "|"; | |
| 3019 Token::tochars[TOKoror] = "||"; | |
| 3020 Token::tochars[TOKarray] = "[]"; | |
| 3021 Token::tochars[TOKindex] = "[i]"; | |
| 3022 Token::tochars[TOKaddress] = "&"; | |
| 3023 Token::tochars[TOKstar] = "*"; | |
| 3024 Token::tochars[TOKtilde] = "~"; | |
| 3025 Token::tochars[TOKdollar] = "$"; | |
| 3026 Token::tochars[TOKcast] = "cast"; | |
| 3027 Token::tochars[TOKplusplus] = "++"; | |
| 3028 Token::tochars[TOKminusminus] = "--"; | |
| 3029 Token::tochars[TOKtype] = "type"; | |
| 3030 Token::tochars[TOKquestion] = "?"; | |
| 3031 Token::tochars[TOKneg] = "-"; | |
| 3032 Token::tochars[TOKuadd] = "+"; | |
| 3033 Token::tochars[TOKvar] = "var"; | |
| 3034 Token::tochars[TOKaddass] = "+="; | |
| 3035 Token::tochars[TOKminass] = "-="; | |
| 3036 Token::tochars[TOKmulass] = "*="; | |
| 3037 Token::tochars[TOKdivass] = "/="; | |
| 3038 Token::tochars[TOKmodass] = "%="; | |
| 3039 Token::tochars[TOKshlass] = "<<="; | |
| 3040 Token::tochars[TOKshrass] = ">>="; | |
| 3041 Token::tochars[TOKushrass] = ">>>="; | |
| 3042 Token::tochars[TOKandass] = "&="; | |
| 3043 Token::tochars[TOKorass] = "|="; | |
| 3044 Token::tochars[TOKcatass] = "~="; | |
| 3045 Token::tochars[TOKcat] = "~"; | |
| 3046 Token::tochars[TOKcall] = "call"; | |
| 3047 Token::tochars[TOKidentity] = "is"; | |
| 3048 Token::tochars[TOKnotidentity] = "!is"; | |
| 3049 | |
| 3050 Token::tochars[TOKorass] = "|="; | |
| 3051 Token::tochars[TOKidentifier] = "identifier"; | |
| 3052 | |
| 3053 // For debugging | |
| 3054 Token::tochars[TOKdotexp] = "dotexp"; | |
| 3055 Token::tochars[TOKdotti] = "dotti"; | |
| 3056 Token::tochars[TOKdotvar] = "dotvar"; | |
| 3057 Token::tochars[TOKdottype] = "dottype"; | |
| 3058 Token::tochars[TOKsymoff] = "symoff"; | |
| 3059 Token::tochars[TOKtypedot] = "typedot"; | |
| 3060 Token::tochars[TOKarraylength] = "arraylength"; | |
| 3061 Token::tochars[TOKarrayliteral] = "arrayliteral"; | |
| 3062 Token::tochars[TOKassocarrayliteral] = "assocarrayliteral"; | |
| 3063 Token::tochars[TOKstructliteral] = "structliteral"; | |
| 3064 Token::tochars[TOKstring] = "string"; | |
| 3065 Token::tochars[TOKdsymbol] = "symbol"; | |
| 3066 Token::tochars[TOKtuple] = "tuple"; | |
| 3067 Token::tochars[TOKdeclaration] = "declaration"; | |
| 3068 Token::tochars[TOKdottd] = "dottd"; | |
| 3069 Token::tochars[TOKon_scope_exit] = "scope(exit)"; | |
| 3070 } |