Mercurial > projects > dil

comparison trunk/src/dil/Lexer.d @ 497:0ffcc4ff82f3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Refactored a few things in the Lexer. Fix: SpecialTokensEnd should be TOK.VERSION not TOK.Version. Fixed assert statement in textSpan().
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Sun, 09 Dec 2007 14:58:38 +0100
parents 5a607597dc22
children 49c201b5c465
comparison
equal deleted inserted replaced
496:5a607597dc22 497:0ffcc4ff82f3
295 idtable[str] = Identifier(TOK.Identifier, str); 295 idtable[str] = Identifier(TOK.Identifier, str);
296 id = str in idtable; 296 id = str in idtable;
297 } 297 }
298 assert(id); 298 assert(id);
299 t.type = id.type; 299 t.type = id.type;
300 if (t.type == TOK.Identifier) 300 if (t.type == TOK.Identifier || t.isKeyword)
301 return; 301 return;
302 if (t.type == TOK.EOF) 302 else if (t.isSpecialToken)
303 { 303 finalizeSpecialToken(t);
304 t.type = TOK.EOF; 304 else if (t.type == TOK.EOF)
305 t.end = p; 305 {
306 tail = &t; 306 tail = &t;
307 assert(t.srcText == "__EOF__"); 307 assert(t.srcText == "__EOF__");
308 } 308 }
309 else if (t.isSpecialToken) 309 else
310 finalizeSpecialToken(t); 310 assert(0, "unexpected token: " ~ t.srcText);
311 return; 311 return;
312 } 312 }
313 313
314 if (isdigit(c)) 314 if (isdigit(c))
315 return scanNumber(t); 315 return scanNumber(t);
608 return scanSpecialTokenSequence(t); 608 return scanSpecialTokenSequence(t);
609 default: 609 default:
610 } 610 }
611 611
612 // Check for EOF 612 // Check for EOF
613 if (c == 0 || c == _Z_) 613 if (isEOF(c))
614 { 614 {
615 assert(*p == 0 || *p == _Z_); 615 assert(isEOF(*p), ""~*p);
616 t.type = TOK.EOF; 616 t.type = TOK.EOF;
617 t.end = p; 617 t.end = p;
618 tail = &t; 618 tail = &t;
619 assert(t.start == t.end); 619 assert(t.start == t.end);
620 return; 620 return;
1047 idtable[str] = Identifier(TOK.Identifier, str); 1047 idtable[str] = Identifier(TOK.Identifier, str);
1048 id = str in idtable; 1048 id = str in idtable;
1049 } 1049 }
1050 assert(id); 1050 assert(id);
1051 t.type = id.type; 1051 t.type = id.type;
1052 if (t.type == TOK.Identifier) 1052 if (t.type == TOK.Identifier || t.isKeyword)
1053 return; 1053 return;
1054 if (t.type == TOK.EOF) 1054 else if (t.isSpecialToken)
1055 { 1055 finalizeSpecialToken(t);
1056 t.type = TOK.EOF; 1056 else if (t.type == TOK.EOF)
1057 t.end = p; 1057 {
1058 tail = &t; 1058 tail = &t;
1059 assert(t.srcText == "__EOF__"); 1059 assert(t.srcText == "__EOF__");
1060 } 1060 }
1061 else if (t.isSpecialToken) 1061 else
1062 finalizeSpecialToken(t); 1062 assert(0, "unexpected token: " ~ t.srcText);
1063 return; 1063 return;
1064 } 1064 }
1065 1065
1066 if (isdigit(c)) 1066 if (isdigit(c))
1067 return scanNumber(t); 1067 return scanNumber(t);
1068 1068
1069 // Check for EOF 1069 // Check for EOF
1070 if (c == 0 || c == _Z_) 1070 if (isEOF(c))
1071 { 1071 {
1072 assert(*p == 0 || *p == _Z_, *p~""); 1072 assert(isEOF(*p), *p~"");
1073 t.type = TOK.EOF; 1073 t.type = TOK.EOF;
1074 t.end = p; 1074 t.end = p;
1075 tail = &t; 1075 tail = &t;
1076 assert(t.start == t.end); 1076 assert(t.start == t.end);
1077 return; 1077 return;