Mercurial > projects > dil

comparison trunk/src/Lexer.d @ 67:996065105910

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem. - Decoding utf-8 characters in scanHexStringLiteral().
author aziz
date Sun, 01 Jul 2007 10:58:03 +0000
parents 8e84db78ad55
children 7eb83dd38901
comparison
equal deleted inserted replaced
66:8e84db78ad55 67:996065105910
9 import Messages; 9 import Messages;
10 import std.stdio; 10 import std.stdio;
11 import std.utf; 11 import std.utf;
12 import std.uni; 12 import std.uni;
13 import std.c.stdlib; 13 import std.c.stdlib;
14 import std.c.stdarg; 14 import std.stdarg;
15 import std.string; 15 import std.string;
16 16
17 const char[3] LS = \u2028; 17 const char[3] LS = \u2028;
18 const char[3] PS = \u2029; 18 const char[3] PS = \u2029;
19 19
32 } 32 }
33 33
34 MID id; 34 MID id;
35 Type type; 35 Type type;
36 uint loc; 36 uint loc;
37 TypeInfo[] tinfos; 37 string[] arguments;
38 void* argptr; 38 /*
39
40 this(Type type, MID id, uint loc) 39 this(Type type, MID id, uint loc)
41 { 40 {
42 this.id = id; 41 this.id = id;
43 this.type = type; 42 this.type = type;
44 this.loc = loc; 43 this.loc = loc;
45 } 44 }
46 45 */
47 this(Type type, MID id, uint loc, TypeInfo[] ti, void* argptr) 46 this(Type type, MID id, uint loc, string[] arguments)
48 { 47 {
49 this(type, id, loc); 48 this.id = id;
50 this.tinfos = ti; 49 this.type = type;
51 this.argptr = argptr; 50 this.loc = loc;
51 this.arguments = arguments;
52 } 52 }
53 53
54 string getMsg() 54 string getMsg()
55 { 55 {
56 char[] msg = messages[id]; 56 char[] msg = messages[id];
57 57
58 if (tinfos.length == 0) 58 if (arguments.length == 0)
59 return msg; 59 return msg;
60 60
61 foreach (i, arg; arguments) 61 foreach (i, arg; arguments)
62 msg = replace(msg, format("{%s}", i), arg); 62 msg = replace(msg, format("{%s}", i+1), arg);
63 } 63
64 64 return msg;
65 private char[][] arguments()
66 {
67 char[][] args;
68 void* argptr = this.argptr;
69 foreach (ti; tinfos)
70 {
71 if (ti == typeid(char[]))
72 args ~= format(va_arg!(char[])(argptr));
73 else if (ti == typeid(int))
74 args ~= format(va_arg!(int)(argptr));
75 else
76 assert(0, "argument type not supported yet.");
77 }
78 return args;
79 } 65 }
80 } 66 }
81 67
82 class Lexer 68 class Lexer
83 { 69 {
655 continue; 641 continue;
656 } 642 }
657 buffer ~= *p++; 643 buffer ~= *p++;
658 } 644 }
659 } 645 }
646 assert(0);
660 } 647 }
661 648
662 void scanCharacterLiteral(ref Token t) 649 void scanCharacterLiteral(ref Token t)
663 { 650 {
664 assert(*p == '\''); 651 assert(*p == '\'');
814 ++n; 801 ++n;
815 continue; 802 continue;
816 } 803 }
817 else if (isspace(c)) 804 else if (isspace(c))
818 continue; 805 continue;
819 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) 806
820 { 807 if (c >= 128)
821 ++p; ++p; 808 {
822 ++loc; 809 c = decodeUTF8();
823 continue; 810 if (c == LSd || c == PSd)
811 {
812 ++p; ++p;
813 ++loc;
814 continue;
815 }
824 } 816 }
825 else if (c == 0 || c == _Z_) 817 else if (c == 0 || c == _Z_)
826 { 818 {
827 error(MID.UnterminatedHexString); 819 error(MID.UnterminatedHexString);
828 t.pf = 0; 820 t.pf = 0;
829 goto Lreturn; 821 goto Lreturn;
830 } 822 }
831 error(MID.NonHexCharInHexString); 823 error(MID.NonHexCharInHexString, cast(dchar)c);
832 } 824 }
833 } 825 }
834 assert(0); 826 assert(0);
835 } 827 }
836 828
1211 } 1203 }
1212 1204
1213 /* 1205 /*
1214 FloatLiteral:= Float[fFL]?i? 1206 FloatLiteral:= Float[fFL]?i?
1215 Float:= DecFloat | HexFloat 1207 Float:= DecFloat | HexFloat
1216 DecFloat:= ([0-9][0-9_]*[.]([0-9_]*DecExponent?)?) | [.][0-9][0-9_]*DecExponent? | [0-9][0-9_]*DecExponent 1208 DecFloat:= ([0-9][0-9_]*[.][0-9_]*DecExponent?) | [.][0-9][0-9_]*DecExponent? | [0-9][0-9_]*DecExponent
1217 DecExponent:= [eE][+-]?[0-9][0-9_]* 1209 DecExponent:= [eE][+-]?[0-9][0-9_]*
1218 HexFloat:= 0[xX](HexDigits[.]HexDigits | [.][0-9a-zA-Z]HexDigits? | HexDigits)HexExponent 1210 HexFloat:= 0[xX](HexDigits[.]HexDigits | [.][0-9a-zA-Z]HexDigits? | HexDigits)HexExponent
1219 HexExponent:= [pP][+-]?[0-9][0-9_]* 1211 HexExponent:= [pP][+-]?[0-9][0-9_]*
1220 */ 1212 */
1221 void scanReal(ref Token t) 1213 void scanReal(ref Token t)
1444 return; 1436 return;
1445 Lerr: 1437 Lerr:
1446 error(mid); 1438 error(mid);
1447 } 1439 }
1448 1440
1449 uint decodeUTF8() 1441 dchar decodeUTF8()
1450 { 1442 {
1451 assert(*p & 128, "check for ASCII char before calling decodeUTF8()."); 1443 assert(*p & 128, "check for ASCII char before calling decodeUTF8().");
1452 size_t idx; 1444 size_t idx;
1453 uint d = 0xFFFF; 1445 uint d = 0xFFFF;
1454 try 1446 try
1471 idtable[k.str] = k; 1463 idtable[k.str] = k;
1472 } 1464 }
1473 1465
1474 void error(MID id, ...) 1466 void error(MID id, ...)
1475 { 1467 {
1476 errors ~= new Problem(Problem.Type.Lexer, id, loc, _arguments, _argptr); 1468 char[][] arguments(TypeInfo[] tinfos, void* argptr)
1469 {
1470 char[][] args;
1471 foreach (ti; tinfos)
1472 {
1473 if (ti == typeid(char[]))
1474 args ~= format(va_arg!(char[])(argptr));
1475 else if (ti == typeid(int))
1476 args ~= format(va_arg!(int)(argptr));
1477 else if (ti == typeid(dchar))
1478 args ~= format(va_arg!(dchar)(argptr));
1479 else
1480 assert(0, "argument type not supported yet.");
1481 }
1482 return args;
1483 }
1484
1485 errors ~= new Problem(Problem.Type.Lexer, id, loc, arguments(_arguments, _argptr));
1477 } 1486 }
1478 1487
1479 public TOK nextToken() 1488 public TOK nextToken()
1480 { 1489 {
1481 scan(this.token); 1490 scan(this.token);
1563 1572
1564 assert(tokens.length == toks.length ); 1573 assert(tokens.length == toks.length );
1565 1574
1566 foreach (i, t; tokens) 1575 foreach (i, t; tokens)
1567 assert(t.span == toks[i], std.string.format("Lexed '%s' but expected '%s'", t.span, toks[i])); 1576 assert(t.span == toks[i], std.string.format("Lexed '%s' but expected '%s'", t.span, toks[i]));
1577 }
1578
1579 unittest
1580 {
1581 // Numbers unittest
1582 // 0L 0ULi 0_L 0_UL 0_Fi 0_e2 0_F 0_i 0x0U 0x0p2
1568 } 1583 }
1569 1584
1570 /// ASCII character properties table. 1585 /// ASCII character properties table.
1571 static const int ptable[256] = [ 1586 static const int ptable[256] = [
1572 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0, 1587 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0,