Mercurial > projects > dil
annotate trunk/src/Lexer.d @ 66:8e84db78ad55
- Added support for variadic arguments in error messages.
author | aziz |
---|---|
date | Sat, 30 Jun 2007 11:27:01 +0000 |
parents | 6c21ae79fbb3 |
children | 996065105910 |
rev | line source |
---|---|
0 | 1 /++ |
2 Author: Aziz Köksal | |
3 License: GPL2 | |
4 +/ | |
5 module Lexer; | |
3 | 6 import Token; |
28 | 7 import Keywords; |
8 import Identifier; | |
54 | 9 import Messages; |
2 | 10 import std.stdio; |
4 | 11 import std.utf; |
12 import std.uni; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
13 import std.c.stdlib; |
66
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
14 import std.c.stdarg; |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
15 import std.string; |
0 | 16 |
11 | 17 const char[3] LS = \u2028; |
18 const char[3] PS = \u2029; | |
19 | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
20 const dchar LSd = 0x2028; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
21 const dchar PSd = 0x2029; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
22 |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
23 const uint _Z_ = 26; /// Control+Z |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
24 |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
25 class Problem |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
26 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
27 enum Type |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
28 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
29 Lexer, |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
30 Parser, |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
31 Semantic |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
32 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
33 |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
34 MID id; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
35 Type type; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
36 uint loc; |
66
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
37 TypeInfo[] tinfos; |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
38 void* argptr; |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
39 |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
40 this(Type type, MID id, uint loc) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
41 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
42 this.id = id; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
43 this.type = type; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
44 this.loc = loc; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
45 } |
66
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
46 |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
47 this(Type type, MID id, uint loc, TypeInfo[] ti, void* argptr) |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
48 { |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
49 this(type, id, loc); |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
50 this.tinfos = ti; |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
51 this.argptr = argptr; |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
52 } |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
53 |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
54 string getMsg() |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
55 { |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
56 char[] msg = messages[id]; |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
57 |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
58 if (tinfos.length == 0) |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
59 return msg; |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
60 |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
61 foreach (i, arg; arguments) |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
62 msg = replace(msg, format("{%s}", i), arg); |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
63 } |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
64 |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
65 private char[][] arguments() |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
66 { |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
67 char[][] args; |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
68 void* argptr = this.argptr; |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
69 foreach (ti; tinfos) |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
70 { |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
71 if (ti == typeid(char[])) |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
72 args ~= format(va_arg!(char[])(argptr)); |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
73 else if (ti == typeid(int)) |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
74 args ~= format(va_arg!(int)(argptr)); |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
75 else |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
76 assert(0, "argument type not supported yet."); |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
77 } |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
78 return args; |
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
79 } |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
80 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
81 |
0 | 82 class Lexer |
83 { | |
4 | 84 Token token; |
30 | 85 string text; |
4 | 86 char* p; |
87 char* end; | |
88 | |
17 | 89 uint loc = 1; /// line of code |
90 | |
30 | 91 char[] fileName; |
92 | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
93 Problem[] errors; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
94 |
28 | 95 Identifier[string] idtable; |
96 | |
30 | 97 this(string text, string fileName) |
4 | 98 { |
30 | 99 this.fileName = fileName; |
100 | |
4 | 101 this.text = text; |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
102 if (text[$-1] != 0) |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
103 { |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
104 this.text.length = this.text.length + 1; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
105 this.text[$-1] = 0; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
106 } |
4 | 107 |
108 this.p = this.text.ptr; | |
109 this.end = this.p + this.text.length; | |
28 | 110 |
111 loadKeywords(); | |
4 | 112 } |
113 | |
114 public void scan(out Token t) | |
3 | 115 { |
4 | 116 assert(p < end); |
0 | 117 |
10 | 118 uint c = *p; |
4 | 119 |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
120 while (1) |
4 | 121 { |
5 | 122 t.start = p; |
16 | 123 |
5 | 124 if (c == 0) |
125 { | |
51 | 126 assert(*p == 0); |
17 | 127 ++p; |
51 | 128 assert(p == end); |
5 | 129 t.type = TOK.EOF; |
17 | 130 t.end = p; |
5 | 131 return; |
132 } | |
4 | 133 |
17 | 134 if (c == '\n') |
135 { | |
136 c = *++p; | |
137 ++loc; | |
138 continue; | |
139 } | |
140 else if (c == '\r') | |
141 { | |
142 c = *++p; | |
143 if (c != '\n') | |
144 ++loc; | |
145 continue; | |
146 } | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
147 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
148 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
149 p += 3; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
150 c = *p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
151 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
152 } |
17 | 153 |
13 | 154 if (isidbeg(c)) |
5 | 155 { |
33 | 156 if (c == 'r' && p[1] == '"' && ++p) |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
157 return scanRawStringLiteral(t); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
158 if (c == 'x' && p[1] == '"') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
159 return scanHexStringLiteral(t); |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
160 Lidentifier: |
5 | 161 do |
162 { c = *++p; } | |
49 | 163 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8())) |
28 | 164 |
5 | 165 t.end = p; |
28 | 166 |
65 | 167 string str = t.srcText; |
28 | 168 Identifier* id = str in idtable; |
169 | |
170 if (!id) | |
171 { | |
172 idtable[str] = Identifier.Identifier(TOK.Identifier, str); | |
173 id = str in idtable; | |
174 } | |
175 assert(id); | |
176 t.type = id.type; | |
5 | 177 return; |
178 } | |
179 | |
15 | 180 if (isdigit(c)) |
181 return scanNumber(t); | |
182 | |
8 | 183 if (c == '/') |
5 | 184 { |
8 | 185 c = *++p; |
14 | 186 switch(c) |
5 | 187 { |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
188 case '=': |
14 | 189 ++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
190 t.type = TOK.DivAssign; |
14 | 191 t.end = p; |
192 return; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
193 case '+': |
8 | 194 uint level = 1; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
195 while (1) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
196 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
197 c = *++p; |
42 | 198 LswitchNC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
199 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
200 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
201 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
202 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
203 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
204 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
205 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
206 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
207 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
208 error(MID.UnterminatedNestedComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
209 goto LreturnNC; |
42 | 210 default: |
211 } | |
212 | |
213 c <<= 8; | |
214 c |= *++p; | |
215 switch (c) | |
216 { | |
217 case 0x2F2B: // /+ | |
218 ++level; | |
219 continue; | |
220 case 0x2B2F: // +/ | |
221 if (--level == 0) | |
222 { | |
223 ++p; | |
224 LreturnNC: | |
225 t.type = TOK.Comment; | |
226 t.end = p; | |
227 return; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
228 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
229 continue; |
42 | 230 case 0xE280: // LS[0..1] || PS[0..1] |
231 if (p[1] == LS[2] || p[1] == PS[2]) | |
232 { | |
233 ++loc; | |
234 ++p; | |
235 } | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
236 continue; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
237 default: |
42 | 238 c &= char.max; |
239 goto LswitchNC; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
240 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
241 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
242 case '*': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
243 while (1) |
7 | 244 { |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
245 c = *++p; |
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
246 LswitchBC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
247 switch (c) |
8 | 248 { |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
249 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
250 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
251 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
252 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
253 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
254 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
255 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
256 error(MID.UnterminatedBlockComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
257 goto LreturnBC; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
258 default: |
8 | 259 } |
41 | 260 |
261 c <<= 8; | |
262 c |= *++p; | |
263 switch (c) | |
264 { | |
265 case 0x2A2F: // */ | |
266 ++p; | |
267 LreturnBC: | |
268 t.type = TOK.Comment; | |
269 t.end = p; | |
270 return; | |
271 case 0xE280: // LS[0..1] || PS[0..1] | |
272 if (p[1] == LS[2] || p[1] == PS[2]) | |
273 { | |
274 ++loc; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
275 ++p; |
41 | 276 } |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
277 continue; |
41 | 278 default: |
279 c &= char.max; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
280 goto LswitchBC; |
41 | 281 } |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
282 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
283 assert(0); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
284 case '/': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
285 while (1) |
7 | 286 { |
8 | 287 c = *++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
288 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
289 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
290 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
291 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
292 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
293 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
294 case 0, _Z_: |
11 | 295 break; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
296 case LS[0]: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
297 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
298 break; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
299 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
300 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
301 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
302 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
303 t.type = TOK.Comment; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
304 t.end = p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
305 return; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
306 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
307 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
308 t.type = TOK.Div; |
10 | 309 t.end = p; |
310 return; | |
311 } | |
5 | 312 } |
313 | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
314 switch (c) |
20 | 315 { |
48 | 316 case '\'': |
317 return scanCharacterLiteral(t); | |
318 case '`': | |
319 return scanRawStringLiteral(t); | |
320 case '"': | |
321 return scanNormalStringLiteral(t); | |
322 case '\\': | |
323 char[] buffer; | |
324 do | |
325 { | |
326 ++p; | |
327 c = scanEscapeSequence(); | |
50 | 328 if (c == 0xFFFF) |
329 break; | |
48 | 330 if (c < 128) |
331 buffer ~= c; | |
332 else | |
333 encodeUTF8(buffer, c); | |
334 } while (*p == '\\') | |
335 buffer ~= 0; | |
336 t.type = TOK.String; | |
337 t.str = buffer; | |
338 t.end = p; | |
339 return; | |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
340 case '>': /* > >= >> >>= >>> >>>= */ |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
341 c = *++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
342 switch (c) |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
343 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
344 case '=': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
345 t.type = TOK.GreaterEqual; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
346 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
347 case '>': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
348 if (p[1] == '>') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
349 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
350 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
351 if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
352 { ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
353 t.type = TOK.URShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
354 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
355 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
356 t.type = TOK.URShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
357 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
358 else if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
359 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
360 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
361 t.type = TOK.RShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
362 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
363 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
364 t.type = TOK.RShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
365 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
366 default: |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
367 t.type = TOK.Greater; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
368 goto Lcommon2; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
369 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
370 assert(0); |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
371 case '<': /* < <= <> <>= << <<= */ |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
372 c = *++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
373 switch (c) |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
374 { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
375 case '=': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
376 t.type = TOK.LessEqual; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
377 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
378 case '<': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
379 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
380 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
381 t.type = TOK.LShiftAssign; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
382 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
383 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
384 t.type = TOK.LShift; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
385 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
386 case '>': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
387 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
388 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
389 t.type = TOK.LorEorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
390 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
391 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
392 t.type = TOK.LorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
393 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
394 default: |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
395 t.type = TOK.Less; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
396 goto Lcommon2; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
397 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
398 assert(0); |
37
7f3bcb97d017
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
36
diff
changeset
|
399 case '!': /* ! !< !> !<= !>= !<> !<>= */ |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
400 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
401 switch (c) |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
402 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
403 case '<': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
404 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
405 if (c == '>') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
406 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
407 if (p[1] == '=') { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
408 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
409 t.type = TOK.Unordered; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
410 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
411 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
412 t.type = TOK.UorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
413 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
414 else if (c == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
415 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
416 t.type = TOK.UorG; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
417 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
418 else { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
419 t.type = TOK.UorGorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
420 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
421 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
422 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
423 case '>': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
424 if (p[1] == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
425 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
426 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
427 t.type = TOK.UorL; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
428 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
429 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
430 t.type = TOK.UorLorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
431 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
432 case '=': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
433 t.type = TOK.NotEqual; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
434 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
435 default: |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
436 t.type = TOK.Not; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
437 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
438 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
439 assert(0); |
63 | 440 case '.': /* . .[0-9] .. ... */ |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
441 if (p[1] == '.') |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
442 { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
443 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
444 if (p[1] == '.') { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
445 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
446 t.type = TOK.Ellipses; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
447 } |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
448 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
449 t.type = TOK.Slice; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
450 } |
63 | 451 else if (isdigit(p[1])) |
452 { | |
453 return scanReal(t); | |
454 } | |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
455 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
456 t.type = TOK.Dot; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
457 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
458 case '|': /* | || |= */ |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
459 c = *++p; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
460 if (c == '=') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
461 t.type = TOK.OrAssign; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
462 else if (c == '|') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
463 t.type = TOK.OrLogical; |
26 | 464 else { |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
465 t.type = TOK.OrBinary; |
26 | 466 goto Lcommon2; |
467 } | |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
468 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
469 case '&': /* & && &= */ |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
470 c = *++p; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
471 if (c == '=') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
472 t.type = TOK.AndAssign; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
473 else if (c == '&') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
474 t.type = TOK.AndLogical; |
26 | 475 else { |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
476 t.type = TOK.AndBinary; |
26 | 477 goto Lcommon2; |
478 } | |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
479 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
480 case '+': /* + ++ += */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
481 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
482 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
483 t.type = TOK.PlusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
484 else if (c == '+') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
485 t.type = TOK.PlusPlus; |
26 | 486 else { |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
487 t.type = TOK.Plus; |
26 | 488 goto Lcommon2; |
489 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
490 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
491 case '-': /* - -- -= */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
492 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
493 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
494 t.type = TOK.MinusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
495 else if (c == '-') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
496 t.type = TOK.MinusMinus; |
26 | 497 else { |
498 t.type = TOK.Minus; | |
499 goto Lcommon2; | |
500 } | |
501 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
502 case '=': /* = == */ |
26 | 503 if (p[1] == '=') { |
504 ++p; | |
505 t.type = TOK.Equal; | |
506 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
507 else |
26 | 508 t.type = TOK.Assign; |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
509 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
510 case '~': /* ~ ~= */ |
27 | 511 if (p[1] == '=') { |
512 ++p; | |
513 t.type = TOK.CatAssign; | |
514 } | |
515 else | |
516 t.type = TOK.Tilde; | |
517 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
518 case '*': /* * *= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
519 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
520 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
521 t.type = TOK.MulAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
522 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
523 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
524 t.type = TOK.Mul; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
525 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
526 case '^': /* ^ ^= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
527 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
528 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
529 t.type = TOK.XorAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
530 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
531 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
532 t.type = TOK.Xor; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
533 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
534 case '%': /* % %= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
535 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
536 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
537 t.type = TOK.ModAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
538 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
539 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
540 t.type = TOK.Mod; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
541 goto Lcommon; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
542 // Single character tokens: |
20 | 543 case '(': |
544 t.type = TOK.LParen; | |
545 goto Lcommon; | |
546 case ')': | |
547 t.type = TOK.RParen; | |
548 goto Lcommon; | |
549 case '[': | |
550 t.type = TOK.LBracket; | |
551 goto Lcommon; | |
552 case ']': | |
553 t.type = TOK.RBracket; | |
554 goto Lcommon; | |
555 case '{': | |
556 t.type = TOK.LBrace; | |
557 goto Lcommon; | |
558 case '}': | |
559 t.type = TOK.RBrace; | |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
560 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
561 case ':': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
562 t.type = TOK.Colon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
563 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
564 case ';': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
565 t.type = TOK.Semicolon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
566 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
567 case '?': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
568 t.type = TOK.Question; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
569 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
570 case ',': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
571 t.type = TOK.Comma; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
572 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
573 case '$': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
574 t.type = TOK.Dollar; |
20 | 575 Lcommon: |
576 ++p; | |
26 | 577 Lcommon2: |
20 | 578 t.end = p; |
579 return; | |
30 | 580 case '#': |
581 scanSpecialToken(); | |
51 | 582 c = *p; |
583 continue; | |
20 | 584 default: |
585 } | |
586 | |
49 | 587 if (c & 128 && isUniAlpha(decodeUTF8())) |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
588 goto Lidentifier; |
5 | 589 c = *++p; |
4 | 590 } |
3 | 591 } |
4 | 592 |
30 | 593 void peek(ref Token t) |
594 { | |
595 char* tmp = p; | |
51 | 596 uint len = errors.length; |
30 | 597 scan(t); |
598 p = tmp; | |
51 | 599 if (errors.length != len) |
600 errors = errors[0..len]; | |
30 | 601 } |
602 | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
603 void scanNormalStringLiteral(ref Token t) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
604 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
605 assert(*p == '"'); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
606 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
607 char[] buffer; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
608 t.type = TOK.String; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
609 while (1) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
610 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
611 switch (*p) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
612 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
613 case '"': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
614 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
615 Lreturn: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
616 buffer ~= 0; |
49 | 617 t.str = buffer; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
618 t.pf = scanPostfix(); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
619 t.end = p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
620 return; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
621 case '\\': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
622 ++p; |
49 | 623 dchar d = scanEscapeSequence(); |
50 | 624 if (d == 0xFFFF) |
625 continue; | |
49 | 626 if (d < 128) |
627 buffer ~= d; | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
628 else |
49 | 629 encodeUTF8(buffer, d); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
630 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
631 case '\r': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
632 if (p[1] == '\n') |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
633 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
634 case '\n': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
635 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
636 ++loc; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
637 buffer ~= '\n'; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
638 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
639 case 0, _Z_: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
640 error(MID.UnterminatedString); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
641 goto Lreturn; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
642 default: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
643 if (*p & 128) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
644 { |
49 | 645 char* begin = p; |
646 dchar d = decodeUTF8(); | |
647 if (d == LSd || d == PSd) | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
648 goto case '\n'; |
49 | 649 |
650 if (d != 0xFFFF) | |
651 { | |
652 ++p; | |
653 buffer ~= begin[0 .. p - begin]; | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
654 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
655 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
656 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
657 buffer ~= *p++; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
658 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
659 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
660 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
661 |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
662 void scanCharacterLiteral(ref Token t) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
663 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
664 assert(*p == '\''); |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
665 MID id = MID.UnterminatedCharacterLiteral; |
46 | 666 ++p; |
667 switch (*p) | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
668 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
669 case '\\': |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
670 ++p; |
46 | 671 t.dchar_ = scanEscapeSequence(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
672 break; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
673 case '\'': |
46 | 674 ++p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
675 id = MID.EmptyCharacterLiteral; |
46 | 676 case '\n', '\r', 0, _Z_: |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
677 goto Lerr; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
678 default: |
46 | 679 uint c = *p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
680 if (c & 128) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
681 { |
49 | 682 c = decodeUTF8(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
683 if (c == LSd || c == PSd) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
684 goto Lerr; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
685 } |
46 | 686 t.dchar_ = c; |
687 ++p; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
688 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
689 |
46 | 690 if (*p == '\'') |
691 ++p; | |
692 else | |
693 Lerr: | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
694 error(id); |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
695 t.type = TOK.Character; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
696 t.end = p; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
697 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
698 |
33 | 699 char scanPostfix() |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
700 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
701 switch (*p) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
702 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
703 case 'c': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
704 case 'w': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
705 case 'd': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
706 return *p++; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
707 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
708 return 0; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
709 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
710 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
711 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
712 void scanRawStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
713 { |
33 | 714 uint delim = *p; |
715 assert(delim == '`' || delim == '"' && p[-1] == 'r'); | |
716 t.type = TOK.String; | |
717 char[] buffer; | |
718 uint c; | |
719 while (1) | |
720 { | |
721 c = *++p; | |
722 switch (c) | |
723 { | |
724 case '\r': | |
725 if (p[1] == '\n') | |
726 ++p; | |
727 c = '\n'; // Convert '\r' and '\r\n' to '\n' | |
728 case '\n': | |
729 ++loc; | |
52
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
730 break; |
33 | 731 case '`': |
732 case '"': | |
733 if (c == delim) | |
734 { | |
735 ++p; | |
736 t.pf = scanPostfix(); | |
737 Lreturn: | |
738 t.str = buffer ~ '\0'; | |
739 t.end = p; | |
740 return; | |
741 } | |
742 break; | |
743 case LS[0]: | |
744 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) | |
745 { | |
52
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
746 c = '\n'; |
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
747 ++p; ++p; |
33 | 748 ++loc; |
749 } | |
750 break; | |
751 case 0, _Z_: | |
752 if (delim == 'r') | |
753 error(MID.UnterminatedRawString); | |
754 else | |
755 error(MID.UnterminatedBackQuoteString); | |
756 goto Lreturn; | |
757 default: | |
758 } | |
759 buffer ~= c; // copy character to buffer | |
760 } | |
761 assert(0); | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
762 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
763 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
764 void scanHexStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
765 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
766 assert(p[0] == 'x' && p[1] == '"'); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
767 t.type = TOK.String; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
768 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
769 uint c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
770 ubyte[] buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
771 ubyte h; // hex number |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
772 uint n; // number of hex digits |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
773 |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
774 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
775 while (1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
776 { |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
777 c = *++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
778 switch (c) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
779 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
780 case '"': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
781 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
782 if (n & 1) |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
783 error(MID.OddNumberOfDigitsInHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
784 t.pf = scanPostfix(); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
785 Lreturn: |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
786 buffer ~= 0; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
787 t.str = cast(string) buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
788 t.end = p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
789 return; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
790 case '\r': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
791 if (p[1] == '\n') |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
792 ++p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
793 case '\n': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
794 ++loc; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
795 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
796 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
797 if (ishexad(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
798 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
799 if (c <= '9') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
800 c -= '0'; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
801 else if (c <= 'F') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
802 c -= 'A' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
803 else |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
804 c -= 'a' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
805 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
806 if (n & 1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
807 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
808 h <<= 4; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
809 h |= c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
810 buffer ~= h; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
811 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
812 else |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
813 h = c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
814 ++n; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
815 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
816 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
817 else if (isspace(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
818 continue; |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
819 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
820 { |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
821 ++p; ++p; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
822 ++loc; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
823 continue; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
824 } |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
825 else if (c == 0 || c == _Z_) |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
826 { |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
827 error(MID.UnterminatedHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
828 t.pf = 0; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
829 goto Lreturn; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
830 } |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
831 error(MID.NonHexCharInHexString); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
832 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
833 } |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
834 assert(0); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
835 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
836 |
45 | 837 dchar scanEscapeSequence() |
838 { | |
839 uint c = char2ev(*p); | |
46 | 840 if (c) { |
841 ++p; | |
45 | 842 return c; |
46 | 843 } |
50 | 844 c = 0xFFFF; |
45 | 845 uint digits = 2; |
846 | |
847 switch (*p) | |
848 { | |
849 case 'x': | |
850 c = 0; | |
851 while (1) | |
852 { | |
853 ++p; | |
854 if (ishexad(*p)) | |
855 { | |
856 c *= 16; | |
857 if (*p <= '9') | |
48 | 858 c += *p - '0'; |
45 | 859 else if (*p <= 'F') |
48 | 860 c += *p - 'A' + 10; |
45 | 861 else |
48 | 862 c += *p - 'a' + 10; |
863 if (!--digits) { | |
864 ++p; | |
45 | 865 break; |
48 | 866 } |
45 | 867 } |
868 else | |
869 { | |
870 error(MID.InsufficientHexDigits); | |
50 | 871 c = 0xFFFF; |
45 | 872 break; |
873 } | |
874 } | |
875 break; | |
876 case 'u': | |
877 digits = 4; | |
878 goto case 'x'; | |
879 case 'U': | |
880 digits = 8; | |
881 goto case 'x'; | |
882 default: | |
48 | 883 if (isoctal(*p)) |
45 | 884 { |
48 | 885 c = 0; |
886 c += *p - '0'; | |
887 ++p; | |
888 if (!isoctal(*p)) | |
889 return c; | |
890 c *= 8; | |
891 c += *p - '0'; | |
892 ++p; | |
893 if (!isoctal(*p)) | |
894 return c; | |
895 c *= 8; | |
896 c += *p - '0'; | |
897 ++p; | |
898 } | |
899 else if(*p == '&') | |
900 { | |
901 if (isalpha(*++p)) | |
45 | 902 { |
48 | 903 while (1) |
904 { | |
905 if (isalnum(*++p)) | |
906 continue; | |
907 if (*p == ';') { | |
908 // TODO: convert entity to unicode codepoint. | |
909 ++p; | |
910 break; | |
911 } | |
912 else { | |
913 error(MID.UnterminatedHTMLEntity); | |
914 break; | |
915 } | |
45 | 916 } |
917 } | |
48 | 918 else |
919 error(MID.InvalidBeginHTMLEntity); | |
45 | 920 } |
921 else | |
48 | 922 error(MID.UndefinedEscapeSequence); |
45 | 923 } |
924 | |
925 return c; | |
926 } | |
927 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
928 /* |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
929 IntegerLiteral:= (Dec|Hex|Bin|Oct)Suffix? |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
930 Dec:= (0|[1-9][0-9_]*) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
931 Hex:= 0[xX] HexDigits |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
932 Bin:= 0[bB][01_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
933 Oct:= 0[0-7_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
934 Suffix:= (L|[uU]|L[uU]|[uU]L)? |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
935 HexDigits:= [0-9a-zA-Z_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
936 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
937 Invalid: "0b_", "0x_", "._" |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
938 */ |
15 | 939 void scanNumber(ref Token t) |
940 { | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
941 ulong ulong_; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
942 bool overflow; |
58 | 943 bool isDecimal; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
944 size_t digits; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
945 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
946 if (*p != '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
947 goto LscanInteger; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
948 ++p; // skip zero |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
949 // check for xX bB ... |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
950 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
951 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
952 case 'x','X': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
953 goto LscanHex; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
954 case 'b','B': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
955 goto LscanBin; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
956 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
957 if (p[1] == 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
958 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
959 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
960 if (p[1] == '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
961 break; |
58 | 962 case 'i','f','F', 'e', 'E': // Imaginary and float literal suffix |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
963 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
964 case '_': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
965 ++p; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
966 goto LscanOct; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
967 default: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
968 if (isoctal(*p)) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
969 goto LscanOct; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
970 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
971 |
58 | 972 ulong_ = p[-1]; |
973 isDecimal = true; | |
974 goto Lfinalize; | |
975 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
976 LscanInteger: |
58 | 977 isDecimal = true; |
63 | 978 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
979 { |
63 | 980 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
981 continue; |
63 | 982 if (!isdigit(*p)) |
983 break; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
984 if (ulong_ < ulong.max/10 || (ulong_ == ulong.max/10 && *p <= '5')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
985 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
986 ulong_ *= 10; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
987 ulong_ += *p - '0'; |
58 | 988 continue; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
989 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
990 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
991 overflow = true; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
992 while (isdigit(*++p)) {} |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
993 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
994 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
995 |
61 | 996 // The number could be a float, so check overflow below. |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
997 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
998 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
999 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1000 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1001 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1002 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1003 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1004 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1005 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1006 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1007 goto LscanReal; |
58 | 1008 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1009 } |
58 | 1010 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1011 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1012 error(MID.OverflowDecimalNumber); |
58 | 1013 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1014 assert((isdigit(p[-1]) || p[-1] == '_') && !isdigit(*p) && *p != '_'); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1015 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1016 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1017 LscanHex: |
61 | 1018 assert(digits == 0); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1019 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1020 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1021 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1022 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1023 if (!ishexad(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1024 break; |
61 | 1025 ++digits; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1026 ulong_ *= 16; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1027 if (*p <= '9') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1028 ulong_ += *p - '0'; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1029 else if (*p <= 'F') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1030 ulong_ += *p - 'A' + 10; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1031 else |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1032 ulong_ += *p - 'a' + 10; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1033 } |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1034 |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1035 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1036 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1037 case '.': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1038 if (p[1] != '.') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1039 goto LscanHexReal; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1040 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1041 case 'L': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1042 if (p[1] != 'i') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1043 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1044 case 'i', 'p', 'P': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1045 goto LscanHexReal; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1046 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1047 } |
61 | 1048 if (digits == 0) |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1049 error(MID.NoDigitsInHexNumber); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1050 else if (digits > 16) |
61 | 1051 { |
1052 // Overflow: skip following digits. | |
1053 error(MID.OverflowHexNumber); | |
1054 while (ishexad(*++p)) {} | |
1055 } | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1056 goto Lfinalize; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1057 LscanHexReal: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1058 return scanHexReal(t); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1059 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1060 LscanBin: |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1061 assert(digits == 0); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1062 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1063 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1064 if (*++p == '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1065 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1066 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1067 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1068 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1069 if (*p == '1') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1070 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1071 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1072 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1073 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1074 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1075 if (*p == '_') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1076 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1077 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1078 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1079 |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1080 if (digits == 0) |
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1081 error(MID.NoDigitsInBinNumber); |
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1082 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1083 if (digits > 64) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1084 error(MID.OverflowBinaryNumber); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1085 assert((p[-1] == '0' || p[-1] == '1' || p[-1] == '_') && !(*p == '0' || *p == '1' || *p == '_')); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1086 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1087 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1088 LscanOct: |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1089 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1090 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1091 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1092 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1093 if (!isoctal(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1094 break; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1095 if (ulong_ < ulong.max/2 || (ulong_ == ulong.max/2 && *p <= '1')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1096 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1097 ulong_ *= 8; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1098 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1099 ++p; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1100 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1101 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1102 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1103 overflow = true; |
63 | 1104 while (isdigit(*++p)) {} |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1105 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1106 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1107 |
63 | 1108 bool hasDecimalDigits; |
1109 if (isdigit(*p)) | |
1110 { | |
1111 hasDecimalDigits = true; | |
1112 while (isdigit(*++p)) {} | |
1113 } | |
1114 | |
1115 // The number could be a float, so check errors below. | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1116 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1117 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1118 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1119 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1120 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1121 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1122 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1123 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1124 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1125 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1126 goto LscanReal; |
63 | 1127 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1128 } |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1129 |
63 | 1130 if (hasDecimalDigits) |
1131 error(MID.OctalNumberHasDecimals); | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1132 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1133 error(MID.OverflowOctalNumber); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1134 // goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1135 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1136 Lfinalize: |
58 | 1137 enum Suffix |
1138 { | |
1139 None = 0, | |
1140 Unsigned = 1, | |
60 | 1141 Long = 2 |
58 | 1142 } |
60 | 1143 |
58 | 1144 Suffix suffix; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1145 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1146 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1147 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1148 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1149 case 'L': |
58 | 1150 if (suffix & Suffix.Long) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1151 break; |
60 | 1152 suffix |= Suffix.Long; |
1153 ++p; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1154 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1155 case 'u', 'U': |
58 | 1156 if (suffix & Suffix.Unsigned) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1157 break; |
60 | 1158 suffix |= Suffix.Unsigned; |
1159 ++p; | |
1160 continue; | |
58 | 1161 default: |
1162 break; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1163 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1164 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1165 } |
58 | 1166 |
1167 switch (suffix) | |
1168 { | |
1169 case Suffix.None: | |
1170 if (ulong_ & 0x8000000000000000) | |
1171 { | |
1172 if (isDecimal) | |
1173 error(MID.OverflowDecimalSign); | |
1174 t.type = TOK.Uint64; | |
1175 } | |
1176 else if (ulong_ & 0xFFFFFFFF00000000) | |
1177 t.type = TOK.Int64; | |
1178 else if (ulong_ & 0x80000000) | |
1179 t.type = isDecimal ? TOK.Int64 : TOK.Uint32; | |
1180 else | |
1181 t.type = TOK.Int32; | |
1182 break; | |
1183 case Suffix.Unsigned: | |
1184 if (ulong_ & 0xFFFFFFFF00000000) | |
1185 t.type = TOK.Uint64; | |
1186 else | |
1187 t.type = TOK.Uint32; | |
1188 break; | |
1189 case Suffix.Long: | |
1190 if (ulong_ & 0x8000000000000000) | |
1191 { | |
1192 if (isDecimal) | |
1193 error(MID.OverflowDecimalSign); | |
1194 t.type = TOK.Uint64; | |
1195 } | |
1196 else | |
1197 t.type = TOK.Int64; | |
1198 break; | |
1199 case Suffix.Unsigned | Suffix.Long: | |
1200 t.type = TOK.Uint64; | |
1201 break; | |
1202 default: | |
1203 assert(0); | |
1204 } | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1205 t.ulong_ = ulong_; |
15 | 1206 t.end = p; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1207 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1208 LscanReal: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1209 scanReal(t); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1210 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1211 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1212 |
63 | 1213 /* |
1214 FloatLiteral:= Float[fFL]?i? | |
1215 Float:= DecFloat | HexFloat | |
1216 DecFloat:= ([0-9][0-9_]*[.]([0-9_]*DecExponent?)?) | [.][0-9][0-9_]*DecExponent? | [0-9][0-9_]*DecExponent | |
1217 DecExponent:= [eE][+-]?[0-9][0-9_]* | |
1218 HexFloat:= 0[xX](HexDigits[.]HexDigits | [.][0-9a-zA-Z]HexDigits? | HexDigits)HexExponent | |
1219 HexExponent:= [pP][+-]?[0-9][0-9_]* | |
1220 */ | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1221 void scanReal(ref Token t) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1222 { |
63 | 1223 if (*p == '.') |
1224 // This function was called by scan() or scanNumber(). | |
1225 while (isdigit(*++p) || *p == '_') {} | |
1226 else | |
1227 { | |
1228 // This function was called by scanNumber(). | |
1229 debug switch (*p) | |
1230 { | |
1231 case 'L': | |
1232 if (p[1] != 'i') | |
1233 assert(0); | |
1234 case 'i', 'f', 'F', 'e', 'E': break; | |
1235 default: assert(0); | |
1236 } | |
1237 } | |
1238 | |
1239 // Scan exponent. | |
1240 if (*p == 'e' || *p == 'E') | |
1241 { | |
1242 ++p; | |
1243 if (*p == '-' || *p == '+') | |
1244 ++p; | |
1245 if (!isdigit(*p)) | |
1246 error(MID.FloatExponentDigitExpected); | |
1247 else | |
64 | 1248 while (isdigit(*++p) || *p == '_') {} |
63 | 1249 } |
1250 | |
1251 // Copy string to buffer ignoring underscores. | |
1252 char[] buffer; | |
1253 char* end = p; | |
1254 p = t.start; | |
1255 do | |
1256 { | |
1257 if (*p == '_') | |
1258 { | |
1259 ++p; | |
1260 continue; | |
1261 } | |
1262 buffer ~= *p; | |
1263 ++p; | |
1264 } while (p != end) | |
1265 buffer ~= 0; | |
1266 finalizeFloat(t, buffer); | |
30 | 1267 } |
1268 | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1269 void scanHexReal(ref Token t) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1270 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1271 assert(*p == '.' || *p == 'i' || *p == 'p' || *p == 'P' || (*p == 'L' && p[1] == 'i')); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1272 MID mid; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1273 if (*p == '.') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1274 while (ishexad(*++p) || *p == '_') {} |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1275 if (*p != 'p' && *p != 'P') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1276 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1277 mid = MID.HexFloatExponentRequired; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1278 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1279 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1280 // Copy mantissa to a buffer ignoring underscores. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1281 char* end = p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1282 p = t.start; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1283 char[] buffer; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1284 do |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1285 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1286 if (*p == '_') |
63 | 1287 { |
1288 ++p; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1289 continue; |
63 | 1290 } |
1291 buffer ~= *p; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1292 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1293 } while (p != end) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1294 |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1295 assert(p == end && (*p == 'p' || *p == 'P')); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1296 // Scan and copy the exponent. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1297 buffer ~= 'p'; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1298 size_t bufflen = buffer.length; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1299 while (1) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1300 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1301 if (*++p == '_') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1302 continue; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1303 if (isdigit(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1304 buffer ~= *p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1305 else |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1306 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1307 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1308 // When the buffer length hasn't changed, no digits were copied. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1309 if (bufflen == buffer.length) { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1310 mid = MID.HexFloatMissingExpDigits; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1311 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1312 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1313 buffer ~= 0; // Terminate for C functions. |
63 | 1314 finalizeFloat(t, buffer); |
1315 return; | |
1316 Lerr: | |
1317 t.type = TOK.Float32; | |
1318 t.end = p; | |
1319 error(mid); | |
1320 } | |
1321 | |
1322 void finalizeFloat(ref Token t, string buffer) | |
1323 { | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1324 // Float number is well-formed. Check suffixes and do conversion. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1325 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1326 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1327 case 'f', 'F': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1328 t.type = TOK.Float32; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1329 t.float_ = strtof(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1330 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1331 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1332 case 'L': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1333 t.type = TOK.Float80; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1334 t.real_ = strtold(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1335 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1336 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1337 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1338 t.type = TOK.Float64; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1339 t.double_ = strtod(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1340 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1341 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1342 if (*p == 'i') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1343 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1344 ++p; |
63 | 1345 t.type += 3; // Switch to imaginary counterpart. |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1346 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1347 if (getErrno == ERANGE) |
63 | 1348 error(MID.OverflowFloatNumber); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1349 t.end = p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1350 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1351 |
30 | 1352 /// Scan special token: #line Integer [Filespec] EndOfLine |
51 | 1353 // TODO: Handle case like: #line 0 #line 2 |
30 | 1354 void scanSpecialToken() |
1355 { | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1356 assert(*p == '#'); |
51 | 1357 |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1358 ++p; |
30 | 1359 MID mid; |
1360 Token t; | |
51 | 1361 uint oldloc = this.loc, newloc; |
30 | 1362 |
51 | 1363 peek(t); |
65 | 1364 if (!(this.loc == oldloc && p == t.start && t.type == TOK.Identifier && t.srcText == "line")) |
51 | 1365 { |
1366 this.loc = oldloc; // reset this.loc because we took a peek at the next token | |
1367 mid = MID.ExpectedIdentifierSTLine; | |
1368 goto Lerr; | |
1369 } | |
1370 p = t.end; // consume token | |
1371 | |
1372 peek(t); | |
1373 if (this.loc == oldloc && t.type == TOK.Number) | |
1374 { | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1375 newloc = t.uint_ - 1; |
51 | 1376 p = t.end; |
1377 } | |
1378 else | |
1379 { | |
1380 this.loc = oldloc; | |
1381 mid = MID.ExpectedNumberAfterSTLine; | |
30 | 1382 goto Lerr; |
1383 } | |
1384 | |
51 | 1385 peek(t); |
1386 if (this.loc != oldloc) | |
30 | 1387 { |
51 | 1388 this.loc = oldloc; |
1389 mid = MID.NewlineInSpecialToken; | |
1390 goto Lerr; | |
30 | 1391 } |
1392 if (t.type == TOK.String) | |
1393 { | |
51 | 1394 if (*t.start != '"') |
1395 { | |
1396 mid = MID.ExpectedNormalStringLiteral; | |
30 | 1397 goto Lerr; |
51 | 1398 } |
65 | 1399 fileName = t.srcText[1..$-1]; // contents of "..." |
30 | 1400 p = t.end; |
1401 } | |
65 | 1402 else if (t.type == TOK.Identifier && t.srcText == "__FILE__") |
30 | 1403 { |
1404 p = t.end; | |
1405 } | |
51 | 1406 /+ |
1407 peek(t); | |
1408 if (this.loc == oldloc && t.type != TOK.EOF) | |
1409 { | |
1410 mid = MID.UnterminatedSpecialToken; | |
1411 goto Lerr; | |
1412 } | |
1413 +/ | |
30 | 1414 while (1) |
1415 { | |
51 | 1416 switch (*p) |
1417 { | |
1418 case '\r': | |
1419 if (p[1] == '\n') | |
1420 ++p; | |
1421 case '\n': | |
1422 ++p; | |
30 | 1423 break; |
51 | 1424 case LS[0]: |
1425 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) | |
1426 { | |
1427 p += 2; | |
1428 break; | |
1429 } | |
1430 case 0, _Z_: | |
1431 break; | |
1432 default: | |
1433 if (isspace(*p)) { | |
1434 ++p; | |
1435 continue; | |
1436 } | |
30 | 1437 mid = MID.UnterminatedSpecialToken; |
1438 goto Lerr; | |
1439 } | |
51 | 1440 break; |
30 | 1441 } |
1442 | |
51 | 1443 this.loc = newloc; |
30 | 1444 return; |
1445 Lerr: | |
1446 error(mid); | |
15 | 1447 } |
1448 | |
49 | 1449 uint decodeUTF8() |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1450 { |
49 | 1451 assert(*p & 128, "check for ASCII char before calling decodeUTF8()."); |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1452 size_t idx; |
49 | 1453 uint d = 0xFFFF; |
1454 try | |
1455 { | |
1456 d = std.utf.decode(p[0 .. end-p], idx); | |
1457 p += idx -1; | |
1458 } | |
1459 catch (UtfException e) | |
1460 { | |
1461 error(MID.InvalidUTF8Sequence); | |
1462 // Skip to next valid utf-8 sequence | |
1463 while (UTF8stride[*++p] != 0xFF) {} | |
1464 } | |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1465 return d; |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1466 } |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1467 |
28 | 1468 void loadKeywords() |
1469 { | |
1470 foreach(k; keywords) | |
1471 idtable[k.str] = k; | |
1472 } | |
1473 | |
66
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
1474 void error(MID id, ...) |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1475 { |
66
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
1476 errors ~= new Problem(Problem.Type.Lexer, id, loc, _arguments, _argptr); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1477 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1478 |
3 | 1479 public TOK nextToken() |
1480 { | |
4 | 1481 scan(this.token); |
1482 return this.token.type; | |
1483 } | |
1484 | |
1485 Token[] getTokens() | |
1486 { | |
1487 Token[] tokens; | |
1488 while (nextToken() != TOK.EOF) | |
1489 tokens ~= this.token; | |
1490 tokens ~= this.token; | |
1491 return tokens; | |
3 | 1492 } |
48 | 1493 |
1494 private void encodeUTF8(inout char[] str, dchar d) | |
1495 { | |
1496 char[6] b; | |
1497 assert(d > 0x7F, "check for ASCII char before calling encodeUTF8()."); | |
1498 if (d < 0x800) | |
1499 { | |
1500 b[0] = 0xC0 | (d >> 6); | |
1501 b[1] = 0x80 | (d & 0x3F); | |
1502 str ~= b[0..2]; | |
1503 } | |
1504 else if (d < 0x10000) | |
1505 { | |
1506 b[0] = 0xE0 | (d >> 12); | |
1507 b[1] = 0x80 | ((d >> 6) & 0x3F); | |
1508 b[2] = 0x80 | (d & 0x3F); | |
1509 str ~= b[0..3]; | |
1510 } | |
1511 else if (d < 0x200000) | |
1512 { | |
1513 b[0] = 0xF0 | (d >> 18); | |
1514 b[1] = 0x80 | ((d >> 12) & 0x3F); | |
1515 b[2] = 0x80 | ((d >> 6) & 0x3F); | |
1516 b[3] = 0x80 | (d & 0x3F); | |
1517 str ~= b[0..4]; | |
1518 } | |
1519 else if (d < 0x4000000) | |
1520 { | |
1521 b[0] = 0xF8 | (d >> 24); | |
1522 b[1] = 0x80 | ((d >> 18) & 0x3F); | |
1523 b[2] = 0x80 | ((d >> 12) & 0x3F); | |
1524 b[3] = 0x80 | ((d >> 6) & 0x3F); | |
1525 b[4] = 0x80 | (d & 0x3F); | |
1526 str ~= b[0..5]; | |
1527 } | |
1528 else if (d < 0x80000000) | |
1529 { | |
1530 b[0] = 0xFC | (d >> 30); | |
1531 b[1] = 0x80 | ((d >> 24) & 0x3F); | |
1532 b[2] = 0x80 | ((d >> 18) & 0x3F); | |
1533 b[3] = 0x80 | ((d >> 12) & 0x3F); | |
1534 b[4] = 0x80 | ((d >> 6) & 0x3F); | |
1535 b[5] = 0x80 | (d & 0x3F); | |
1536 str ~= b[0..6]; | |
1537 } | |
1538 else | |
1539 error(MID.InvalidUnicodeCharacter); | |
1540 } | |
5 | 1541 } |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1542 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1543 unittest |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1544 { |
40 | 1545 string[] toks = [ |
1546 ">", ">=", ">>", ">>=", ">>>", ">>>=", "<", "<=", "<>", | |
1547 "<>=", "<<", "<<=", "!", "!<", "!>", "!<=", "!>=", "!<>", | |
1548 "!<>=", ".", "..", "...", "&", "&&", "&=", "+", "++", | |
1549 "+=", "-", "--", "-=", "=", "==", "~", "~=", "*", | |
1550 "*=", "/", "/=", "^", "^=", "%", "%=", "(", ")", | |
1551 "[", "]", "{", "}", ":", ";", "?", ",", "$" | |
1552 ]; | |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1553 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1554 char[] src; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1555 |
40 | 1556 foreach (op; toks) |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1557 src ~= op ~ " "; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1558 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1559 auto lx = new Lexer(src, ""); |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1560 auto tokens = lx.getTokens(); |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1561 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1562 tokens = tokens[0..$-1]; // exclude TOK.EOF |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1563 |
40 | 1564 assert(tokens.length == toks.length ); |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1565 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1566 foreach (i, t; tokens) |
40 | 1567 assert(t.span == toks[i], std.string.format("Lexed '%s' but expected '%s'", t.span, toks[i])); |
41 | 1568 } |
55 | 1569 |
1570 /// ASCII character properties table. | |
1571 static const int ptable[256] = [ | |
1572 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0, | |
1573 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1574 32, 0, 0x2200, 0, 0, 0, 0, 0x2700, 0, 0, 0, 0, 0, 0, 0, 0, | |
1575 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0x3f00, | |
1576 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8, | |
1577 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0x5c00, 0, 0,16, | |
1578 0, 0x70c, 0x80c,12,12,12, 0xc0c, 8, 8, 8, 8, 8, 8, 8, 0xa08, 8, | |
1579 8, 8, 0xd08, 8, 0x908, 8, 0xb08, 8, 8, 8, 8, 0, 0, 0, 0, 0, | |
1580 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1581 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1582 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1583 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1584 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1585 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1586 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1587 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 | |
1588 ]; | |
1589 | |
1590 enum CProperty | |
1591 { | |
1592 Octal = 1, | |
1593 Digit = 1<<1, | |
1594 Hex = 1<<2, | |
1595 Alpha = 1<<3, | |
1596 Underscore = 1<<4, | |
1597 Whitespace = 1<<5 | |
1598 } | |
1599 | |
1600 const uint EVMask = 0xFF00; // Bit mask for escape value | |
1601 | |
1602 private alias CProperty CP; | |
1603 int isoctal(char c) { return ptable[c] & CP.Octal; } | |
1604 int isdigit(char c) { return ptable[c] & CP.Digit; } | |
1605 int ishexad(char c) { return ptable[c] & CP.Hex; } | |
1606 int isalpha(char c) { return ptable[c] & CP.Alpha; } | |
1607 int isalnum(char c) { return ptable[c] & (CP.Alpha | CP.Digit); } | |
1608 int isidbeg(char c) { return ptable[c] & (CP.Alpha | CP.Underscore); } | |
1609 int isident(char c) { return ptable[c] & (CP.Alpha | CP.Underscore | CP.Digit); } | |
1610 int isspace(char c) { return ptable[c] & CP.Whitespace; } | |
1611 int char2ev(char c) { return ptable[c] >> 8; /*(ptable[c] & EVMask) >> 8;*/ } | |
1612 | |
1613 version(gen_ptable) | |
1614 static this() | |
1615 { | |
1616 alias ptable p; | |
1617 // Initialize character properties table. | |
1618 for (int i; i < p.length; ++i) | |
1619 { | |
1620 p[i] = 0; | |
1621 if ('0' <= i && i <= '7') | |
1622 p[i] |= CP.Octal; | |
1623 if ('0' <= i && i <= '9') | |
1624 p[i] |= CP.Digit; | |
1625 if (isdigit(i) || 'a' <= i && i <= 'f' || 'A' <= i && i <= 'F') | |
1626 p[i] |= CP.Hex; | |
1627 if ('a' <= i && i <= 'z' || 'A' <= i && i <= 'Z') | |
1628 p[i] |= CP.Alpha; | |
1629 if (i == '_') | |
1630 p[i] |= CP.Underscore; | |
1631 if (i == ' ' || i == '\t' || i == '\v' || i == '\f') | |
1632 p[i] |= CP.Whitespace; | |
1633 } | |
1634 // Store escape sequence values in second byte. | |
1635 assert(CProperty.max <= ubyte.max, "character property flags and escape value byte overlap."); | |
1636 p['\''] |= 39 << 8; | |
1637 p['"'] |= 34 << 8; | |
1638 p['?'] |= 63 << 8; | |
1639 p['\\'] |= 92 << 8; | |
1640 p['a'] |= 7 << 8; | |
1641 p['b'] |= 8 << 8; | |
1642 p['f'] |= 12 << 8; | |
1643 p['n'] |= 10 << 8; | |
1644 p['r'] |= 13 << 8; | |
1645 p['t'] |= 9 << 8; | |
1646 p['v'] |= 11 << 8; | |
1647 // Print a formatted array literal. | |
1648 char[] array = "[\n"; | |
1649 for (int i; i < p.length; ++i) | |
1650 { | |
1651 int c = p[i]; | |
1652 array ~= std.string.format(c>255?" 0x%x,":"%2d,", c, ((i+1) % 16) ? "":"\n"); | |
1653 } | |
1654 array[$-2..$] = "\n]"; | |
1655 writefln(array); | |
1656 } |