Mercurial > projects > dil
annotate trunk/src/dil/Lexer.d @ 371:01887f05d4b0
- Added members loc_old and loc_hline to class Lexer.
- Added method errorLoc() for calculating error line number.
author | aziz |
---|---|
date | Thu, 06 Sep 2007 07:10:03 +0000 |
parents | ae4afb66768f |
children | 66477017cb95 |
rev | line source |
---|---|
0 | 1 /++ |
2 Author: Aziz Köksal | |
249 | 3 License: GPL3 |
0 | 4 +/ |
326 | 5 module dil.Lexer; |
327 | 6 import dil.Token; |
7 import dil.Information; | |
8 import dil.Keywords; | |
9 import dil.Identifier; | |
10 import dil.Messages; | |
11 import dil.HtmlEntities; | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
12 import dil.Settings; |
2 | 13 import std.stdio; |
4 | 14 import std.utf; |
15 import std.uni; | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
16 import std.c.stdlib : strtof, strtod, strtold, getErrno, ERANGE; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
17 import std.c.time : time_t, time, ctime; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
18 import std.c.string : strlen; |
66
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
19 import std.string; |
0 | 20 |
11 | 21 const char[3] LS = \u2028; |
22 const char[3] PS = \u2029; | |
23 | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
24 const dchar LSd = 0x2028; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
25 const dchar PSd = 0x2029; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
26 |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
27 const uint _Z_ = 26; /// Control+Z |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
28 |
0 | 29 class Lexer |
30 { | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
31 Token* head; /// The head of the doubly linked token list. |
344 | 32 Token* tail; /// The tail of the linked list. Set in scan(). |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
33 Token* token; /// Points to the current token in the token list. |
30 | 34 string text; |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
35 char* p; /// Points to the current character in the source text. |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
36 char* end; /// Points one character past the end of the source text. |
4 | 37 |
371 | 38 uint loc = 1; /// Actual line of code. |
39 | |
40 uint loc_old; /// Store actual line number when #line token is parsed. | |
41 uint loc_hline; /// Line number set by #line. | |
17 | 42 |
30 | 43 char[] fileName; |
44 | |
69
24db7c5522d5
- Added module Information for compiler messages like warnings, info and errors to the user.
aziz
parents:
68
diff
changeset
|
45 Information[] errors; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
46 |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
47 // bool reportErrors; |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
48 |
28 | 49 Identifier[string] idtable; |
50 | |
30 | 51 this(string text, string fileName) |
4 | 52 { |
30 | 53 this.fileName = fileName; |
54 | |
4 | 55 this.text = text; |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
56 if (text[$-1] != 0) |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
57 { |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
58 this.text.length = this.text.length + 1; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
59 this.text[$-1] = 0; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
60 } |
4 | 61 |
62 this.p = this.text.ptr; | |
63 this.end = this.p + this.text.length; | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
64 // this.reportErrors = true; |
28 | 65 loadKeywords(); |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
66 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
67 this.head = new Token; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
68 this.head.type = TOK.HEAD; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
69 this.token = this.head; |
315 | 70 scanShebang(); |
71 } | |
72 | |
344 | 73 ~this() |
74 { | |
75 auto token = head.next; | |
76 do | |
77 { | |
78 assert(token.type == TOK.EOF ? token == tail && token.next is null : 1); | |
79 delete token.prev; | |
80 token = token.next; | |
81 } while (token !is null) | |
82 delete tail; | |
83 } | |
84 | |
315 | 85 void scanShebang() |
86 { | |
87 if (*p == '#' && p[1] == '!') | |
88 { | |
89 Token* t = new Token; | |
90 t.start = p; | |
91 t.type = TOK.Shebang; | |
92 ++p; | |
93 while (1) | |
94 { | |
95 switch (*++p) | |
96 { | |
97 case '\r': | |
98 if (p[1] == '\n') | |
99 ++p; | |
100 case '\n': | |
101 ++loc; | |
102 if (p[-1] == '\r') | |
103 t.end = p-1; | |
104 else | |
105 t.end = p; | |
106 break; | |
107 case LS[0]: | |
108 t.end = p; | |
109 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) | |
110 { | |
111 ++p; ++p; | |
112 ++loc; | |
113 } | |
114 break; | |
115 case 0, _Z_: | |
116 t.end = p; | |
117 break; | |
118 default: | |
119 continue; | |
120 } | |
121 break; // Exit loop. | |
122 } | |
123 this.head.next = t; | |
124 t.prev = this.head; | |
125 } | |
4 | 126 } |
127 | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
128 void finalizeSpecialToken(ref Token t) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
129 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
130 assert(t.srcText[0..2] == "__"); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
131 switch (t.type) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
132 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
133 case TOK.FILE: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
134 t.str = this.fileName; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
135 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
136 case TOK.LINE: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
137 t.uint_ = this.loc; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
138 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
139 case TOK.DATE, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
140 TOK.TIME, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
141 TOK.TIMESTAMP: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
142 time_t time_val; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
143 time(&time_val); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
144 char* str = ctime(&time_val); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
145 char[] time_str = str[0 .. strlen(str)]; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
146 switch (t.type) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
147 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
148 case TOK.DATE: |
346
ce9e2e77743f
- Fix: include one space when slicing time_str for __DATE__.
aziz
parents:
344
diff
changeset
|
149 time_str = time_str[4..11] ~ time_str[20..24] ~ \0; break; |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
150 case TOK.TIME: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
151 time_str = time_str[11..19] ~ \0; break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
152 case TOK.TIMESTAMP: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
153 time_str = time_str[0..24] ~ \0; break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
154 default: assert(0); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
155 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
156 t.str = time_str; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
157 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
158 case TOK.VENDOR: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
159 t.str = VENDOR; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
160 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
161 case TOK.VERSION: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
162 t.uint_ = VERSION_MAJOR*1000 + VERSION_MINOR; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
163 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
164 default: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
165 assert(0); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
166 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
167 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
168 |
4 | 169 public void scan(out Token t) |
207 | 170 in |
3 | 171 { |
207 | 172 assert(text.ptr <= p && p < end); |
173 } | |
174 out | |
175 { | |
176 assert(text.ptr <= t.start && t.start < end); | |
177 assert(text.ptr < t.end && t.end <= end, std.string.format(t.type)); | |
178 } | |
179 body | |
180 { | |
10 | 181 uint c = *p; |
4 | 182 |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
183 while (1) |
4 | 184 { |
5 | 185 t.start = p; |
16 | 186 |
212 | 187 if (c == 0 || c == _Z_) |
5 | 188 { |
212 | 189 assert(*p == 0 || *p == _Z_); |
5 | 190 t.type = TOK.EOF; |
17 | 191 t.end = p; |
344 | 192 tail = &t; |
212 | 193 assert(t.start == t.end); |
5 | 194 return; |
195 } | |
4 | 196 |
17 | 197 if (c == '\n') |
198 { | |
199 c = *++p; | |
200 ++loc; | |
201 continue; | |
202 } | |
203 else if (c == '\r') | |
204 { | |
205 c = *++p; | |
206 if (c != '\n') | |
207 ++loc; | |
208 continue; | |
209 } | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
210 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
211 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
212 p += 3; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
213 c = *p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
214 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
215 } |
17 | 216 |
13 | 217 if (isidbeg(c)) |
5 | 218 { |
33 | 219 if (c == 'r' && p[1] == '"' && ++p) |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
220 return scanRawStringLiteral(t); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
221 if (c == 'x' && p[1] == '"') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
222 return scanHexStringLiteral(t); |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
223 Lidentifier: |
5 | 224 do |
225 { c = *++p; } | |
49 | 226 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8())) |
28 | 227 |
5 | 228 t.end = p; |
28 | 229 |
65 | 230 string str = t.srcText; |
28 | 231 Identifier* id = str in idtable; |
232 | |
233 if (!id) | |
234 { | |
327 | 235 idtable[str] = Identifier(TOK.Identifier, str); |
28 | 236 id = str in idtable; |
237 } | |
238 assert(id); | |
239 t.type = id.type; | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
240 if (t.isSpecialToken) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
241 finalizeSpecialToken(t); |
5 | 242 return; |
243 } | |
244 | |
15 | 245 if (isdigit(c)) |
246 return scanNumber(t); | |
247 | |
8 | 248 if (c == '/') |
5 | 249 { |
8 | 250 c = *++p; |
14 | 251 switch(c) |
5 | 252 { |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
253 case '=': |
14 | 254 ++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
255 t.type = TOK.DivAssign; |
14 | 256 t.end = p; |
257 return; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
258 case '+': |
8 | 259 uint level = 1; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
260 while (1) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
261 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
262 c = *++p; |
42 | 263 LswitchNC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
264 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
265 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
266 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
267 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
268 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
269 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
270 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
271 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
272 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
273 error(MID.UnterminatedNestedComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
274 goto LreturnNC; |
42 | 275 default: |
276 } | |
277 | |
278 c <<= 8; | |
279 c |= *++p; | |
280 switch (c) | |
281 { | |
282 case 0x2F2B: // /+ | |
283 ++level; | |
284 continue; | |
285 case 0x2B2F: // +/ | |
286 if (--level == 0) | |
287 { | |
288 ++p; | |
289 LreturnNC: | |
290 t.type = TOK.Comment; | |
291 t.end = p; | |
292 return; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
293 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
294 continue; |
42 | 295 case 0xE280: // LS[0..1] || PS[0..1] |
296 if (p[1] == LS[2] || p[1] == PS[2]) | |
297 { | |
298 ++loc; | |
299 ++p; | |
300 } | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
301 continue; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
302 default: |
42 | 303 c &= char.max; |
304 goto LswitchNC; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
305 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
306 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
307 case '*': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
308 while (1) |
7 | 309 { |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
310 c = *++p; |
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
311 LswitchBC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
312 switch (c) |
8 | 313 { |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
314 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
315 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
316 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
317 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
318 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
319 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
320 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
321 error(MID.UnterminatedBlockComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
322 goto LreturnBC; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
323 default: |
8 | 324 } |
41 | 325 |
326 c <<= 8; | |
327 c |= *++p; | |
328 switch (c) | |
329 { | |
330 case 0x2A2F: // */ | |
331 ++p; | |
332 LreturnBC: | |
333 t.type = TOK.Comment; | |
334 t.end = p; | |
335 return; | |
336 case 0xE280: // LS[0..1] || PS[0..1] | |
337 if (p[1] == LS[2] || p[1] == PS[2]) | |
338 { | |
339 ++loc; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
340 ++p; |
41 | 341 } |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
342 continue; |
41 | 343 default: |
344 c &= char.max; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
345 goto LswitchBC; |
41 | 346 } |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
347 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
348 assert(0); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
349 case '/': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
350 while (1) |
7 | 351 { |
8 | 352 c = *++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
353 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
354 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
355 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
356 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
357 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
358 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
359 case 0, _Z_: |
11 | 360 break; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
361 case LS[0]: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
362 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
363 break; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
364 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
365 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
366 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
367 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
368 t.type = TOK.Comment; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
369 t.end = p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
370 return; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
371 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
372 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
373 t.type = TOK.Div; |
10 | 374 t.end = p; |
375 return; | |
376 } | |
5 | 377 } |
378 | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
379 switch (c) |
20 | 380 { |
48 | 381 case '\'': |
382 return scanCharacterLiteral(t); | |
383 case '`': | |
384 return scanRawStringLiteral(t); | |
385 case '"': | |
386 return scanNormalStringLiteral(t); | |
387 case '\\': | |
388 char[] buffer; | |
389 do | |
390 { | |
391 ++p; | |
392 c = scanEscapeSequence(); | |
393 if (c < 128) | |
394 buffer ~= c; | |
395 else | |
396 encodeUTF8(buffer, c); | |
397 } while (*p == '\\') | |
398 buffer ~= 0; | |
399 t.type = TOK.String; | |
400 t.str = buffer; | |
401 t.end = p; | |
402 return; | |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
403 case '>': /* > >= >> >>= >>> >>>= */ |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
404 c = *++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
405 switch (c) |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
406 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
407 case '=': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
408 t.type = TOK.GreaterEqual; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
409 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
410 case '>': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
411 if (p[1] == '>') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
412 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
413 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
414 if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
415 { ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
416 t.type = TOK.URShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
417 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
418 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
419 t.type = TOK.URShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
420 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
421 else if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
422 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
423 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
424 t.type = TOK.RShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
425 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
426 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
427 t.type = TOK.RShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
428 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
429 default: |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
430 t.type = TOK.Greater; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
431 goto Lcommon2; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
432 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
433 assert(0); |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
434 case '<': /* < <= <> <>= << <<= */ |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
435 c = *++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
436 switch (c) |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
437 { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
438 case '=': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
439 t.type = TOK.LessEqual; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
440 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
441 case '<': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
442 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
443 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
444 t.type = TOK.LShiftAssign; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
445 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
446 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
447 t.type = TOK.LShift; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
448 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
449 case '>': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
450 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
451 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
452 t.type = TOK.LorEorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
453 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
454 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
455 t.type = TOK.LorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
456 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
457 default: |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
458 t.type = TOK.Less; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
459 goto Lcommon2; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
460 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
461 assert(0); |
37
7f3bcb97d017
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
36
diff
changeset
|
462 case '!': /* ! !< !> !<= !>= !<> !<>= */ |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
463 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
464 switch (c) |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
465 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
466 case '<': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
467 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
468 if (c == '>') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
469 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
470 if (p[1] == '=') { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
471 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
472 t.type = TOK.Unordered; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
473 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
474 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
475 t.type = TOK.UorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
476 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
477 else if (c == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
478 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
479 t.type = TOK.UorG; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
480 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
481 else { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
482 t.type = TOK.UorGorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
483 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
484 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
485 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
486 case '>': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
487 if (p[1] == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
488 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
489 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
490 t.type = TOK.UorL; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
491 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
492 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
493 t.type = TOK.UorLorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
494 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
495 case '=': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
496 t.type = TOK.NotEqual; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
497 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
498 default: |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
499 t.type = TOK.Not; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
500 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
501 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
502 assert(0); |
63 | 503 case '.': /* . .[0-9] .. ... */ |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
504 if (p[1] == '.') |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
505 { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
506 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
507 if (p[1] == '.') { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
508 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
509 t.type = TOK.Ellipses; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
510 } |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
511 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
512 t.type = TOK.Slice; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
513 } |
63 | 514 else if (isdigit(p[1])) |
515 { | |
516 return scanReal(t); | |
517 } | |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
518 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
519 t.type = TOK.Dot; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
520 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
521 case '|': /* | || |= */ |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
522 c = *++p; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
523 if (c == '=') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
524 t.type = TOK.OrAssign; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
525 else if (c == '|') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
526 t.type = TOK.OrLogical; |
26 | 527 else { |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
528 t.type = TOK.OrBinary; |
26 | 529 goto Lcommon2; |
530 } | |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
531 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
532 case '&': /* & && &= */ |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
533 c = *++p; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
534 if (c == '=') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
535 t.type = TOK.AndAssign; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
536 else if (c == '&') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
537 t.type = TOK.AndLogical; |
26 | 538 else { |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
539 t.type = TOK.AndBinary; |
26 | 540 goto Lcommon2; |
541 } | |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
542 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
543 case '+': /* + ++ += */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
544 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
545 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
546 t.type = TOK.PlusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
547 else if (c == '+') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
548 t.type = TOK.PlusPlus; |
26 | 549 else { |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
550 t.type = TOK.Plus; |
26 | 551 goto Lcommon2; |
552 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
553 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
554 case '-': /* - -- -= */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
555 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
556 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
557 t.type = TOK.MinusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
558 else if (c == '-') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
559 t.type = TOK.MinusMinus; |
26 | 560 else { |
561 t.type = TOK.Minus; | |
562 goto Lcommon2; | |
563 } | |
564 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
565 case '=': /* = == */ |
26 | 566 if (p[1] == '=') { |
567 ++p; | |
568 t.type = TOK.Equal; | |
569 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
570 else |
26 | 571 t.type = TOK.Assign; |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
572 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
573 case '~': /* ~ ~= */ |
27 | 574 if (p[1] == '=') { |
575 ++p; | |
576 t.type = TOK.CatAssign; | |
577 } | |
578 else | |
579 t.type = TOK.Tilde; | |
580 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
581 case '*': /* * *= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
582 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
583 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
584 t.type = TOK.MulAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
585 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
586 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
587 t.type = TOK.Mul; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
588 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
589 case '^': /* ^ ^= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
590 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
591 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
592 t.type = TOK.XorAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
593 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
594 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
595 t.type = TOK.Xor; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
596 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
597 case '%': /* % %= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
598 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
599 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
600 t.type = TOK.ModAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
601 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
602 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
603 t.type = TOK.Mod; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
604 goto Lcommon; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
605 // Single character tokens: |
20 | 606 case '(': |
607 t.type = TOK.LParen; | |
608 goto Lcommon; | |
609 case ')': | |
610 t.type = TOK.RParen; | |
611 goto Lcommon; | |
612 case '[': | |
613 t.type = TOK.LBracket; | |
614 goto Lcommon; | |
615 case ']': | |
616 t.type = TOK.RBracket; | |
617 goto Lcommon; | |
618 case '{': | |
619 t.type = TOK.LBrace; | |
620 goto Lcommon; | |
621 case '}': | |
622 t.type = TOK.RBrace; | |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
623 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
624 case ':': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
625 t.type = TOK.Colon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
626 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
627 case ';': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
628 t.type = TOK.Semicolon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
629 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
630 case '?': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
631 t.type = TOK.Question; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
632 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
633 case ',': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
634 t.type = TOK.Comma; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
635 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
636 case '$': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
637 t.type = TOK.Dollar; |
20 | 638 Lcommon: |
639 ++p; | |
26 | 640 Lcommon2: |
20 | 641 t.end = p; |
642 return; | |
30 | 643 case '#': |
360
b6a3755eba94
- Renamed scanSpecialToken() to scanSpecialTokenSequence().
aziz
parents:
350
diff
changeset
|
644 return scanSpecialTokenSequence(t); |
20 | 645 default: |
646 } | |
647 | |
49 | 648 if (c & 128 && isUniAlpha(decodeUTF8())) |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
649 goto Lidentifier; |
5 | 650 c = *++p; |
4 | 651 } |
3 | 652 } |
4 | 653 |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
654 void scanNormalStringLiteral(ref Token t) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
655 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
656 assert(*p == '"'); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
657 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
658 char[] buffer; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
659 t.type = TOK.String; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
660 while (1) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
661 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
662 switch (*p) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
663 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
664 case '"': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
665 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
666 Lreturn: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
667 buffer ~= 0; |
49 | 668 t.str = buffer; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
669 t.pf = scanPostfix(); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
670 t.end = p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
671 return; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
672 case '\\': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
673 ++p; |
49 | 674 dchar d = scanEscapeSequence(); |
675 if (d < 128) | |
676 buffer ~= d; | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
677 else |
49 | 678 encodeUTF8(buffer, d); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
679 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
680 case '\r': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
681 if (p[1] == '\n') |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
682 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
683 case '\n': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
684 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
685 ++loc; |
221 | 686 buffer ~= '\n'; // Convert EndOfLine to \n. |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
687 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
688 case 0, _Z_: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
689 error(MID.UnterminatedString); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
690 goto Lreturn; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
691 default: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
692 if (*p & 128) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
693 { |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
694 // char* begin = p; |
49 | 695 dchar d = decodeUTF8(); |
696 if (d == LSd || d == PSd) | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
697 goto case '\n'; |
49 | 698 |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
699 // We don't copy per pointer because we might include |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
700 // invalid, skipped utf-8 sequences. See decodeUTF8(). |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
701 // ++p; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
702 // buffer ~= begin[0 .. p - begin]; |
221 | 703 ++p; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
704 encodeUTF8(buffer, d); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
705 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
706 } |
223 | 707 // Copy ASCII character. |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
708 buffer ~= *p++; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
709 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
710 } |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
711 assert(0); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
712 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
713 |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
714 void scanCharacterLiteral(ref Token t) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
715 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
716 assert(*p == '\''); |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
717 MID id = MID.UnterminatedCharacterLiteral; |
46 | 718 ++p; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
719 TOK type = TOK.CharLiteral; |
46 | 720 switch (*p) |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
721 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
722 case '\\': |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
723 ++p; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
724 switch (*p) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
725 { |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
726 case 'u': |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
727 type = TOK.WCharLiteral; break; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
728 case 'U': |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
729 type = TOK.DCharLiteral; break; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
730 default: |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
731 } |
46 | 732 t.dchar_ = scanEscapeSequence(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
733 break; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
734 case '\'': |
46 | 735 ++p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
736 id = MID.EmptyCharacterLiteral; |
46 | 737 case '\n', '\r', 0, _Z_: |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
738 goto Lerr; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
739 default: |
46 | 740 uint c = *p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
741 if (c & 128) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
742 { |
49 | 743 c = decodeUTF8(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
744 if (c == LSd || c == PSd) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
745 goto Lerr; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
746 if (c <= 0xFFFF) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
747 type = TOK.WCharLiteral; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
748 else |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
749 type = TOK.DCharLiteral; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
750 } |
46 | 751 t.dchar_ = c; |
752 ++p; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
753 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
754 |
46 | 755 if (*p == '\'') |
756 ++p; | |
757 else | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
758 Lerr: |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
759 error(id); |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
760 t.type = type; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
761 t.end = p; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
762 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
763 |
33 | 764 char scanPostfix() |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
765 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
766 switch (*p) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
767 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
768 case 'c': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
769 case 'w': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
770 case 'd': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
771 return *p++; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
772 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
773 return 0; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
774 } |
112 | 775 assert(0); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
776 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
777 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
778 void scanRawStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
779 { |
33 | 780 uint delim = *p; |
781 assert(delim == '`' || delim == '"' && p[-1] == 'r'); | |
782 t.type = TOK.String; | |
783 char[] buffer; | |
784 uint c; | |
785 while (1) | |
786 { | |
787 c = *++p; | |
788 switch (c) | |
789 { | |
790 case '\r': | |
791 if (p[1] == '\n') | |
792 ++p; | |
793 c = '\n'; // Convert '\r' and '\r\n' to '\n' | |
794 case '\n': | |
795 ++loc; | |
52
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
796 break; |
33 | 797 case '`': |
798 case '"': | |
799 if (c == delim) | |
800 { | |
801 ++p; | |
802 t.pf = scanPostfix(); | |
803 Lreturn: | |
804 t.str = buffer ~ '\0'; | |
805 t.end = p; | |
806 return; | |
807 } | |
808 break; | |
809 case LS[0]: | |
810 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) | |
811 { | |
52
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
812 c = '\n'; |
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
813 ++p; ++p; |
33 | 814 ++loc; |
815 } | |
816 break; | |
817 case 0, _Z_: | |
818 if (delim == 'r') | |
819 error(MID.UnterminatedRawString); | |
820 else | |
821 error(MID.UnterminatedBackQuoteString); | |
822 goto Lreturn; | |
823 default: | |
824 } | |
825 buffer ~= c; // copy character to buffer | |
826 } | |
827 assert(0); | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
828 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
829 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
830 void scanHexStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
831 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
832 assert(p[0] == 'x' && p[1] == '"'); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
833 t.type = TOK.String; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
834 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
835 uint c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
836 ubyte[] buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
837 ubyte h; // hex number |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
838 uint n; // number of hex digits |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
839 |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
840 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
841 while (1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
842 { |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
843 c = *++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
844 switch (c) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
845 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
846 case '"': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
847 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
848 if (n & 1) |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
849 error(MID.OddNumberOfDigitsInHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
850 t.pf = scanPostfix(); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
851 Lreturn: |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
852 buffer ~= 0; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
853 t.str = cast(string) buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
854 t.end = p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
855 return; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
856 case '\r': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
857 if (p[1] == '\n') |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
858 ++p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
859 case '\n': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
860 ++loc; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
861 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
862 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
863 if (ishexad(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
864 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
865 if (c <= '9') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
866 c -= '0'; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
867 else if (c <= 'F') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
868 c -= 'A' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
869 else |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
870 c -= 'a' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
871 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
872 if (n & 1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
873 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
874 h <<= 4; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
875 h |= c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
876 buffer ~= h; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
877 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
878 else |
112 | 879 h = cast(ubyte)c; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
880 ++n; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
881 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
882 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
883 else if (isspace(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
884 continue; |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
885 |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
886 if (c >= 128) |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
887 { |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
888 c = decodeUTF8(); |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
889 if (c == LSd || c == PSd) |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
890 { |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
891 ++p; ++p; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
892 ++loc; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
893 continue; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
894 } |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
895 } |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
896 else if (c == 0 || c == _Z_) |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
897 { |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
898 error(MID.UnterminatedHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
899 t.pf = 0; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
900 goto Lreturn; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
901 } |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
902 error(MID.NonHexCharInHexString, cast(dchar)c); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
903 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
904 } |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
905 assert(0); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
906 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
907 |
45 | 908 dchar scanEscapeSequence() |
909 { | |
910 uint c = char2ev(*p); | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
911 if (c) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
912 { |
46 | 913 ++p; |
45 | 914 return c; |
46 | 915 } |
45 | 916 uint digits = 2; |
917 | |
918 switch (*p) | |
919 { | |
920 case 'x': | |
921 c = 0; | |
922 while (1) | |
923 { | |
924 ++p; | |
925 if (ishexad(*p)) | |
926 { | |
927 c *= 16; | |
928 if (*p <= '9') | |
48 | 929 c += *p - '0'; |
45 | 930 else if (*p <= 'F') |
48 | 931 c += *p - 'A' + 10; |
45 | 932 else |
48 | 933 c += *p - 'a' + 10; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
934 |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
935 if (!--digits) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
936 { |
48 | 937 ++p; |
45 | 938 break; |
48 | 939 } |
45 | 940 } |
941 else | |
942 { | |
943 error(MID.InsufficientHexDigits); | |
944 break; | |
945 } | |
946 } | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
947 if (!isValidDchar(c)) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
948 error(MID.InvalidUnicodeCharacter); |
45 | 949 break; |
950 case 'u': | |
951 digits = 4; | |
952 goto case 'x'; | |
953 case 'U': | |
954 digits = 8; | |
955 goto case 'x'; | |
956 default: | |
48 | 957 if (isoctal(*p)) |
45 | 958 { |
48 | 959 c = 0; |
960 c += *p - '0'; | |
961 ++p; | |
962 if (!isoctal(*p)) | |
963 return c; | |
964 c *= 8; | |
965 c += *p - '0'; | |
966 ++p; | |
967 if (!isoctal(*p)) | |
968 return c; | |
969 c *= 8; | |
970 c += *p - '0'; | |
971 ++p; | |
972 } | |
973 else if(*p == '&') | |
974 { | |
975 if (isalpha(*++p)) | |
45 | 976 { |
272
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
977 auto begin = p; |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
978 while (isalnum(*++p)) |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
979 {} |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
980 |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
981 if (*p == ';') |
48 | 982 { |
272
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
983 c = entity2Unicode(begin[0..p - begin]); |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
984 ++p; |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
985 if (c == 0xFFFF) |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
986 error(MID.UndefinedHTMLEntity, (begin-1)[0..p-(begin-1)]); |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
987 } |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
988 else |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
989 error(MID.UnterminatedHTMLEntity); |
45 | 990 } |
48 | 991 else |
992 error(MID.InvalidBeginHTMLEntity); | |
45 | 993 } |
994 else | |
48 | 995 error(MID.UndefinedEscapeSequence); |
45 | 996 } |
997 | |
998 return c; | |
999 } | |
1000 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1001 /* |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1002 IntegerLiteral:= (Dec|Hex|Bin|Oct)Suffix? |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1003 Dec:= (0|[1-9][0-9_]*) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1004 Hex:= 0[xX] HexDigits |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1005 Bin:= 0[bB][01_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1006 Oct:= 0[0-7_]+ |
68
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1007 Suffix:= (L[uU]?|[uU]L?) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1008 HexDigits:= [0-9a-zA-Z_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1009 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1010 Invalid: "0b_", "0x_", "._" |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1011 */ |
15 | 1012 void scanNumber(ref Token t) |
1013 { | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1014 ulong ulong_; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1015 bool overflow; |
58 | 1016 bool isDecimal; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1017 size_t digits; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1018 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1019 if (*p != '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1020 goto LscanInteger; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1021 ++p; // skip zero |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1022 // check for xX bB ... |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1023 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1024 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1025 case 'x','X': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1026 goto LscanHex; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1027 case 'b','B': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1028 goto LscanBin; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1029 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1030 if (p[1] == 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1031 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1032 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1033 if (p[1] == '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1034 break; |
58 | 1035 case 'i','f','F', 'e', 'E': // Imaginary and float literal suffix |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1036 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1037 default: |
323 | 1038 if (*p == '_' || isoctal(*p)) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1039 goto LscanOct; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1040 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1041 |
323 | 1042 // Number 0 |
1043 assert(p[-1] == '0'); | |
1044 assert(ulong_ == 0); | |
58 | 1045 isDecimal = true; |
1046 goto Lfinalize; | |
1047 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1048 LscanInteger: |
323 | 1049 assert(*p != 0 && isdigit(*p)); |
58 | 1050 isDecimal = true; |
323 | 1051 goto Lenter_loop_int; |
63 | 1052 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1053 { |
63 | 1054 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1055 continue; |
63 | 1056 if (!isdigit(*p)) |
1057 break; | |
323 | 1058 Lenter_loop_int: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1059 if (ulong_ < ulong.max/10 || (ulong_ == ulong.max/10 && *p <= '5')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1060 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1061 ulong_ *= 10; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1062 ulong_ += *p - '0'; |
58 | 1063 continue; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1064 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1065 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1066 overflow = true; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1067 while (isdigit(*++p)) {} |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1068 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1069 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1070 |
61 | 1071 // The number could be a float, so check overflow below. |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1072 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1073 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1074 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1075 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1076 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1077 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1078 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1079 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1080 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1081 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1082 goto LscanReal; |
58 | 1083 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1084 } |
58 | 1085 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1086 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1087 error(MID.OverflowDecimalNumber); |
58 | 1088 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1089 assert((isdigit(p[-1]) || p[-1] == '_') && !isdigit(*p) && *p != '_'); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1090 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1091 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1092 LscanHex: |
61 | 1093 assert(digits == 0); |
323 | 1094 assert(*p == 'x'); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1095 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1096 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1097 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1098 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1099 if (!ishexad(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1100 break; |
61 | 1101 ++digits; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1102 ulong_ *= 16; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1103 if (*p <= '9') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1104 ulong_ += *p - '0'; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1105 else if (*p <= 'F') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1106 ulong_ += *p - 'A' + 10; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1107 else |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1108 ulong_ += *p - 'a' + 10; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1109 } |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1110 |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1111 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1112 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1113 case '.': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1114 if (p[1] != '.') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1115 goto LscanHexReal; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1116 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1117 case 'L': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1118 if (p[1] != 'i') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1119 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1120 case 'i', 'p', 'P': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1121 goto LscanHexReal; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1122 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1123 } |
61 | 1124 if (digits == 0) |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1125 error(MID.NoDigitsInHexNumber); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1126 else if (digits > 16) |
61 | 1127 { |
1128 // Overflow: skip following digits. | |
1129 error(MID.OverflowHexNumber); | |
1130 while (ishexad(*++p)) {} | |
1131 } | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1132 goto Lfinalize; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1133 LscanHexReal: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1134 return scanHexReal(t); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1135 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1136 LscanBin: |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1137 assert(digits == 0); |
323 | 1138 assert(*p == 'b'); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1139 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1140 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1141 if (*++p == '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1142 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1143 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1144 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1145 } |
367
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1146 else if (*p == '1') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1147 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1148 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1149 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1150 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1151 } |
367
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1152 else if (*p == '_') |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1153 continue; |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1154 else |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1155 break; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1156 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1157 |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1158 if (digits == 0) |
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1159 error(MID.NoDigitsInBinNumber); |
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1160 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1161 if (digits > 64) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1162 error(MID.OverflowBinaryNumber); |
367
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1163 assert(p[-1] == '0' || p[-1] == '1' || p[-1] == '_', p[-1] ~ ""); |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1164 assert( !(*p == '0' || *p == '1' || *p == '_') ); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1165 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1166 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1167 LscanOct: |
323 | 1168 assert(*p == '_' || isoctal(*p)); |
1169 if (*p != '_') | |
1170 goto Lenter_loop_oct; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1171 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1172 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1173 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1174 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1175 if (!isoctal(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1176 break; |
323 | 1177 Lenter_loop_oct: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1178 if (ulong_ < ulong.max/2 || (ulong_ == ulong.max/2 && *p <= '1')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1179 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1180 ulong_ *= 8; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1181 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1182 ++p; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1183 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1184 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1185 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1186 overflow = true; |
63 | 1187 while (isdigit(*++p)) {} |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1188 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1189 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1190 |
63 | 1191 bool hasDecimalDigits; |
1192 if (isdigit(*p)) | |
1193 { | |
1194 hasDecimalDigits = true; | |
1195 while (isdigit(*++p)) {} | |
1196 } | |
1197 | |
1198 // The number could be a float, so check errors below. | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1199 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1200 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1201 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1202 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1203 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1204 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1205 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1206 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1207 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1208 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1209 goto LscanReal; |
63 | 1210 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1211 } |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1212 |
63 | 1213 if (hasDecimalDigits) |
1214 error(MID.OctalNumberHasDecimals); | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1215 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1216 error(MID.OverflowOctalNumber); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1217 // goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1218 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1219 Lfinalize: |
58 | 1220 enum Suffix |
1221 { | |
1222 None = 0, | |
1223 Unsigned = 1, | |
60 | 1224 Long = 2 |
58 | 1225 } |
60 | 1226 |
58 | 1227 Suffix suffix; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1228 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1229 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1230 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1231 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1232 case 'L': |
58 | 1233 if (suffix & Suffix.Long) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1234 break; |
60 | 1235 suffix |= Suffix.Long; |
1236 ++p; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1237 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1238 case 'u', 'U': |
58 | 1239 if (suffix & Suffix.Unsigned) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1240 break; |
60 | 1241 suffix |= Suffix.Unsigned; |
1242 ++p; | |
1243 continue; | |
58 | 1244 default: |
1245 break; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1246 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1247 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1248 } |
58 | 1249 |
1250 switch (suffix) | |
1251 { | |
1252 case Suffix.None: | |
1253 if (ulong_ & 0x8000000000000000) | |
1254 { | |
1255 if (isDecimal) | |
1256 error(MID.OverflowDecimalSign); | |
1257 t.type = TOK.Uint64; | |
1258 } | |
1259 else if (ulong_ & 0xFFFFFFFF00000000) | |
1260 t.type = TOK.Int64; | |
1261 else if (ulong_ & 0x80000000) | |
1262 t.type = isDecimal ? TOK.Int64 : TOK.Uint32; | |
1263 else | |
1264 t.type = TOK.Int32; | |
1265 break; | |
1266 case Suffix.Unsigned: | |
1267 if (ulong_ & 0xFFFFFFFF00000000) | |
1268 t.type = TOK.Uint64; | |
1269 else | |
1270 t.type = TOK.Uint32; | |
1271 break; | |
1272 case Suffix.Long: | |
1273 if (ulong_ & 0x8000000000000000) | |
1274 { | |
1275 if (isDecimal) | |
1276 error(MID.OverflowDecimalSign); | |
1277 t.type = TOK.Uint64; | |
1278 } | |
1279 else | |
1280 t.type = TOK.Int64; | |
1281 break; | |
1282 case Suffix.Unsigned | Suffix.Long: | |
1283 t.type = TOK.Uint64; | |
1284 break; | |
1285 default: | |
1286 assert(0); | |
1287 } | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1288 t.ulong_ = ulong_; |
15 | 1289 t.end = p; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1290 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1291 LscanReal: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1292 scanReal(t); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1293 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1294 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1295 |
63 | 1296 /* |
1297 FloatLiteral:= Float[fFL]?i? | |
1298 Float:= DecFloat | HexFloat | |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1299 DecFloat:= ([0-9][0-9_]*[.][0-9_]*DecExponent?) | [.][0-9][0-9_]*DecExponent? | [0-9][0-9_]*DecExponent |
63 | 1300 DecExponent:= [eE][+-]?[0-9][0-9_]* |
1301 HexFloat:= 0[xX](HexDigits[.]HexDigits | [.][0-9a-zA-Z]HexDigits? | HexDigits)HexExponent | |
1302 HexExponent:= [pP][+-]?[0-9][0-9_]* | |
1303 */ | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1304 void scanReal(ref Token t) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1305 { |
63 | 1306 if (*p == '.') |
1307 // This function was called by scan() or scanNumber(). | |
1308 while (isdigit(*++p) || *p == '_') {} | |
1309 else | |
1310 { | |
1311 // This function was called by scanNumber(). | |
1312 debug switch (*p) | |
1313 { | |
1314 case 'L': | |
1315 if (p[1] != 'i') | |
1316 assert(0); | |
1317 case 'i', 'f', 'F', 'e', 'E': break; | |
1318 default: assert(0); | |
1319 } | |
1320 } | |
1321 | |
1322 // Scan exponent. | |
1323 if (*p == 'e' || *p == 'E') | |
1324 { | |
1325 ++p; | |
1326 if (*p == '-' || *p == '+') | |
1327 ++p; | |
1328 if (!isdigit(*p)) | |
1329 error(MID.FloatExponentDigitExpected); | |
1330 else | |
64 | 1331 while (isdigit(*++p) || *p == '_') {} |
63 | 1332 } |
1333 | |
1334 // Copy string to buffer ignoring underscores. | |
1335 char[] buffer; | |
1336 char* end = p; | |
1337 p = t.start; | |
1338 do | |
1339 { | |
1340 if (*p == '_') | |
1341 { | |
1342 ++p; | |
1343 continue; | |
1344 } | |
1345 buffer ~= *p; | |
1346 ++p; | |
1347 } while (p != end) | |
1348 buffer ~= 0; | |
1349 finalizeFloat(t, buffer); | |
30 | 1350 } |
1351 | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1352 void scanHexReal(ref Token t) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1353 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1354 assert(*p == '.' || *p == 'i' || *p == 'p' || *p == 'P' || (*p == 'L' && p[1] == 'i')); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1355 MID mid; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1356 if (*p == '.') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1357 while (ishexad(*++p) || *p == '_') {} |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1358 if (*p != 'p' && *p != 'P') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1359 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1360 mid = MID.HexFloatExponentRequired; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1361 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1362 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1363 // Copy mantissa to a buffer ignoring underscores. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1364 char* end = p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1365 p = t.start; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1366 char[] buffer; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1367 do |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1368 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1369 if (*p == '_') |
63 | 1370 { |
1371 ++p; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1372 continue; |
63 | 1373 } |
1374 buffer ~= *p; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1375 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1376 } while (p != end) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1377 |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1378 assert(p == end && (*p == 'p' || *p == 'P')); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1379 // Scan and copy the exponent. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1380 buffer ~= 'p'; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1381 size_t bufflen = buffer.length; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1382 while (1) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1383 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1384 if (*++p == '_') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1385 continue; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1386 if (isdigit(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1387 buffer ~= *p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1388 else |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1389 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1390 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1391 // When the buffer length hasn't changed, no digits were copied. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1392 if (bufflen == buffer.length) { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1393 mid = MID.HexFloatMissingExpDigits; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1394 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1395 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1396 buffer ~= 0; // Terminate for C functions. |
63 | 1397 finalizeFloat(t, buffer); |
1398 return; | |
1399 Lerr: | |
1400 t.type = TOK.Float32; | |
1401 t.end = p; | |
1402 error(mid); | |
1403 } | |
1404 | |
1405 void finalizeFloat(ref Token t, string buffer) | |
1406 { | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1407 // Float number is well-formed. Check suffixes and do conversion. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1408 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1409 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1410 case 'f', 'F': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1411 t.type = TOK.Float32; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1412 t.float_ = strtof(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1413 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1414 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1415 case 'L': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1416 t.type = TOK.Float80; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1417 t.real_ = strtold(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1418 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1419 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1420 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1421 t.type = TOK.Float64; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1422 t.double_ = strtod(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1423 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1424 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1425 if (*p == 'i') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1426 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1427 ++p; |
63 | 1428 t.type += 3; // Switch to imaginary counterpart. |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1429 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1430 if (getErrno == ERANGE) |
63 | 1431 error(MID.OverflowFloatNumber); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1432 t.end = p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1433 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1434 |
30 | 1435 /// Scan special token: #line Integer [Filespec] EndOfLine |
360
b6a3755eba94
- Renamed scanSpecialToken() to scanSpecialTokenSequence().
aziz
parents:
350
diff
changeset
|
1436 void scanSpecialTokenSequence(ref Token t) |
30 | 1437 { |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1438 assert(*p == '#'); |
51 | 1439 |
323 | 1440 t.type = TOK.HashLine; |
1441 | |
30 | 1442 MID mid; |
1443 | |
323 | 1444 ++p; |
1445 if (p[0] != 'l' || p[1] != 'i' || p[2] != 'n' || p[3] != 'e') | |
51 | 1446 { |
370 | 1447 mid = MID.ExpectedIdentifierSTLine; |
51 | 1448 goto Lerr; |
1449 } | |
323 | 1450 p += 3; |
1451 | |
1452 enum State | |
1453 { Number, Filespec, End } | |
51 | 1454 |
323 | 1455 State state; |
1456 | |
1457 Loop: | |
1458 while (1) | |
51 | 1459 { |
323 | 1460 switch (*++p) |
1461 { | |
1462 case '\r': | |
1463 if (p[1] == '\n') | |
1464 ++p; | |
1465 case '\n', 0, _Z_: | |
1466 break Loop; | |
1467 case LS[0]: | |
1468 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) | |
1469 { | |
1470 ++p; ++p; | |
1471 break Loop; | |
1472 } | |
1473 goto default; | |
1474 default: | |
1475 if (isspace(*p)) | |
1476 continue; | |
1477 if (state == State.Number) | |
1478 { | |
1479 if (!isdigit(*p)) | |
1480 { | |
1481 mid = MID.ExpectedNumberAfterSTLine; | |
1482 goto Lerr; | |
1483 } | |
1484 t.line_num = new Token; | |
1485 scan(*t.line_num); | |
1486 --p; | |
1487 state = State.Filespec; | |
1488 } | |
1489 else if (state == State.Filespec) | |
1490 { | |
1491 if (*p != '"') | |
1492 { | |
1493 mid = MID.ExpectedFilespec; | |
1494 goto Lerr; | |
1495 } | |
1496 t.line_filespec = new Token; | |
1497 t.line_filespec.start = p; | |
1498 t.line_filespec.type = TOK.Filespec; | |
1499 while (1) | |
1500 { | |
1501 switch (*++p) | |
1502 { | |
1503 case '"': | |
1504 break; | |
1505 case LS[0]: | |
1506 if (!(p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))) | |
1507 goto default; | |
1508 case '\r', '\n', 0, _Z_: | |
1509 mid = MID.UnterminatedFilespec; | |
1510 t.line_filespec.end = p; | |
1511 goto Lerr; | |
1512 default: | |
1513 if (*p & 128) | |
1514 decodeUTF8(); | |
1515 continue; | |
1516 } | |
1517 break; // Exit loop. | |
1518 } | |
1519 auto start = t.line_filespec.start +1; // +1 skips '"' | |
1520 t.line_filespec.str = start[0 .. p - start]; | |
1521 t.line_filespec.end = p + 1; | |
1522 state = State.End; | |
1523 } | |
1524 else/+ if (state == State.End)+/ | |
1525 { | |
1526 mid = MID.UnterminatedSpecialToken; | |
1527 goto Lerr; | |
1528 } | |
1529 } | |
51 | 1530 } |
323 | 1531 |
1532 if (state == State.Number) | |
51 | 1533 { |
1534 mid = MID.ExpectedNumberAfterSTLine; | |
30 | 1535 goto Lerr; |
1536 } | |
1537 | |
371 | 1538 this.loc_old = this.loc; |
1539 this.loc_hline = t.line_num.uint_ - 1; | |
323 | 1540 if (t.line_filespec) |
1541 this.fileName = t.line_filespec.str; | |
1542 t.end = p; | |
30 | 1543 |
1544 return; | |
1545 Lerr: | |
323 | 1546 t.end = p; |
30 | 1547 error(mid); |
15 | 1548 } |
1549 | |
371 | 1550 uint errorLoc() |
1551 { | |
1552 // ∆loc + line_num_of(#line) | |
1553 return this.loc - this.loc_old + this.loc_hline; | |
1554 } | |
1555 | |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1556 dchar decodeUTF8() |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1557 { |
49 | 1558 assert(*p & 128, "check for ASCII char before calling decodeUTF8()."); |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1559 size_t idx; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1560 dchar d; |
49 | 1561 try |
1562 { | |
1563 d = std.utf.decode(p[0 .. end-p], idx); | |
1564 p += idx -1; | |
1565 } | |
1566 catch (UtfException e) | |
1567 { | |
1568 error(MID.InvalidUTF8Sequence); | |
1569 // Skip to next valid utf-8 sequence | |
1570 while (UTF8stride[*++p] != 0xFF) {} | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1571 --p; |
49 | 1572 } |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1573 return d; |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1574 } |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1575 |
28 | 1576 void loadKeywords() |
1577 { | |
1578 foreach(k; keywords) | |
1579 idtable[k.str] = k; | |
1580 } | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1581 /+ |
105 | 1582 struct State |
1583 { | |
1584 Lexer lexer; | |
209 | 1585 Token token; |
105 | 1586 char* scanPointer; |
209 | 1587 int loc; |
212 | 1588 string fileName; |
105 | 1589 size_t errorLen; |
212 | 1590 static State opCall(Lexer lx) |
105 | 1591 { |
1592 State s; | |
212 | 1593 s.lexer = lx; |
1594 s.token = lx.token; | |
1595 s.scanPointer = lx.p; | |
1596 s.loc = lx.loc; | |
1597 s.fileName = lx.fileName; | |
1598 s.errorLen = lx.errors.length; | |
105 | 1599 return s; |
1600 } | |
1601 void restore() | |
1602 { | |
1603 lexer.p = scanPointer; | |
209 | 1604 lexer.token = token; |
1605 lexer.loc = loc; | |
212 | 1606 lexer.fileName = fileName; |
105 | 1607 lexer.errors = lexer.errors[0..errorLen]; |
1608 } | |
1609 } | |
1610 | |
1611 State getState() | |
1612 { | |
212 | 1613 return State(this); |
105 | 1614 } |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1615 +/ |
105 | 1616 |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1617 private void scanNext(ref Token* t) |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1618 { |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1619 assert(t !is null); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1620 if (t.next) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1621 t = t.next; |
344 | 1622 else if (t != this.tail) |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1623 { |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1624 Token* new_t = new Token; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1625 scan(*new_t); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1626 new_t.prev = t; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1627 t.next = new_t; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1628 t = new_t; |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1629 } |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1630 } |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1631 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1632 void peek(ref Token* t) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1633 { |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1634 scanNext(t); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1635 } |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1636 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1637 TOK nextToken() |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1638 { |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1639 scanNext(this.token); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1640 return this.token.type; |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1641 } |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1642 |
66
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
1643 void error(MID id, ...) |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1644 { |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1645 // if (reportErrors) |
371 | 1646 errors ~= new Information(InfoType.Lexer, id, this.errorLoc, arguments(_arguments, _argptr)); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1647 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1648 |
207 | 1649 unittest |
1650 { | |
1651 string sourceText = "unittest { }"; | |
1652 auto lx = new Lexer(sourceText, null); | |
1653 | |
350 | 1654 Token* next = lx.head; |
207 | 1655 lx.peek(next); |
350 | 1656 assert(next.type == TOK.Unittest); |
207 | 1657 lx.peek(next); |
350 | 1658 assert(next.type == TOK.LBrace); |
207 | 1659 lx.peek(next); |
350 | 1660 assert(next.type == TOK.RBrace); |
207 | 1661 lx.peek(next); |
350 | 1662 assert(next.type == TOK.EOF); |
207 | 1663 writefln("end of peek() unittest"); |
1664 } | |
1665 | |
306 | 1666 Token* getTokens() |
4 | 1667 { |
306 | 1668 while (nextToken() != TOK.EOF) |
1669 {} | |
1670 return head; | |
3 | 1671 } |
48 | 1672 |
365 | 1673 static bool isNonReservedIdentifier(char[] ident) |
1674 { | |
1675 static Identifier[string] reserved_ids_table; | |
1676 if (reserved_ids_table is null) | |
1677 foreach(k; keywords) | |
1678 reserved_ids_table[k.str] = k; | |
1679 | |
1680 try | |
1681 foreach (dchar c; ident) | |
1682 if (!isident(c) && !isUniAlpha(c)) | |
1683 return false; | |
1684 catch (Exception e) | |
1685 return false; | |
1686 | |
1687 return !(ident in reserved_ids_table); | |
1688 } | |
1689 | |
48 | 1690 private void encodeUTF8(inout char[] str, dchar d) |
1691 { | |
1692 char[6] b; | |
1693 assert(d > 0x7F, "check for ASCII char before calling encodeUTF8()."); | |
1694 if (d < 0x800) | |
1695 { | |
1696 b[0] = 0xC0 | (d >> 6); | |
1697 b[1] = 0x80 | (d & 0x3F); | |
1698 str ~= b[0..2]; | |
1699 } | |
1700 else if (d < 0x10000) | |
1701 { | |
1702 b[0] = 0xE0 | (d >> 12); | |
1703 b[1] = 0x80 | ((d >> 6) & 0x3F); | |
1704 b[2] = 0x80 | (d & 0x3F); | |
1705 str ~= b[0..3]; | |
1706 } | |
1707 else if (d < 0x200000) | |
1708 { | |
1709 b[0] = 0xF0 | (d >> 18); | |
1710 b[1] = 0x80 | ((d >> 12) & 0x3F); | |
1711 b[2] = 0x80 | ((d >> 6) & 0x3F); | |
1712 b[3] = 0x80 | (d & 0x3F); | |
1713 str ~= b[0..4]; | |
1714 } | |
1715 else if (d < 0x4000000) | |
1716 { | |
1717 b[0] = 0xF8 | (d >> 24); | |
1718 b[1] = 0x80 | ((d >> 18) & 0x3F); | |
1719 b[2] = 0x80 | ((d >> 12) & 0x3F); | |
1720 b[3] = 0x80 | ((d >> 6) & 0x3F); | |
1721 b[4] = 0x80 | (d & 0x3F); | |
1722 str ~= b[0..5]; | |
1723 } | |
1724 else if (d < 0x80000000) | |
1725 { | |
1726 b[0] = 0xFC | (d >> 30); | |
1727 b[1] = 0x80 | ((d >> 24) & 0x3F); | |
1728 b[2] = 0x80 | ((d >> 18) & 0x3F); | |
1729 b[3] = 0x80 | ((d >> 12) & 0x3F); | |
1730 b[4] = 0x80 | ((d >> 6) & 0x3F); | |
1731 b[5] = 0x80 | (d & 0x3F); | |
1732 str ~= b[0..6]; | |
1733 } | |
1734 else | |
1735 error(MID.InvalidUnicodeCharacter); | |
1736 } | |
5 | 1737 } |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1738 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1739 unittest |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1740 { |
350 | 1741 writefln("Testing Lexer."); |
40 | 1742 string[] toks = [ |
1743 ">", ">=", ">>", ">>=", ">>>", ">>>=", "<", "<=", "<>", | |
1744 "<>=", "<<", "<<=", "!", "!<", "!>", "!<=", "!>=", "!<>", | |
1745 "!<>=", ".", "..", "...", "&", "&&", "&=", "+", "++", | |
1746 "+=", "-", "--", "-=", "=", "==", "~", "~=", "*", | |
1747 "*=", "/", "/=", "^", "^=", "%", "%=", "(", ")", | |
1748 "[", "]", "{", "}", ":", ";", "?", ",", "$" | |
1749 ]; | |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1750 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1751 char[] src; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1752 |
40 | 1753 foreach (op; toks) |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1754 src ~= op ~ " "; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1755 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1756 auto lx = new Lexer(src, ""); |
350 | 1757 auto token = lx.getTokens(); |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1758 |
350 | 1759 uint i; |
1760 assert(token == lx.head); | |
1761 token = token.next; | |
1762 do | |
1763 { | |
1764 assert(i < toks.length); | |
1765 assert(token.srcText == toks[i], std.string.format("Scanned '%s' but expected '%s'", token.srcText, toks[i])); | |
1766 ++i; | |
1767 token = token.next; | |
1768 } while (token.type != TOK.EOF) | |
41 | 1769 } |
55 | 1770 |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1771 unittest |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1772 { |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1773 // Numbers unittest |
68
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1774 // 0L 0ULi 0_L 0_UL 0x0U 0x0p2 0_Fi 0_e2 0_F 0_i |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1775 // 0u 0U 0uL 0UL 0L 0LU 0Lu |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1776 // 0Li 0f 0F 0fi 0Fi 0i |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1777 // 0b_1_LU 0b1000u |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1778 // 0x232Lu |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1779 } |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1780 |
55 | 1781 /// ASCII character properties table. |
1782 static const int ptable[256] = [ | |
1783 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0, | |
1784 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1785 32, 0, 0x2200, 0, 0, 0, 0, 0x2700, 0, 0, 0, 0, 0, 0, 0, 0, | |
1786 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0x3f00, | |
1787 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8, | |
1788 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0x5c00, 0, 0,16, | |
1789 0, 0x70c, 0x80c,12,12,12, 0xc0c, 8, 8, 8, 8, 8, 8, 8, 0xa08, 8, | |
1790 8, 8, 0xd08, 8, 0x908, 8, 0xb08, 8, 8, 8, 8, 0, 0, 0, 0, 0, | |
1791 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1792 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1793 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1794 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1795 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1796 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1797 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1798 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 | |
1799 ]; | |
1800 | |
1801 enum CProperty | |
1802 { | |
1803 Octal = 1, | |
1804 Digit = 1<<1, | |
1805 Hex = 1<<2, | |
1806 Alpha = 1<<3, | |
1807 Underscore = 1<<4, | |
1808 Whitespace = 1<<5 | |
1809 } | |
1810 | |
1811 const uint EVMask = 0xFF00; // Bit mask for escape value | |
1812 | |
1813 private alias CProperty CP; | |
1814 int isoctal(char c) { return ptable[c] & CP.Octal; } | |
1815 int isdigit(char c) { return ptable[c] & CP.Digit; } | |
1816 int ishexad(char c) { return ptable[c] & CP.Hex; } | |
1817 int isalpha(char c) { return ptable[c] & CP.Alpha; } | |
1818 int isalnum(char c) { return ptable[c] & (CP.Alpha | CP.Digit); } | |
1819 int isidbeg(char c) { return ptable[c] & (CP.Alpha | CP.Underscore); } | |
1820 int isident(char c) { return ptable[c] & (CP.Alpha | CP.Underscore | CP.Digit); } | |
1821 int isspace(char c) { return ptable[c] & CP.Whitespace; } | |
1822 int char2ev(char c) { return ptable[c] >> 8; /*(ptable[c] & EVMask) >> 8;*/ } | |
1823 | |
1824 version(gen_ptable) | |
1825 static this() | |
1826 { | |
1827 alias ptable p; | |
1828 // Initialize character properties table. | |
1829 for (int i; i < p.length; ++i) | |
1830 { | |
1831 p[i] = 0; | |
1832 if ('0' <= i && i <= '7') | |
1833 p[i] |= CP.Octal; | |
1834 if ('0' <= i && i <= '9') | |
1835 p[i] |= CP.Digit; | |
1836 if (isdigit(i) || 'a' <= i && i <= 'f' || 'A' <= i && i <= 'F') | |
1837 p[i] |= CP.Hex; | |
1838 if ('a' <= i && i <= 'z' || 'A' <= i && i <= 'Z') | |
1839 p[i] |= CP.Alpha; | |
1840 if (i == '_') | |
1841 p[i] |= CP.Underscore; | |
1842 if (i == ' ' || i == '\t' || i == '\v' || i == '\f') | |
1843 p[i] |= CP.Whitespace; | |
1844 } | |
1845 // Store escape sequence values in second byte. | |
1846 assert(CProperty.max <= ubyte.max, "character property flags and escape value byte overlap."); | |
1847 p['\''] |= 39 << 8; | |
1848 p['"'] |= 34 << 8; | |
1849 p['?'] |= 63 << 8; | |
1850 p['\\'] |= 92 << 8; | |
1851 p['a'] |= 7 << 8; | |
1852 p['b'] |= 8 << 8; | |
1853 p['f'] |= 12 << 8; | |
1854 p['n'] |= 10 << 8; | |
1855 p['r'] |= 13 << 8; | |
1856 p['t'] |= 9 << 8; | |
1857 p['v'] |= 11 << 8; | |
1858 // Print a formatted array literal. | |
1859 char[] array = "[\n"; | |
1860 for (int i; i < p.length; ++i) | |
1861 { | |
1862 int c = p[i]; | |
1863 array ~= std.string.format(c>255?" 0x%x,":"%2d,", c, ((i+1) % 16) ? "":"\n"); | |
1864 } | |
1865 array[$-2..$] = "\n]"; | |
1866 writefln(array); | |
1867 } |