Mercurial > projects > dil
annotate trunk/src/Lexer.d @ 239:7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
- Tokens are stored in a doubly linked list by the Lexer now.
- Added method scanNext() which can be used by peek() and nextToken().
- Saving 'token' in Parser.try_().
author | aziz |
---|---|
date | Mon, 30 Jul 2007 12:36:04 +0000 |
parents | 043722eed46e |
children | 32d354584b28 |
rev | line source |
---|---|
0 | 1 /++ |
2 Author: Aziz Köksal | |
3 License: GPL2 | |
4 +/ | |
5 module Lexer; | |
3 | 6 import Token; |
69
24db7c5522d5
- Added module Information for compiler messages like warnings, info and errors to the user.
aziz
parents:
68
diff
changeset
|
7 import Information; |
28 | 8 import Keywords; |
9 import Identifier; | |
54 | 10 import Messages; |
2 | 11 import std.stdio; |
4 | 12 import std.utf; |
13 import std.uni; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
14 import std.c.stdlib; |
66
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
15 import std.string; |
0 | 16 |
11 | 17 const char[3] LS = \u2028; |
18 const char[3] PS = \u2029; | |
19 | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
20 const dchar LSd = 0x2028; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
21 const dchar PSd = 0x2029; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
22 |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
23 const uint _Z_ = 26; /// Control+Z |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
24 |
0 | 25 class Lexer |
26 { | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
27 Token* head; /// The head of the doubly linked token list. |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
28 Token* token; /// Points to the current token in the token list. |
30 | 29 string text; |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
30 char* p; /// Points to the current character in the source text. |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
31 char* end; /// Points one character past the end of the source text. |
4 | 32 |
17 | 33 uint loc = 1; /// line of code |
34 | |
30 | 35 char[] fileName; |
36 | |
69
24db7c5522d5
- Added module Information for compiler messages like warnings, info and errors to the user.
aziz
parents:
68
diff
changeset
|
37 Information[] errors; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
38 |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
39 // bool reportErrors; |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
40 |
28 | 41 Identifier[string] idtable; |
42 | |
30 | 43 this(string text, string fileName) |
4 | 44 { |
30 | 45 this.fileName = fileName; |
46 | |
4 | 47 this.text = text; |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
48 if (text[$-1] != 0) |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
49 { |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
50 this.text.length = this.text.length + 1; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
51 this.text[$-1] = 0; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
52 } |
4 | 53 |
54 this.p = this.text.ptr; | |
55 this.end = this.p + this.text.length; | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
56 // this.reportErrors = true; |
28 | 57 loadKeywords(); |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
58 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
59 this.head = new Token; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
60 this.head.type = TOK.HEAD; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
61 this.token = this.head; |
4 | 62 } |
63 | |
64 public void scan(out Token t) | |
207 | 65 in |
3 | 66 { |
207 | 67 assert(text.ptr <= p && p < end); |
68 } | |
69 out | |
70 { | |
71 assert(text.ptr <= t.start && t.start < end); | |
72 assert(text.ptr < t.end && t.end <= end, std.string.format(t.type)); | |
73 } | |
74 body | |
75 { | |
10 | 76 uint c = *p; |
4 | 77 |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
78 while (1) |
4 | 79 { |
5 | 80 t.start = p; |
16 | 81 |
212 | 82 if (c == 0 || c == _Z_) |
5 | 83 { |
212 | 84 assert(*p == 0 || *p == _Z_); |
5 | 85 t.type = TOK.EOF; |
17 | 86 t.end = p; |
212 | 87 assert(t.start == t.end); |
5 | 88 return; |
89 } | |
4 | 90 |
17 | 91 if (c == '\n') |
92 { | |
93 c = *++p; | |
94 ++loc; | |
95 continue; | |
96 } | |
97 else if (c == '\r') | |
98 { | |
99 c = *++p; | |
100 if (c != '\n') | |
101 ++loc; | |
102 continue; | |
103 } | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
104 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
105 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
106 p += 3; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
107 c = *p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
108 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
109 } |
17 | 110 |
13 | 111 if (isidbeg(c)) |
5 | 112 { |
33 | 113 if (c == 'r' && p[1] == '"' && ++p) |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
114 return scanRawStringLiteral(t); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
115 if (c == 'x' && p[1] == '"') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
116 return scanHexStringLiteral(t); |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
117 Lidentifier: |
5 | 118 do |
119 { c = *++p; } | |
49 | 120 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8())) |
28 | 121 |
5 | 122 t.end = p; |
28 | 123 |
65 | 124 string str = t.srcText; |
28 | 125 Identifier* id = str in idtable; |
126 | |
127 if (!id) | |
128 { | |
129 idtable[str] = Identifier.Identifier(TOK.Identifier, str); | |
130 id = str in idtable; | |
131 } | |
132 assert(id); | |
133 t.type = id.type; | |
5 | 134 return; |
135 } | |
136 | |
15 | 137 if (isdigit(c)) |
138 return scanNumber(t); | |
139 | |
8 | 140 if (c == '/') |
5 | 141 { |
8 | 142 c = *++p; |
14 | 143 switch(c) |
5 | 144 { |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
145 case '=': |
14 | 146 ++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
147 t.type = TOK.DivAssign; |
14 | 148 t.end = p; |
149 return; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
150 case '+': |
8 | 151 uint level = 1; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
152 while (1) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
153 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
154 c = *++p; |
42 | 155 LswitchNC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
156 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
157 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
158 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
159 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
160 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
161 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
162 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
163 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
164 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
165 error(MID.UnterminatedNestedComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
166 goto LreturnNC; |
42 | 167 default: |
168 } | |
169 | |
170 c <<= 8; | |
171 c |= *++p; | |
172 switch (c) | |
173 { | |
174 case 0x2F2B: // /+ | |
175 ++level; | |
176 continue; | |
177 case 0x2B2F: // +/ | |
178 if (--level == 0) | |
179 { | |
180 ++p; | |
181 LreturnNC: | |
182 t.type = TOK.Comment; | |
183 t.end = p; | |
184 return; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
185 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
186 continue; |
42 | 187 case 0xE280: // LS[0..1] || PS[0..1] |
188 if (p[1] == LS[2] || p[1] == PS[2]) | |
189 { | |
190 ++loc; | |
191 ++p; | |
192 } | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
193 continue; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
194 default: |
42 | 195 c &= char.max; |
196 goto LswitchNC; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
197 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
198 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
199 case '*': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
200 while (1) |
7 | 201 { |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
202 c = *++p; |
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
203 LswitchBC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
204 switch (c) |
8 | 205 { |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
206 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
207 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
208 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
209 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
210 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
211 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
212 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
213 error(MID.UnterminatedBlockComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
214 goto LreturnBC; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
215 default: |
8 | 216 } |
41 | 217 |
218 c <<= 8; | |
219 c |= *++p; | |
220 switch (c) | |
221 { | |
222 case 0x2A2F: // */ | |
223 ++p; | |
224 LreturnBC: | |
225 t.type = TOK.Comment; | |
226 t.end = p; | |
227 return; | |
228 case 0xE280: // LS[0..1] || PS[0..1] | |
229 if (p[1] == LS[2] || p[1] == PS[2]) | |
230 { | |
231 ++loc; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
232 ++p; |
41 | 233 } |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
234 continue; |
41 | 235 default: |
236 c &= char.max; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
237 goto LswitchBC; |
41 | 238 } |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
239 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
240 assert(0); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
241 case '/': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
242 while (1) |
7 | 243 { |
8 | 244 c = *++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
245 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
246 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
247 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
248 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
249 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
250 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
251 case 0, _Z_: |
11 | 252 break; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
253 case LS[0]: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
254 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
255 break; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
256 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
257 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
258 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
259 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
260 t.type = TOK.Comment; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
261 t.end = p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
262 return; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
263 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
264 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
265 t.type = TOK.Div; |
10 | 266 t.end = p; |
267 return; | |
268 } | |
5 | 269 } |
270 | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
271 switch (c) |
20 | 272 { |
48 | 273 case '\'': |
274 return scanCharacterLiteral(t); | |
275 case '`': | |
276 return scanRawStringLiteral(t); | |
277 case '"': | |
278 return scanNormalStringLiteral(t); | |
279 case '\\': | |
280 char[] buffer; | |
281 do | |
282 { | |
283 ++p; | |
284 c = scanEscapeSequence(); | |
285 if (c < 128) | |
286 buffer ~= c; | |
287 else | |
288 encodeUTF8(buffer, c); | |
289 } while (*p == '\\') | |
290 buffer ~= 0; | |
291 t.type = TOK.String; | |
292 t.str = buffer; | |
293 t.end = p; | |
294 return; | |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
295 case '>': /* > >= >> >>= >>> >>>= */ |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
296 c = *++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
297 switch (c) |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
298 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
299 case '=': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
300 t.type = TOK.GreaterEqual; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
301 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
302 case '>': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
303 if (p[1] == '>') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
304 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
305 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
306 if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
307 { ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
308 t.type = TOK.URShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
309 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
310 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
311 t.type = TOK.URShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
312 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
313 else if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
314 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
315 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
316 t.type = TOK.RShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
317 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
318 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
319 t.type = TOK.RShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
320 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
321 default: |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
322 t.type = TOK.Greater; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
323 goto Lcommon2; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
324 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
325 assert(0); |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
326 case '<': /* < <= <> <>= << <<= */ |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
327 c = *++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
328 switch (c) |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
329 { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
330 case '=': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
331 t.type = TOK.LessEqual; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
332 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
333 case '<': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
334 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
335 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
336 t.type = TOK.LShiftAssign; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
337 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
338 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
339 t.type = TOK.LShift; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
340 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
341 case '>': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
342 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
343 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
344 t.type = TOK.LorEorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
345 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
346 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
347 t.type = TOK.LorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
348 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
349 default: |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
350 t.type = TOK.Less; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
351 goto Lcommon2; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
352 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
353 assert(0); |
37
7f3bcb97d017
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
36
diff
changeset
|
354 case '!': /* ! !< !> !<= !>= !<> !<>= */ |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
355 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
356 switch (c) |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
357 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
358 case '<': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
359 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
360 if (c == '>') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
361 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
362 if (p[1] == '=') { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
363 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
364 t.type = TOK.Unordered; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
365 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
366 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
367 t.type = TOK.UorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
368 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
369 else if (c == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
370 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
371 t.type = TOK.UorG; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
372 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
373 else { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
374 t.type = TOK.UorGorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
375 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
376 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
377 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
378 case '>': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
379 if (p[1] == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
380 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
381 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
382 t.type = TOK.UorL; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
383 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
384 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
385 t.type = TOK.UorLorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
386 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
387 case '=': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
388 t.type = TOK.NotEqual; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
389 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
390 default: |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
391 t.type = TOK.Not; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
392 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
393 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
394 assert(0); |
63 | 395 case '.': /* . .[0-9] .. ... */ |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
396 if (p[1] == '.') |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
397 { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
398 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
399 if (p[1] == '.') { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
400 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
401 t.type = TOK.Ellipses; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
402 } |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
403 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
404 t.type = TOK.Slice; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
405 } |
63 | 406 else if (isdigit(p[1])) |
407 { | |
408 return scanReal(t); | |
409 } | |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
410 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
411 t.type = TOK.Dot; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
412 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
413 case '|': /* | || |= */ |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
414 c = *++p; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
415 if (c == '=') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
416 t.type = TOK.OrAssign; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
417 else if (c == '|') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
418 t.type = TOK.OrLogical; |
26 | 419 else { |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
420 t.type = TOK.OrBinary; |
26 | 421 goto Lcommon2; |
422 } | |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
423 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
424 case '&': /* & && &= */ |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
425 c = *++p; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
426 if (c == '=') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
427 t.type = TOK.AndAssign; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
428 else if (c == '&') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
429 t.type = TOK.AndLogical; |
26 | 430 else { |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
431 t.type = TOK.AndBinary; |
26 | 432 goto Lcommon2; |
433 } | |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
434 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
435 case '+': /* + ++ += */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
436 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
437 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
438 t.type = TOK.PlusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
439 else if (c == '+') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
440 t.type = TOK.PlusPlus; |
26 | 441 else { |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
442 t.type = TOK.Plus; |
26 | 443 goto Lcommon2; |
444 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
445 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
446 case '-': /* - -- -= */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
447 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
448 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
449 t.type = TOK.MinusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
450 else if (c == '-') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
451 t.type = TOK.MinusMinus; |
26 | 452 else { |
453 t.type = TOK.Minus; | |
454 goto Lcommon2; | |
455 } | |
456 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
457 case '=': /* = == */ |
26 | 458 if (p[1] == '=') { |
459 ++p; | |
460 t.type = TOK.Equal; | |
461 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
462 else |
26 | 463 t.type = TOK.Assign; |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
464 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
465 case '~': /* ~ ~= */ |
27 | 466 if (p[1] == '=') { |
467 ++p; | |
468 t.type = TOK.CatAssign; | |
469 } | |
470 else | |
471 t.type = TOK.Tilde; | |
472 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
473 case '*': /* * *= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
474 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
475 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
476 t.type = TOK.MulAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
477 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
478 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
479 t.type = TOK.Mul; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
480 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
481 case '^': /* ^ ^= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
482 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
483 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
484 t.type = TOK.XorAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
485 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
486 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
487 t.type = TOK.Xor; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
488 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
489 case '%': /* % %= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
490 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
491 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
492 t.type = TOK.ModAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
493 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
494 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
495 t.type = TOK.Mod; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
496 goto Lcommon; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
497 // Single character tokens: |
20 | 498 case '(': |
499 t.type = TOK.LParen; | |
500 goto Lcommon; | |
501 case ')': | |
502 t.type = TOK.RParen; | |
503 goto Lcommon; | |
504 case '[': | |
505 t.type = TOK.LBracket; | |
506 goto Lcommon; | |
507 case ']': | |
508 t.type = TOK.RBracket; | |
509 goto Lcommon; | |
510 case '{': | |
511 t.type = TOK.LBrace; | |
512 goto Lcommon; | |
513 case '}': | |
514 t.type = TOK.RBrace; | |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
515 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
516 case ':': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
517 t.type = TOK.Colon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
518 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
519 case ';': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
520 t.type = TOK.Semicolon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
521 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
522 case '?': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
523 t.type = TOK.Question; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
524 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
525 case ',': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
526 t.type = TOK.Comma; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
527 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
528 case '$': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
529 t.type = TOK.Dollar; |
20 | 530 Lcommon: |
531 ++p; | |
26 | 532 Lcommon2: |
20 | 533 t.end = p; |
534 return; | |
30 | 535 case '#': |
536 scanSpecialToken(); | |
51 | 537 c = *p; |
538 continue; | |
20 | 539 default: |
540 } | |
541 | |
49 | 542 if (c & 128 && isUniAlpha(decodeUTF8())) |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
543 goto Lidentifier; |
5 | 544 c = *++p; |
4 | 545 } |
3 | 546 } |
4 | 547 |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
548 void scanNormalStringLiteral(ref Token t) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
549 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
550 assert(*p == '"'); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
551 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
552 char[] buffer; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
553 t.type = TOK.String; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
554 while (1) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
555 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
556 switch (*p) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
557 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
558 case '"': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
559 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
560 Lreturn: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
561 buffer ~= 0; |
49 | 562 t.str = buffer; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
563 t.pf = scanPostfix(); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
564 t.end = p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
565 return; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
566 case '\\': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
567 ++p; |
49 | 568 dchar d = scanEscapeSequence(); |
569 if (d < 128) | |
570 buffer ~= d; | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
571 else |
49 | 572 encodeUTF8(buffer, d); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
573 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
574 case '\r': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
575 if (p[1] == '\n') |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
576 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
577 case '\n': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
578 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
579 ++loc; |
221 | 580 buffer ~= '\n'; // Convert EndOfLine to \n. |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
581 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
582 case 0, _Z_: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
583 error(MID.UnterminatedString); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
584 goto Lreturn; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
585 default: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
586 if (*p & 128) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
587 { |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
588 // char* begin = p; |
49 | 589 dchar d = decodeUTF8(); |
590 if (d == LSd || d == PSd) | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
591 goto case '\n'; |
49 | 592 |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
593 // We don't copy per pointer because we might include |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
594 // invalid, skipped utf-8 sequences. See decodeUTF8(). |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
595 // ++p; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
596 // buffer ~= begin[0 .. p - begin]; |
221 | 597 ++p; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
598 encodeUTF8(buffer, d); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
599 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
600 } |
223 | 601 // Copy ASCII character. |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
602 buffer ~= *p++; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
603 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
604 } |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
605 assert(0); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
606 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
607 |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
608 void scanCharacterLiteral(ref Token t) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
609 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
610 assert(*p == '\''); |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
611 MID id = MID.UnterminatedCharacterLiteral; |
46 | 612 ++p; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
613 TOK type = TOK.CharLiteral; |
46 | 614 switch (*p) |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
615 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
616 case '\\': |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
617 ++p; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
618 switch (*p) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
619 { |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
620 case 'u': |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
621 type = TOK.WCharLiteral; break; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
622 case 'U': |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
623 type = TOK.DCharLiteral; break; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
624 default: |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
625 } |
46 | 626 t.dchar_ = scanEscapeSequence(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
627 break; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
628 case '\'': |
46 | 629 ++p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
630 id = MID.EmptyCharacterLiteral; |
46 | 631 case '\n', '\r', 0, _Z_: |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
632 goto Lerr; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
633 default: |
46 | 634 uint c = *p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
635 if (c & 128) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
636 { |
49 | 637 c = decodeUTF8(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
638 if (c == LSd || c == PSd) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
639 goto Lerr; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
640 if (c <= 0xFFFF) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
641 type = TOK.WCharLiteral; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
642 else |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
643 type = TOK.DCharLiteral; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
644 } |
46 | 645 t.dchar_ = c; |
646 ++p; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
647 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
648 |
46 | 649 if (*p == '\'') |
650 ++p; | |
651 else | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
652 Lerr: |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
653 error(id); |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
654 t.type = type; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
655 t.end = p; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
656 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
657 |
33 | 658 char scanPostfix() |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
659 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
660 switch (*p) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
661 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
662 case 'c': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
663 case 'w': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
664 case 'd': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
665 return *p++; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
666 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
667 return 0; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
668 } |
112 | 669 assert(0); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
670 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
671 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
672 void scanRawStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
673 { |
33 | 674 uint delim = *p; |
675 assert(delim == '`' || delim == '"' && p[-1] == 'r'); | |
676 t.type = TOK.String; | |
677 char[] buffer; | |
678 uint c; | |
679 while (1) | |
680 { | |
681 c = *++p; | |
682 switch (c) | |
683 { | |
684 case '\r': | |
685 if (p[1] == '\n') | |
686 ++p; | |
687 c = '\n'; // Convert '\r' and '\r\n' to '\n' | |
688 case '\n': | |
689 ++loc; | |
52
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
690 break; |
33 | 691 case '`': |
692 case '"': | |
693 if (c == delim) | |
694 { | |
695 ++p; | |
696 t.pf = scanPostfix(); | |
697 Lreturn: | |
698 t.str = buffer ~ '\0'; | |
699 t.end = p; | |
700 return; | |
701 } | |
702 break; | |
703 case LS[0]: | |
704 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) | |
705 { | |
52
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
706 c = '\n'; |
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
707 ++p; ++p; |
33 | 708 ++loc; |
709 } | |
710 break; | |
711 case 0, _Z_: | |
712 if (delim == 'r') | |
713 error(MID.UnterminatedRawString); | |
714 else | |
715 error(MID.UnterminatedBackQuoteString); | |
716 goto Lreturn; | |
717 default: | |
718 } | |
719 buffer ~= c; // copy character to buffer | |
720 } | |
721 assert(0); | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
722 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
723 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
724 void scanHexStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
725 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
726 assert(p[0] == 'x' && p[1] == '"'); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
727 t.type = TOK.String; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
728 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
729 uint c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
730 ubyte[] buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
731 ubyte h; // hex number |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
732 uint n; // number of hex digits |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
733 |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
734 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
735 while (1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
736 { |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
737 c = *++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
738 switch (c) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
739 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
740 case '"': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
741 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
742 if (n & 1) |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
743 error(MID.OddNumberOfDigitsInHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
744 t.pf = scanPostfix(); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
745 Lreturn: |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
746 buffer ~= 0; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
747 t.str = cast(string) buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
748 t.end = p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
749 return; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
750 case '\r': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
751 if (p[1] == '\n') |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
752 ++p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
753 case '\n': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
754 ++loc; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
755 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
756 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
757 if (ishexad(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
758 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
759 if (c <= '9') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
760 c -= '0'; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
761 else if (c <= 'F') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
762 c -= 'A' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
763 else |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
764 c -= 'a' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
765 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
766 if (n & 1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
767 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
768 h <<= 4; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
769 h |= c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
770 buffer ~= h; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
771 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
772 else |
112 | 773 h = cast(ubyte)c; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
774 ++n; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
775 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
776 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
777 else if (isspace(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
778 continue; |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
779 |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
780 if (c >= 128) |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
781 { |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
782 c = decodeUTF8(); |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
783 if (c == LSd || c == PSd) |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
784 { |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
785 ++p; ++p; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
786 ++loc; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
787 continue; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
788 } |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
789 } |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
790 else if (c == 0 || c == _Z_) |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
791 { |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
792 error(MID.UnterminatedHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
793 t.pf = 0; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
794 goto Lreturn; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
795 } |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
796 error(MID.NonHexCharInHexString, cast(dchar)c); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
797 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
798 } |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
799 assert(0); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
800 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
801 |
45 | 802 dchar scanEscapeSequence() |
803 { | |
804 uint c = char2ev(*p); | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
805 if (c) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
806 { |
46 | 807 ++p; |
45 | 808 return c; |
46 | 809 } |
45 | 810 uint digits = 2; |
811 | |
812 switch (*p) | |
813 { | |
814 case 'x': | |
815 c = 0; | |
816 while (1) | |
817 { | |
818 ++p; | |
819 if (ishexad(*p)) | |
820 { | |
821 c *= 16; | |
822 if (*p <= '9') | |
48 | 823 c += *p - '0'; |
45 | 824 else if (*p <= 'F') |
48 | 825 c += *p - 'A' + 10; |
45 | 826 else |
48 | 827 c += *p - 'a' + 10; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
828 |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
829 if (!--digits) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
830 { |
48 | 831 ++p; |
45 | 832 break; |
48 | 833 } |
45 | 834 } |
835 else | |
836 { | |
837 error(MID.InsufficientHexDigits); | |
838 break; | |
839 } | |
840 } | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
841 if (!isValidDchar(c)) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
842 error(MID.InvalidUnicodeCharacter); |
45 | 843 break; |
844 case 'u': | |
845 digits = 4; | |
846 goto case 'x'; | |
847 case 'U': | |
848 digits = 8; | |
849 goto case 'x'; | |
850 default: | |
48 | 851 if (isoctal(*p)) |
45 | 852 { |
48 | 853 c = 0; |
854 c += *p - '0'; | |
855 ++p; | |
856 if (!isoctal(*p)) | |
857 return c; | |
858 c *= 8; | |
859 c += *p - '0'; | |
860 ++p; | |
861 if (!isoctal(*p)) | |
862 return c; | |
863 c *= 8; | |
864 c += *p - '0'; | |
865 ++p; | |
866 } | |
867 else if(*p == '&') | |
868 { | |
869 if (isalpha(*++p)) | |
45 | 870 { |
48 | 871 while (1) |
872 { | |
873 if (isalnum(*++p)) | |
874 continue; | |
875 if (*p == ';') { | |
876 // TODO: convert entity to unicode codepoint. | |
877 ++p; | |
878 break; | |
879 } | |
880 else { | |
881 error(MID.UnterminatedHTMLEntity); | |
882 break; | |
883 } | |
45 | 884 } |
885 } | |
48 | 886 else |
887 error(MID.InvalidBeginHTMLEntity); | |
45 | 888 } |
889 else | |
48 | 890 error(MID.UndefinedEscapeSequence); |
45 | 891 } |
892 | |
893 return c; | |
894 } | |
895 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
896 /* |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
897 IntegerLiteral:= (Dec|Hex|Bin|Oct)Suffix? |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
898 Dec:= (0|[1-9][0-9_]*) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
899 Hex:= 0[xX] HexDigits |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
900 Bin:= 0[bB][01_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
901 Oct:= 0[0-7_]+ |
68
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
902 Suffix:= (L[uU]?|[uU]L?) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
903 HexDigits:= [0-9a-zA-Z_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
904 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
905 Invalid: "0b_", "0x_", "._" |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
906 */ |
15 | 907 void scanNumber(ref Token t) |
908 { | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
909 ulong ulong_; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
910 bool overflow; |
58 | 911 bool isDecimal; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
912 size_t digits; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
913 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
914 if (*p != '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
915 goto LscanInteger; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
916 ++p; // skip zero |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
917 // check for xX bB ... |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
918 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
919 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
920 case 'x','X': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
921 goto LscanHex; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
922 case 'b','B': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
923 goto LscanBin; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
924 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
925 if (p[1] == 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
926 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
927 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
928 if (p[1] == '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
929 break; |
58 | 930 case 'i','f','F', 'e', 'E': // Imaginary and float literal suffix |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
931 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
932 case '_': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
933 ++p; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
934 goto LscanOct; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
935 default: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
936 if (isoctal(*p)) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
937 goto LscanOct; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
938 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
939 |
58 | 940 ulong_ = p[-1]; |
941 isDecimal = true; | |
942 goto Lfinalize; | |
943 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
944 LscanInteger: |
58 | 945 isDecimal = true; |
63 | 946 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
947 { |
63 | 948 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
949 continue; |
63 | 950 if (!isdigit(*p)) |
951 break; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
952 if (ulong_ < ulong.max/10 || (ulong_ == ulong.max/10 && *p <= '5')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
953 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
954 ulong_ *= 10; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
955 ulong_ += *p - '0'; |
58 | 956 continue; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
957 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
958 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
959 overflow = true; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
960 while (isdigit(*++p)) {} |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
961 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
962 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
963 |
61 | 964 // The number could be a float, so check overflow below. |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
965 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
966 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
967 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
968 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
969 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
970 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
971 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
972 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
973 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
974 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
975 goto LscanReal; |
58 | 976 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
977 } |
58 | 978 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
979 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
980 error(MID.OverflowDecimalNumber); |
58 | 981 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
982 assert((isdigit(p[-1]) || p[-1] == '_') && !isdigit(*p) && *p != '_'); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
983 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
984 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
985 LscanHex: |
61 | 986 assert(digits == 0); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
987 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
988 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
989 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
990 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
991 if (!ishexad(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
992 break; |
61 | 993 ++digits; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
994 ulong_ *= 16; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
995 if (*p <= '9') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
996 ulong_ += *p - '0'; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
997 else if (*p <= 'F') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
998 ulong_ += *p - 'A' + 10; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
999 else |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1000 ulong_ += *p - 'a' + 10; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1001 } |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1002 |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1003 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1004 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1005 case '.': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1006 if (p[1] != '.') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1007 goto LscanHexReal; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1008 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1009 case 'L': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1010 if (p[1] != 'i') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1011 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1012 case 'i', 'p', 'P': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1013 goto LscanHexReal; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1014 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1015 } |
61 | 1016 if (digits == 0) |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1017 error(MID.NoDigitsInHexNumber); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1018 else if (digits > 16) |
61 | 1019 { |
1020 // Overflow: skip following digits. | |
1021 error(MID.OverflowHexNumber); | |
1022 while (ishexad(*++p)) {} | |
1023 } | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1024 goto Lfinalize; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1025 LscanHexReal: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1026 return scanHexReal(t); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1027 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1028 LscanBin: |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1029 assert(digits == 0); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1030 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1031 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1032 if (*++p == '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1033 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1034 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1035 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1036 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1037 if (*p == '1') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1038 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1039 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1040 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1041 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1042 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1043 if (*p == '_') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1044 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1045 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1046 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1047 |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1048 if (digits == 0) |
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1049 error(MID.NoDigitsInBinNumber); |
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1050 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1051 if (digits > 64) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1052 error(MID.OverflowBinaryNumber); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1053 assert((p[-1] == '0' || p[-1] == '1' || p[-1] == '_') && !(*p == '0' || *p == '1' || *p == '_')); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1054 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1055 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1056 LscanOct: |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1057 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1058 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1059 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1060 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1061 if (!isoctal(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1062 break; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1063 if (ulong_ < ulong.max/2 || (ulong_ == ulong.max/2 && *p <= '1')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1064 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1065 ulong_ *= 8; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1066 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1067 ++p; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1068 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1069 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1070 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1071 overflow = true; |
63 | 1072 while (isdigit(*++p)) {} |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1073 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1074 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1075 |
63 | 1076 bool hasDecimalDigits; |
1077 if (isdigit(*p)) | |
1078 { | |
1079 hasDecimalDigits = true; | |
1080 while (isdigit(*++p)) {} | |
1081 } | |
1082 | |
1083 // The number could be a float, so check errors below. | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1084 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1085 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1086 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1087 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1088 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1089 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1090 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1091 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1092 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1093 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1094 goto LscanReal; |
63 | 1095 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1096 } |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1097 |
63 | 1098 if (hasDecimalDigits) |
1099 error(MID.OctalNumberHasDecimals); | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1100 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1101 error(MID.OverflowOctalNumber); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1102 // goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1103 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1104 Lfinalize: |
58 | 1105 enum Suffix |
1106 { | |
1107 None = 0, | |
1108 Unsigned = 1, | |
60 | 1109 Long = 2 |
58 | 1110 } |
60 | 1111 |
58 | 1112 Suffix suffix; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1113 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1114 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1115 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1116 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1117 case 'L': |
58 | 1118 if (suffix & Suffix.Long) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1119 break; |
60 | 1120 suffix |= Suffix.Long; |
1121 ++p; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1122 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1123 case 'u', 'U': |
58 | 1124 if (suffix & Suffix.Unsigned) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1125 break; |
60 | 1126 suffix |= Suffix.Unsigned; |
1127 ++p; | |
1128 continue; | |
58 | 1129 default: |
1130 break; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1131 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1132 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1133 } |
58 | 1134 |
1135 switch (suffix) | |
1136 { | |
1137 case Suffix.None: | |
1138 if (ulong_ & 0x8000000000000000) | |
1139 { | |
1140 if (isDecimal) | |
1141 error(MID.OverflowDecimalSign); | |
1142 t.type = TOK.Uint64; | |
1143 } | |
1144 else if (ulong_ & 0xFFFFFFFF00000000) | |
1145 t.type = TOK.Int64; | |
1146 else if (ulong_ & 0x80000000) | |
1147 t.type = isDecimal ? TOK.Int64 : TOK.Uint32; | |
1148 else | |
1149 t.type = TOK.Int32; | |
1150 break; | |
1151 case Suffix.Unsigned: | |
1152 if (ulong_ & 0xFFFFFFFF00000000) | |
1153 t.type = TOK.Uint64; | |
1154 else | |
1155 t.type = TOK.Uint32; | |
1156 break; | |
1157 case Suffix.Long: | |
1158 if (ulong_ & 0x8000000000000000) | |
1159 { | |
1160 if (isDecimal) | |
1161 error(MID.OverflowDecimalSign); | |
1162 t.type = TOK.Uint64; | |
1163 } | |
1164 else | |
1165 t.type = TOK.Int64; | |
1166 break; | |
1167 case Suffix.Unsigned | Suffix.Long: | |
1168 t.type = TOK.Uint64; | |
1169 break; | |
1170 default: | |
1171 assert(0); | |
1172 } | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1173 t.ulong_ = ulong_; |
15 | 1174 t.end = p; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1175 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1176 LscanReal: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1177 scanReal(t); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1178 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1179 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1180 |
63 | 1181 /* |
1182 FloatLiteral:= Float[fFL]?i? | |
1183 Float:= DecFloat | HexFloat | |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1184 DecFloat:= ([0-9][0-9_]*[.][0-9_]*DecExponent?) | [.][0-9][0-9_]*DecExponent? | [0-9][0-9_]*DecExponent |
63 | 1185 DecExponent:= [eE][+-]?[0-9][0-9_]* |
1186 HexFloat:= 0[xX](HexDigits[.]HexDigits | [.][0-9a-zA-Z]HexDigits? | HexDigits)HexExponent | |
1187 HexExponent:= [pP][+-]?[0-9][0-9_]* | |
1188 */ | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1189 void scanReal(ref Token t) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1190 { |
63 | 1191 if (*p == '.') |
1192 // This function was called by scan() or scanNumber(). | |
1193 while (isdigit(*++p) || *p == '_') {} | |
1194 else | |
1195 { | |
1196 // This function was called by scanNumber(). | |
1197 debug switch (*p) | |
1198 { | |
1199 case 'L': | |
1200 if (p[1] != 'i') | |
1201 assert(0); | |
1202 case 'i', 'f', 'F', 'e', 'E': break; | |
1203 default: assert(0); | |
1204 } | |
1205 } | |
1206 | |
1207 // Scan exponent. | |
1208 if (*p == 'e' || *p == 'E') | |
1209 { | |
1210 ++p; | |
1211 if (*p == '-' || *p == '+') | |
1212 ++p; | |
1213 if (!isdigit(*p)) | |
1214 error(MID.FloatExponentDigitExpected); | |
1215 else | |
64 | 1216 while (isdigit(*++p) || *p == '_') {} |
63 | 1217 } |
1218 | |
1219 // Copy string to buffer ignoring underscores. | |
1220 char[] buffer; | |
1221 char* end = p; | |
1222 p = t.start; | |
1223 do | |
1224 { | |
1225 if (*p == '_') | |
1226 { | |
1227 ++p; | |
1228 continue; | |
1229 } | |
1230 buffer ~= *p; | |
1231 ++p; | |
1232 } while (p != end) | |
1233 buffer ~= 0; | |
1234 finalizeFloat(t, buffer); | |
30 | 1235 } |
1236 | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1237 void scanHexReal(ref Token t) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1238 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1239 assert(*p == '.' || *p == 'i' || *p == 'p' || *p == 'P' || (*p == 'L' && p[1] == 'i')); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1240 MID mid; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1241 if (*p == '.') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1242 while (ishexad(*++p) || *p == '_') {} |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1243 if (*p != 'p' && *p != 'P') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1244 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1245 mid = MID.HexFloatExponentRequired; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1246 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1247 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1248 // Copy mantissa to a buffer ignoring underscores. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1249 char* end = p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1250 p = t.start; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1251 char[] buffer; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1252 do |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1253 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1254 if (*p == '_') |
63 | 1255 { |
1256 ++p; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1257 continue; |
63 | 1258 } |
1259 buffer ~= *p; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1260 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1261 } while (p != end) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1262 |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1263 assert(p == end && (*p == 'p' || *p == 'P')); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1264 // Scan and copy the exponent. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1265 buffer ~= 'p'; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1266 size_t bufflen = buffer.length; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1267 while (1) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1268 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1269 if (*++p == '_') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1270 continue; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1271 if (isdigit(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1272 buffer ~= *p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1273 else |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1274 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1275 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1276 // When the buffer length hasn't changed, no digits were copied. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1277 if (bufflen == buffer.length) { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1278 mid = MID.HexFloatMissingExpDigits; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1279 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1280 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1281 buffer ~= 0; // Terminate for C functions. |
63 | 1282 finalizeFloat(t, buffer); |
1283 return; | |
1284 Lerr: | |
1285 t.type = TOK.Float32; | |
1286 t.end = p; | |
1287 error(mid); | |
1288 } | |
1289 | |
1290 void finalizeFloat(ref Token t, string buffer) | |
1291 { | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1292 // Float number is well-formed. Check suffixes and do conversion. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1293 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1294 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1295 case 'f', 'F': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1296 t.type = TOK.Float32; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1297 t.float_ = strtof(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1298 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1299 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1300 case 'L': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1301 t.type = TOK.Float80; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1302 t.real_ = strtold(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1303 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1304 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1305 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1306 t.type = TOK.Float64; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1307 t.double_ = strtod(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1308 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1309 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1310 if (*p == 'i') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1311 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1312 ++p; |
63 | 1313 t.type += 3; // Switch to imaginary counterpart. |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1314 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1315 if (getErrno == ERANGE) |
63 | 1316 error(MID.OverflowFloatNumber); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1317 t.end = p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1318 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1319 |
30 | 1320 /// Scan special token: #line Integer [Filespec] EndOfLine |
51 | 1321 // TODO: Handle case like: #line 0 #line 2 |
30 | 1322 void scanSpecialToken() |
1323 { | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1324 assert(*p == '#'); |
51 | 1325 |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1326 ++p; |
30 | 1327 MID mid; |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1328 Token* t; |
51 | 1329 uint oldloc = this.loc, newloc; |
30 | 1330 |
51 | 1331 peek(t); |
65 | 1332 if (!(this.loc == oldloc && p == t.start && t.type == TOK.Identifier && t.srcText == "line")) |
51 | 1333 { |
1334 this.loc = oldloc; // reset this.loc because we took a peek at the next token | |
1335 mid = MID.ExpectedIdentifierSTLine; | |
1336 goto Lerr; | |
1337 } | |
1338 p = t.end; // consume token | |
1339 | |
1340 peek(t); | |
131 | 1341 if (this.loc == oldloc && t.type == TOK.Int32) |
51 | 1342 { |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1343 newloc = t.uint_ - 1; |
51 | 1344 p = t.end; |
1345 } | |
1346 else | |
1347 { | |
1348 this.loc = oldloc; | |
1349 mid = MID.ExpectedNumberAfterSTLine; | |
30 | 1350 goto Lerr; |
1351 } | |
1352 | |
51 | 1353 peek(t); |
1354 if (this.loc != oldloc) | |
30 | 1355 { |
51 | 1356 this.loc = oldloc; |
1357 mid = MID.NewlineInSpecialToken; | |
1358 goto Lerr; | |
30 | 1359 } |
1360 if (t.type == TOK.String) | |
1361 { | |
51 | 1362 if (*t.start != '"') |
1363 { | |
1364 mid = MID.ExpectedNormalStringLiteral; | |
30 | 1365 goto Lerr; |
51 | 1366 } |
65 | 1367 fileName = t.srcText[1..$-1]; // contents of "..." |
30 | 1368 p = t.end; |
1369 } | |
65 | 1370 else if (t.type == TOK.Identifier && t.srcText == "__FILE__") |
30 | 1371 { |
1372 p = t.end; | |
1373 } | |
51 | 1374 /+ |
1375 peek(t); | |
1376 if (this.loc == oldloc && t.type != TOK.EOF) | |
1377 { | |
1378 mid = MID.UnterminatedSpecialToken; | |
1379 goto Lerr; | |
1380 } | |
1381 +/ | |
30 | 1382 while (1) |
1383 { | |
51 | 1384 switch (*p) |
1385 { | |
1386 case '\r': | |
1387 if (p[1] == '\n') | |
1388 ++p; | |
1389 case '\n': | |
1390 ++p; | |
30 | 1391 break; |
51 | 1392 case LS[0]: |
1393 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) | |
1394 { | |
1395 p += 2; | |
1396 break; | |
1397 } | |
1398 case 0, _Z_: | |
1399 break; | |
1400 default: | |
1401 if (isspace(*p)) { | |
1402 ++p; | |
1403 continue; | |
1404 } | |
30 | 1405 mid = MID.UnterminatedSpecialToken; |
1406 goto Lerr; | |
1407 } | |
51 | 1408 break; |
30 | 1409 } |
1410 | |
51 | 1411 this.loc = newloc; |
30 | 1412 return; |
1413 Lerr: | |
1414 error(mid); | |
15 | 1415 } |
1416 | |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1417 dchar decodeUTF8() |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1418 { |
49 | 1419 assert(*p & 128, "check for ASCII char before calling decodeUTF8()."); |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1420 size_t idx; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1421 dchar d; |
49 | 1422 try |
1423 { | |
1424 d = std.utf.decode(p[0 .. end-p], idx); | |
1425 p += idx -1; | |
1426 } | |
1427 catch (UtfException e) | |
1428 { | |
1429 error(MID.InvalidUTF8Sequence); | |
1430 // Skip to next valid utf-8 sequence | |
1431 while (UTF8stride[*++p] != 0xFF) {} | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1432 --p; |
49 | 1433 } |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1434 return d; |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1435 } |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1436 |
28 | 1437 void loadKeywords() |
1438 { | |
1439 foreach(k; keywords) | |
1440 idtable[k.str] = k; | |
1441 } | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1442 /+ |
105 | 1443 struct State |
1444 { | |
1445 Lexer lexer; | |
209 | 1446 Token token; |
105 | 1447 char* scanPointer; |
209 | 1448 int loc; |
212 | 1449 string fileName; |
105 | 1450 size_t errorLen; |
212 | 1451 static State opCall(Lexer lx) |
105 | 1452 { |
1453 State s; | |
212 | 1454 s.lexer = lx; |
1455 s.token = lx.token; | |
1456 s.scanPointer = lx.p; | |
1457 s.loc = lx.loc; | |
1458 s.fileName = lx.fileName; | |
1459 s.errorLen = lx.errors.length; | |
105 | 1460 return s; |
1461 } | |
1462 void restore() | |
1463 { | |
1464 lexer.p = scanPointer; | |
209 | 1465 lexer.token = token; |
1466 lexer.loc = loc; | |
212 | 1467 lexer.fileName = fileName; |
105 | 1468 lexer.errors = lexer.errors[0..errorLen]; |
1469 } | |
1470 } | |
1471 | |
1472 State getState() | |
1473 { | |
212 | 1474 return State(this); |
105 | 1475 } |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1476 +/ |
105 | 1477 |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1478 private void scanNext(ref Token* t) |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1479 { |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1480 assert(t !is null); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1481 if (t.next) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1482 t = t.next; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1483 else if (t.type != TOK.EOF) |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1484 { |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1485 Token* new_t = new Token; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1486 scan(*new_t); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1487 new_t.prev = t; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1488 t.next = new_t; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1489 t = new_t; |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1490 } |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1491 } |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1492 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1493 void peek(ref Token* t) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1494 { |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1495 scanNext(t); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1496 } |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1497 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1498 TOK nextToken() |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1499 { |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1500 scanNext(this.token); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1501 return this.token.type; |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1502 } |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1503 |
66
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
1504 void error(MID id, ...) |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1505 { |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1506 // if (reportErrors) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1507 errors ~= new Information(InfoType.Lexer, id, loc, arguments(_arguments, _argptr)); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1508 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1509 |
207 | 1510 unittest |
1511 { | |
1512 string sourceText = "unittest { }"; | |
1513 auto lx = new Lexer(sourceText, null); | |
1514 | |
1515 Token next; | |
1516 lx.peek(next); | |
1517 assert(next == TOK.Unittest); | |
1518 lx.peek(next); | |
1519 assert(next == TOK.LBrace); | |
1520 lx.peek(next); | |
1521 assert(next == TOK.RBrace); | |
1522 lx.peek(next); | |
1523 assert(next == TOK.EOF); | |
1524 writefln("end of peek() unittest"); | |
1525 } | |
1526 | |
4 | 1527 Token[] getTokens() |
1528 { | |
1529 Token[] tokens; | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1530 // while (nextToken() != TOK.EOF) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1531 // tokens ~= this.token; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1532 // tokens ~= this.token; |
4 | 1533 return tokens; |
3 | 1534 } |
48 | 1535 |
1536 private void encodeUTF8(inout char[] str, dchar d) | |
1537 { | |
1538 char[6] b; | |
1539 assert(d > 0x7F, "check for ASCII char before calling encodeUTF8()."); | |
1540 if (d < 0x800) | |
1541 { | |
1542 b[0] = 0xC0 | (d >> 6); | |
1543 b[1] = 0x80 | (d & 0x3F); | |
1544 str ~= b[0..2]; | |
1545 } | |
1546 else if (d < 0x10000) | |
1547 { | |
1548 b[0] = 0xE0 | (d >> 12); | |
1549 b[1] = 0x80 | ((d >> 6) & 0x3F); | |
1550 b[2] = 0x80 | (d & 0x3F); | |
1551 str ~= b[0..3]; | |
1552 } | |
1553 else if (d < 0x200000) | |
1554 { | |
1555 b[0] = 0xF0 | (d >> 18); | |
1556 b[1] = 0x80 | ((d >> 12) & 0x3F); | |
1557 b[2] = 0x80 | ((d >> 6) & 0x3F); | |
1558 b[3] = 0x80 | (d & 0x3F); | |
1559 str ~= b[0..4]; | |
1560 } | |
1561 else if (d < 0x4000000) | |
1562 { | |
1563 b[0] = 0xF8 | (d >> 24); | |
1564 b[1] = 0x80 | ((d >> 18) & 0x3F); | |
1565 b[2] = 0x80 | ((d >> 12) & 0x3F); | |
1566 b[3] = 0x80 | ((d >> 6) & 0x3F); | |
1567 b[4] = 0x80 | (d & 0x3F); | |
1568 str ~= b[0..5]; | |
1569 } | |
1570 else if (d < 0x80000000) | |
1571 { | |
1572 b[0] = 0xFC | (d >> 30); | |
1573 b[1] = 0x80 | ((d >> 24) & 0x3F); | |
1574 b[2] = 0x80 | ((d >> 18) & 0x3F); | |
1575 b[3] = 0x80 | ((d >> 12) & 0x3F); | |
1576 b[4] = 0x80 | ((d >> 6) & 0x3F); | |
1577 b[5] = 0x80 | (d & 0x3F); | |
1578 str ~= b[0..6]; | |
1579 } | |
1580 else | |
1581 error(MID.InvalidUnicodeCharacter); | |
1582 } | |
5 | 1583 } |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1584 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1585 unittest |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1586 { |
40 | 1587 string[] toks = [ |
1588 ">", ">=", ">>", ">>=", ">>>", ">>>=", "<", "<=", "<>", | |
1589 "<>=", "<<", "<<=", "!", "!<", "!>", "!<=", "!>=", "!<>", | |
1590 "!<>=", ".", "..", "...", "&", "&&", "&=", "+", "++", | |
1591 "+=", "-", "--", "-=", "=", "==", "~", "~=", "*", | |
1592 "*=", "/", "/=", "^", "^=", "%", "%=", "(", ")", | |
1593 "[", "]", "{", "}", ":", ";", "?", ",", "$" | |
1594 ]; | |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1595 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1596 char[] src; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1597 |
40 | 1598 foreach (op; toks) |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1599 src ~= op ~ " "; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1600 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1601 auto lx = new Lexer(src, ""); |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1602 auto tokens = lx.getTokens(); |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1603 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1604 tokens = tokens[0..$-1]; // exclude TOK.EOF |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1605 |
40 | 1606 assert(tokens.length == toks.length ); |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1607 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1608 foreach (i, t; tokens) |
207 | 1609 assert(t.srcText == toks[i], std.string.format("Lexed '%s' but expected '%s'", t.srcText, toks[i])); |
41 | 1610 } |
55 | 1611 |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1612 unittest |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1613 { |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1614 // Numbers unittest |
68
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1615 // 0L 0ULi 0_L 0_UL 0x0U 0x0p2 0_Fi 0_e2 0_F 0_i |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1616 // 0u 0U 0uL 0UL 0L 0LU 0Lu |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1617 // 0Li 0f 0F 0fi 0Fi 0i |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1618 // 0b_1_LU 0b1000u |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1619 // 0x232Lu |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1620 } |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1621 |
55 | 1622 /// ASCII character properties table. |
1623 static const int ptable[256] = [ | |
1624 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0, | |
1625 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1626 32, 0, 0x2200, 0, 0, 0, 0, 0x2700, 0, 0, 0, 0, 0, 0, 0, 0, | |
1627 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0x3f00, | |
1628 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8, | |
1629 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0x5c00, 0, 0,16, | |
1630 0, 0x70c, 0x80c,12,12,12, 0xc0c, 8, 8, 8, 8, 8, 8, 8, 0xa08, 8, | |
1631 8, 8, 0xd08, 8, 0x908, 8, 0xb08, 8, 8, 8, 8, 0, 0, 0, 0, 0, | |
1632 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1633 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1634 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1635 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1636 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1637 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1638 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1639 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 | |
1640 ]; | |
1641 | |
1642 enum CProperty | |
1643 { | |
1644 Octal = 1, | |
1645 Digit = 1<<1, | |
1646 Hex = 1<<2, | |
1647 Alpha = 1<<3, | |
1648 Underscore = 1<<4, | |
1649 Whitespace = 1<<5 | |
1650 } | |
1651 | |
1652 const uint EVMask = 0xFF00; // Bit mask for escape value | |
1653 | |
1654 private alias CProperty CP; | |
1655 int isoctal(char c) { return ptable[c] & CP.Octal; } | |
1656 int isdigit(char c) { return ptable[c] & CP.Digit; } | |
1657 int ishexad(char c) { return ptable[c] & CP.Hex; } | |
1658 int isalpha(char c) { return ptable[c] & CP.Alpha; } | |
1659 int isalnum(char c) { return ptable[c] & (CP.Alpha | CP.Digit); } | |
1660 int isidbeg(char c) { return ptable[c] & (CP.Alpha | CP.Underscore); } | |
1661 int isident(char c) { return ptable[c] & (CP.Alpha | CP.Underscore | CP.Digit); } | |
1662 int isspace(char c) { return ptable[c] & CP.Whitespace; } | |
1663 int char2ev(char c) { return ptable[c] >> 8; /*(ptable[c] & EVMask) >> 8;*/ } | |
1664 | |
1665 version(gen_ptable) | |
1666 static this() | |
1667 { | |
1668 alias ptable p; | |
1669 // Initialize character properties table. | |
1670 for (int i; i < p.length; ++i) | |
1671 { | |
1672 p[i] = 0; | |
1673 if ('0' <= i && i <= '7') | |
1674 p[i] |= CP.Octal; | |
1675 if ('0' <= i && i <= '9') | |
1676 p[i] |= CP.Digit; | |
1677 if (isdigit(i) || 'a' <= i && i <= 'f' || 'A' <= i && i <= 'F') | |
1678 p[i] |= CP.Hex; | |
1679 if ('a' <= i && i <= 'z' || 'A' <= i && i <= 'Z') | |
1680 p[i] |= CP.Alpha; | |
1681 if (i == '_') | |
1682 p[i] |= CP.Underscore; | |
1683 if (i == ' ' || i == '\t' || i == '\v' || i == '\f') | |
1684 p[i] |= CP.Whitespace; | |
1685 } | |
1686 // Store escape sequence values in second byte. | |
1687 assert(CProperty.max <= ubyte.max, "character property flags and escape value byte overlap."); | |
1688 p['\''] |= 39 << 8; | |
1689 p['"'] |= 34 << 8; | |
1690 p['?'] |= 63 << 8; | |
1691 p['\\'] |= 92 << 8; | |
1692 p['a'] |= 7 << 8; | |
1693 p['b'] |= 8 << 8; | |
1694 p['f'] |= 12 << 8; | |
1695 p['n'] |= 10 << 8; | |
1696 p['r'] |= 13 << 8; | |
1697 p['t'] |= 9 << 8; | |
1698 p['v'] |= 11 << 8; | |
1699 // Print a formatted array literal. | |
1700 char[] array = "[\n"; | |
1701 for (int i; i < p.length; ++i) | |
1702 { | |
1703 int c = p[i]; | |
1704 array ~= std.string.format(c>255?" 0x%x,":"%2d,", c, ((i+1) % 16) ? "":"\n"); | |
1705 } | |
1706 array[$-2..$] = "\n]"; | |
1707 writefln(array); | |
1708 } |