Mercurial > projects > dil
annotate trunk/src/Lexer.d @ 103:511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
- Started implementation of parseDeclaration(), parseAttributeSpecifier() and parseImportDeclaration().
author | aziz |
---|---|
date | Sun, 08 Jul 2007 10:14:00 +0000 |
parents | 0fe650a7a8d1 |
children | df34ec47fb81 |
rev | line source |
---|---|
0 | 1 /++ |
2 Author: Aziz Köksal | |
3 License: GPL2 | |
4 +/ | |
5 module Lexer; | |
3 | 6 import Token; |
69
24db7c5522d5
- Added module Information for compiler messages like warnings, info and errors to the user.
aziz
parents:
68
diff
changeset
|
7 import Information; |
28 | 8 import Keywords; |
9 import Identifier; | |
54 | 10 import Messages; |
2 | 11 import std.stdio; |
4 | 12 import std.utf; |
13 import std.uni; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
14 import std.c.stdlib; |
66
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
15 import std.string; |
0 | 16 |
11 | 17 const char[3] LS = \u2028; |
18 const char[3] PS = \u2029; | |
19 | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
20 const dchar LSd = 0x2028; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
21 const dchar PSd = 0x2029; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
22 |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
23 const uint _Z_ = 26; /// Control+Z |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
24 |
0 | 25 class Lexer |
26 { | |
4 | 27 Token token; |
30 | 28 string text; |
4 | 29 char* p; |
30 char* end; | |
31 | |
17 | 32 uint loc = 1; /// line of code |
33 | |
30 | 34 char[] fileName; |
35 | |
69
24db7c5522d5
- Added module Information for compiler messages like warnings, info and errors to the user.
aziz
parents:
68
diff
changeset
|
36 Information[] errors; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
37 |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
38 bool reportErrors; |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
39 |
28 | 40 Identifier[string] idtable; |
41 | |
30 | 42 this(string text, string fileName) |
4 | 43 { |
30 | 44 this.fileName = fileName; |
45 | |
4 | 46 this.text = text; |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
47 if (text[$-1] != 0) |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
48 { |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
49 this.text.length = this.text.length + 1; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
50 this.text[$-1] = 0; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
51 } |
4 | 52 |
53 this.p = this.text.ptr; | |
54 this.end = this.p + this.text.length; | |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
55 this.reportErrors = true; |
28 | 56 loadKeywords(); |
4 | 57 } |
58 | |
59 public void scan(out Token t) | |
3 | 60 { |
4 | 61 assert(p < end); |
0 | 62 |
10 | 63 uint c = *p; |
4 | 64 |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
65 while (1) |
4 | 66 { |
5 | 67 t.start = p; |
16 | 68 |
5 | 69 if (c == 0) |
70 { | |
51 | 71 assert(*p == 0); |
17 | 72 ++p; |
51 | 73 assert(p == end); |
5 | 74 t.type = TOK.EOF; |
17 | 75 t.end = p; |
5 | 76 return; |
77 } | |
4 | 78 |
17 | 79 if (c == '\n') |
80 { | |
81 c = *++p; | |
82 ++loc; | |
83 continue; | |
84 } | |
85 else if (c == '\r') | |
86 { | |
87 c = *++p; | |
88 if (c != '\n') | |
89 ++loc; | |
90 continue; | |
91 } | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
92 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
93 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
94 p += 3; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
95 c = *p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
96 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
97 } |
17 | 98 |
13 | 99 if (isidbeg(c)) |
5 | 100 { |
33 | 101 if (c == 'r' && p[1] == '"' && ++p) |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
102 return scanRawStringLiteral(t); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
103 if (c == 'x' && p[1] == '"') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
104 return scanHexStringLiteral(t); |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
105 Lidentifier: |
5 | 106 do |
107 { c = *++p; } | |
49 | 108 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8())) |
28 | 109 |
5 | 110 t.end = p; |
28 | 111 |
65 | 112 string str = t.srcText; |
28 | 113 Identifier* id = str in idtable; |
114 | |
115 if (!id) | |
116 { | |
117 idtable[str] = Identifier.Identifier(TOK.Identifier, str); | |
118 id = str in idtable; | |
119 } | |
120 assert(id); | |
121 t.type = id.type; | |
5 | 122 return; |
123 } | |
124 | |
15 | 125 if (isdigit(c)) |
126 return scanNumber(t); | |
127 | |
8 | 128 if (c == '/') |
5 | 129 { |
8 | 130 c = *++p; |
14 | 131 switch(c) |
5 | 132 { |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
133 case '=': |
14 | 134 ++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
135 t.type = TOK.DivAssign; |
14 | 136 t.end = p; |
137 return; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
138 case '+': |
8 | 139 uint level = 1; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
140 while (1) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
141 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
142 c = *++p; |
42 | 143 LswitchNC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
144 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
145 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
146 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
147 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
148 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
149 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
150 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
151 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
152 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
153 error(MID.UnterminatedNestedComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
154 goto LreturnNC; |
42 | 155 default: |
156 } | |
157 | |
158 c <<= 8; | |
159 c |= *++p; | |
160 switch (c) | |
161 { | |
162 case 0x2F2B: // /+ | |
163 ++level; | |
164 continue; | |
165 case 0x2B2F: // +/ | |
166 if (--level == 0) | |
167 { | |
168 ++p; | |
169 LreturnNC: | |
170 t.type = TOK.Comment; | |
171 t.end = p; | |
172 return; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
173 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
174 continue; |
42 | 175 case 0xE280: // LS[0..1] || PS[0..1] |
176 if (p[1] == LS[2] || p[1] == PS[2]) | |
177 { | |
178 ++loc; | |
179 ++p; | |
180 } | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
181 continue; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
182 default: |
42 | 183 c &= char.max; |
184 goto LswitchNC; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
185 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
186 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
187 case '*': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
188 while (1) |
7 | 189 { |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
190 c = *++p; |
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
191 LswitchBC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
192 switch (c) |
8 | 193 { |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
194 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
195 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
196 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
197 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
198 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
199 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
200 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
201 error(MID.UnterminatedBlockComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
202 goto LreturnBC; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
203 default: |
8 | 204 } |
41 | 205 |
206 c <<= 8; | |
207 c |= *++p; | |
208 switch (c) | |
209 { | |
210 case 0x2A2F: // */ | |
211 ++p; | |
212 LreturnBC: | |
213 t.type = TOK.Comment; | |
214 t.end = p; | |
215 return; | |
216 case 0xE280: // LS[0..1] || PS[0..1] | |
217 if (p[1] == LS[2] || p[1] == PS[2]) | |
218 { | |
219 ++loc; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
220 ++p; |
41 | 221 } |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
222 continue; |
41 | 223 default: |
224 c &= char.max; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
225 goto LswitchBC; |
41 | 226 } |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
227 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
228 assert(0); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
229 case '/': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
230 while (1) |
7 | 231 { |
8 | 232 c = *++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
233 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
234 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
235 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
236 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
237 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
238 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
239 case 0, _Z_: |
11 | 240 break; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
241 case LS[0]: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
242 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
243 break; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
244 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
245 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
246 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
247 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
248 t.type = TOK.Comment; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
249 t.end = p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
250 return; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
251 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
252 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
253 t.type = TOK.Div; |
10 | 254 t.end = p; |
255 return; | |
256 } | |
5 | 257 } |
258 | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
259 switch (c) |
20 | 260 { |
48 | 261 case '\'': |
262 return scanCharacterLiteral(t); | |
263 case '`': | |
264 return scanRawStringLiteral(t); | |
265 case '"': | |
266 return scanNormalStringLiteral(t); | |
267 case '\\': | |
268 char[] buffer; | |
269 do | |
270 { | |
271 ++p; | |
272 c = scanEscapeSequence(); | |
273 if (c < 128) | |
274 buffer ~= c; | |
275 else | |
276 encodeUTF8(buffer, c); | |
277 } while (*p == '\\') | |
278 buffer ~= 0; | |
279 t.type = TOK.String; | |
280 t.str = buffer; | |
281 t.end = p; | |
282 return; | |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
283 case '>': /* > >= >> >>= >>> >>>= */ |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
284 c = *++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
285 switch (c) |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
286 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
287 case '=': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
288 t.type = TOK.GreaterEqual; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
289 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
290 case '>': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
291 if (p[1] == '>') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
292 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
293 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
294 if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
295 { ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
296 t.type = TOK.URShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
297 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
298 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
299 t.type = TOK.URShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
300 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
301 else if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
302 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
303 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
304 t.type = TOK.RShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
305 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
306 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
307 t.type = TOK.RShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
308 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
309 default: |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
310 t.type = TOK.Greater; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
311 goto Lcommon2; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
312 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
313 assert(0); |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
314 case '<': /* < <= <> <>= << <<= */ |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
315 c = *++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
316 switch (c) |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
317 { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
318 case '=': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
319 t.type = TOK.LessEqual; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
320 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
321 case '<': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
322 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
323 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
324 t.type = TOK.LShiftAssign; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
325 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
326 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
327 t.type = TOK.LShift; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
328 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
329 case '>': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
330 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
331 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
332 t.type = TOK.LorEorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
333 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
334 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
335 t.type = TOK.LorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
336 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
337 default: |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
338 t.type = TOK.Less; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
339 goto Lcommon2; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
340 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
341 assert(0); |
37
7f3bcb97d017
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
36
diff
changeset
|
342 case '!': /* ! !< !> !<= !>= !<> !<>= */ |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
343 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
344 switch (c) |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
345 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
346 case '<': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
347 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
348 if (c == '>') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
349 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
350 if (p[1] == '=') { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
351 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
352 t.type = TOK.Unordered; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
353 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
354 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
355 t.type = TOK.UorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
356 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
357 else if (c == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
358 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
359 t.type = TOK.UorG; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
360 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
361 else { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
362 t.type = TOK.UorGorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
363 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
364 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
365 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
366 case '>': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
367 if (p[1] == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
368 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
369 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
370 t.type = TOK.UorL; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
371 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
372 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
373 t.type = TOK.UorLorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
374 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
375 case '=': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
376 t.type = TOK.NotEqual; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
377 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
378 default: |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
379 t.type = TOK.Not; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
380 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
381 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
382 assert(0); |
63 | 383 case '.': /* . .[0-9] .. ... */ |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
384 if (p[1] == '.') |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
385 { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
386 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
387 if (p[1] == '.') { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
388 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
389 t.type = TOK.Ellipses; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
390 } |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
391 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
392 t.type = TOK.Slice; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
393 } |
63 | 394 else if (isdigit(p[1])) |
395 { | |
396 return scanReal(t); | |
397 } | |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
398 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
399 t.type = TOK.Dot; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
400 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
401 case '|': /* | || |= */ |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
402 c = *++p; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
403 if (c == '=') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
404 t.type = TOK.OrAssign; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
405 else if (c == '|') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
406 t.type = TOK.OrLogical; |
26 | 407 else { |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
408 t.type = TOK.OrBinary; |
26 | 409 goto Lcommon2; |
410 } | |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
411 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
412 case '&': /* & && &= */ |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
413 c = *++p; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
414 if (c == '=') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
415 t.type = TOK.AndAssign; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
416 else if (c == '&') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
417 t.type = TOK.AndLogical; |
26 | 418 else { |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
419 t.type = TOK.AndBinary; |
26 | 420 goto Lcommon2; |
421 } | |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
422 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
423 case '+': /* + ++ += */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
424 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
425 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
426 t.type = TOK.PlusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
427 else if (c == '+') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
428 t.type = TOK.PlusPlus; |
26 | 429 else { |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
430 t.type = TOK.Plus; |
26 | 431 goto Lcommon2; |
432 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
433 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
434 case '-': /* - -- -= */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
435 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
436 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
437 t.type = TOK.MinusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
438 else if (c == '-') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
439 t.type = TOK.MinusMinus; |
26 | 440 else { |
441 t.type = TOK.Minus; | |
442 goto Lcommon2; | |
443 } | |
444 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
445 case '=': /* = == */ |
26 | 446 if (p[1] == '=') { |
447 ++p; | |
448 t.type = TOK.Equal; | |
449 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
450 else |
26 | 451 t.type = TOK.Assign; |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
452 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
453 case '~': /* ~ ~= */ |
27 | 454 if (p[1] == '=') { |
455 ++p; | |
456 t.type = TOK.CatAssign; | |
457 } | |
458 else | |
459 t.type = TOK.Tilde; | |
460 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
461 case '*': /* * *= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
462 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
463 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
464 t.type = TOK.MulAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
465 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
466 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
467 t.type = TOK.Mul; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
468 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
469 case '^': /* ^ ^= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
470 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
471 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
472 t.type = TOK.XorAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
473 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
474 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
475 t.type = TOK.Xor; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
476 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
477 case '%': /* % %= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
478 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
479 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
480 t.type = TOK.ModAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
481 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
482 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
483 t.type = TOK.Mod; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
484 goto Lcommon; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
485 // Single character tokens: |
20 | 486 case '(': |
487 t.type = TOK.LParen; | |
488 goto Lcommon; | |
489 case ')': | |
490 t.type = TOK.RParen; | |
491 goto Lcommon; | |
492 case '[': | |
493 t.type = TOK.LBracket; | |
494 goto Lcommon; | |
495 case ']': | |
496 t.type = TOK.RBracket; | |
497 goto Lcommon; | |
498 case '{': | |
499 t.type = TOK.LBrace; | |
500 goto Lcommon; | |
501 case '}': | |
502 t.type = TOK.RBrace; | |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
503 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
504 case ':': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
505 t.type = TOK.Colon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
506 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
507 case ';': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
508 t.type = TOK.Semicolon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
509 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
510 case '?': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
511 t.type = TOK.Question; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
512 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
513 case ',': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
514 t.type = TOK.Comma; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
515 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
516 case '$': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
517 t.type = TOK.Dollar; |
20 | 518 Lcommon: |
519 ++p; | |
26 | 520 Lcommon2: |
20 | 521 t.end = p; |
522 return; | |
30 | 523 case '#': |
524 scanSpecialToken(); | |
51 | 525 c = *p; |
526 continue; | |
20 | 527 default: |
528 } | |
529 | |
49 | 530 if (c & 128 && isUniAlpha(decodeUTF8())) |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
531 goto Lidentifier; |
5 | 532 c = *++p; |
4 | 533 } |
3 | 534 } |
4 | 535 |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
536 void scanNormalStringLiteral(ref Token t) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
537 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
538 assert(*p == '"'); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
539 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
540 char[] buffer; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
541 t.type = TOK.String; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
542 while (1) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
543 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
544 switch (*p) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
545 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
546 case '"': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
547 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
548 Lreturn: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
549 buffer ~= 0; |
49 | 550 t.str = buffer; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
551 t.pf = scanPostfix(); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
552 t.end = p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
553 return; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
554 case '\\': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
555 ++p; |
49 | 556 dchar d = scanEscapeSequence(); |
557 if (d < 128) | |
558 buffer ~= d; | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
559 else |
49 | 560 encodeUTF8(buffer, d); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
561 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
562 case '\r': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
563 if (p[1] == '\n') |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
564 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
565 case '\n': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
566 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
567 ++loc; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
568 buffer ~= '\n'; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
569 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
570 case 0, _Z_: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
571 error(MID.UnterminatedString); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
572 goto Lreturn; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
573 default: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
574 if (*p & 128) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
575 { |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
576 // char* begin = p; |
49 | 577 dchar d = decodeUTF8(); |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
578 |
49 | 579 if (d == LSd || d == PSd) |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
580 goto case '\n'; |
49 | 581 |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
582 // We don't copy per pointer because we might include |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
583 // invalid, skipped utf-8 sequences. See decodeUTF8(). |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
584 // ++p; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
585 // buffer ~= begin[0 .. p - begin]; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
586 encodeUTF8(buffer, d); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
587 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
588 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
589 buffer ~= *p++; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
590 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
591 } |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
592 assert(0); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
593 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
594 |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
595 void scanCharacterLiteral(ref Token t) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
596 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
597 assert(*p == '\''); |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
598 MID id = MID.UnterminatedCharacterLiteral; |
46 | 599 ++p; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
600 TOK type = TOK.CharLiteral; |
46 | 601 switch (*p) |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
602 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
603 case '\\': |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
604 ++p; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
605 switch (*p) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
606 { |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
607 case 'u': |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
608 type = TOK.WCharLiteral; break; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
609 case 'U': |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
610 type = TOK.DCharLiteral; break; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
611 default: |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
612 } |
46 | 613 t.dchar_ = scanEscapeSequence(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
614 break; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
615 case '\'': |
46 | 616 ++p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
617 id = MID.EmptyCharacterLiteral; |
46 | 618 case '\n', '\r', 0, _Z_: |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
619 goto Lerr; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
620 default: |
46 | 621 uint c = *p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
622 if (c & 128) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
623 { |
49 | 624 c = decodeUTF8(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
625 if (c == LSd || c == PSd) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
626 goto Lerr; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
627 if (c <= 0xFFFF) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
628 type = TOK.WCharLiteral; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
629 else |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
630 type = TOK.DCharLiteral; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
631 } |
46 | 632 t.dchar_ = c; |
633 ++p; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
634 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
635 |
46 | 636 if (*p == '\'') |
637 ++p; | |
638 else | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
639 Lerr: |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
640 error(id); |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
641 t.type = type; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
642 t.end = p; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
643 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
644 |
33 | 645 char scanPostfix() |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
646 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
647 switch (*p) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
648 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
649 case 'c': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
650 case 'w': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
651 case 'd': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
652 return *p++; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
653 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
654 return 0; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
655 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
656 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
657 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
658 void scanRawStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
659 { |
33 | 660 uint delim = *p; |
661 assert(delim == '`' || delim == '"' && p[-1] == 'r'); | |
662 t.type = TOK.String; | |
663 char[] buffer; | |
664 uint c; | |
665 while (1) | |
666 { | |
667 c = *++p; | |
668 switch (c) | |
669 { | |
670 case '\r': | |
671 if (p[1] == '\n') | |
672 ++p; | |
673 c = '\n'; // Convert '\r' and '\r\n' to '\n' | |
674 case '\n': | |
675 ++loc; | |
52
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
676 break; |
33 | 677 case '`': |
678 case '"': | |
679 if (c == delim) | |
680 { | |
681 ++p; | |
682 t.pf = scanPostfix(); | |
683 Lreturn: | |
684 t.str = buffer ~ '\0'; | |
685 t.end = p; | |
686 return; | |
687 } | |
688 break; | |
689 case LS[0]: | |
690 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) | |
691 { | |
52
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
692 c = '\n'; |
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
693 ++p; ++p; |
33 | 694 ++loc; |
695 } | |
696 break; | |
697 case 0, _Z_: | |
698 if (delim == 'r') | |
699 error(MID.UnterminatedRawString); | |
700 else | |
701 error(MID.UnterminatedBackQuoteString); | |
702 goto Lreturn; | |
703 default: | |
704 } | |
705 buffer ~= c; // copy character to buffer | |
706 } | |
707 assert(0); | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
708 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
709 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
710 void scanHexStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
711 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
712 assert(p[0] == 'x' && p[1] == '"'); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
713 t.type = TOK.String; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
714 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
715 uint c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
716 ubyte[] buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
717 ubyte h; // hex number |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
718 uint n; // number of hex digits |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
719 |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
720 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
721 while (1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
722 { |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
723 c = *++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
724 switch (c) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
725 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
726 case '"': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
727 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
728 if (n & 1) |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
729 error(MID.OddNumberOfDigitsInHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
730 t.pf = scanPostfix(); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
731 Lreturn: |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
732 buffer ~= 0; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
733 t.str = cast(string) buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
734 t.end = p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
735 return; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
736 case '\r': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
737 if (p[1] == '\n') |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
738 ++p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
739 case '\n': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
740 ++loc; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
741 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
742 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
743 if (ishexad(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
744 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
745 if (c <= '9') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
746 c -= '0'; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
747 else if (c <= 'F') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
748 c -= 'A' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
749 else |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
750 c -= 'a' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
751 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
752 if (n & 1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
753 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
754 h <<= 4; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
755 h |= c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
756 buffer ~= h; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
757 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
758 else |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
759 h = c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
760 ++n; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
761 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
762 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
763 else if (isspace(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
764 continue; |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
765 |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
766 if (c >= 128) |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
767 { |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
768 c = decodeUTF8(); |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
769 if (c == LSd || c == PSd) |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
770 { |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
771 ++p; ++p; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
772 ++loc; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
773 continue; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
774 } |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
775 } |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
776 else if (c == 0 || c == _Z_) |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
777 { |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
778 error(MID.UnterminatedHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
779 t.pf = 0; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
780 goto Lreturn; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
781 } |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
782 error(MID.NonHexCharInHexString, cast(dchar)c); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
783 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
784 } |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
785 assert(0); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
786 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
787 |
45 | 788 dchar scanEscapeSequence() |
789 { | |
790 uint c = char2ev(*p); | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
791 if (c) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
792 { |
46 | 793 ++p; |
45 | 794 return c; |
46 | 795 } |
45 | 796 uint digits = 2; |
797 | |
798 switch (*p) | |
799 { | |
800 case 'x': | |
801 c = 0; | |
802 while (1) | |
803 { | |
804 ++p; | |
805 if (ishexad(*p)) | |
806 { | |
807 c *= 16; | |
808 if (*p <= '9') | |
48 | 809 c += *p - '0'; |
45 | 810 else if (*p <= 'F') |
48 | 811 c += *p - 'A' + 10; |
45 | 812 else |
48 | 813 c += *p - 'a' + 10; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
814 |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
815 if (!--digits) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
816 { |
48 | 817 ++p; |
45 | 818 break; |
48 | 819 } |
45 | 820 } |
821 else | |
822 { | |
823 error(MID.InsufficientHexDigits); | |
824 break; | |
825 } | |
826 } | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
827 if (!isValidDchar(c)) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
828 error(MID.InvalidUnicodeCharacter); |
45 | 829 break; |
830 case 'u': | |
831 digits = 4; | |
832 goto case 'x'; | |
833 case 'U': | |
834 digits = 8; | |
835 goto case 'x'; | |
836 default: | |
48 | 837 if (isoctal(*p)) |
45 | 838 { |
48 | 839 c = 0; |
840 c += *p - '0'; | |
841 ++p; | |
842 if (!isoctal(*p)) | |
843 return c; | |
844 c *= 8; | |
845 c += *p - '0'; | |
846 ++p; | |
847 if (!isoctal(*p)) | |
848 return c; | |
849 c *= 8; | |
850 c += *p - '0'; | |
851 ++p; | |
852 } | |
853 else if(*p == '&') | |
854 { | |
855 if (isalpha(*++p)) | |
45 | 856 { |
48 | 857 while (1) |
858 { | |
859 if (isalnum(*++p)) | |
860 continue; | |
861 if (*p == ';') { | |
862 // TODO: convert entity to unicode codepoint. | |
863 ++p; | |
864 break; | |
865 } | |
866 else { | |
867 error(MID.UnterminatedHTMLEntity); | |
868 break; | |
869 } | |
45 | 870 } |
871 } | |
48 | 872 else |
873 error(MID.InvalidBeginHTMLEntity); | |
45 | 874 } |
875 else | |
48 | 876 error(MID.UndefinedEscapeSequence); |
45 | 877 } |
878 | |
879 return c; | |
880 } | |
881 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
882 /* |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
883 IntegerLiteral:= (Dec|Hex|Bin|Oct)Suffix? |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
884 Dec:= (0|[1-9][0-9_]*) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
885 Hex:= 0[xX] HexDigits |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
886 Bin:= 0[bB][01_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
887 Oct:= 0[0-7_]+ |
68
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
888 Suffix:= (L[uU]?|[uU]L?) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
889 HexDigits:= [0-9a-zA-Z_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
890 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
891 Invalid: "0b_", "0x_", "._" |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
892 */ |
15 | 893 void scanNumber(ref Token t) |
894 { | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
895 ulong ulong_; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
896 bool overflow; |
58 | 897 bool isDecimal; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
898 size_t digits; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
899 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
900 if (*p != '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
901 goto LscanInteger; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
902 ++p; // skip zero |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
903 // check for xX bB ... |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
904 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
905 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
906 case 'x','X': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
907 goto LscanHex; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
908 case 'b','B': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
909 goto LscanBin; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
910 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
911 if (p[1] == 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
912 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
913 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
914 if (p[1] == '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
915 break; |
58 | 916 case 'i','f','F', 'e', 'E': // Imaginary and float literal suffix |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
917 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
918 case '_': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
919 ++p; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
920 goto LscanOct; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
921 default: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
922 if (isoctal(*p)) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
923 goto LscanOct; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
924 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
925 |
58 | 926 ulong_ = p[-1]; |
927 isDecimal = true; | |
928 goto Lfinalize; | |
929 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
930 LscanInteger: |
58 | 931 isDecimal = true; |
63 | 932 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
933 { |
63 | 934 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
935 continue; |
63 | 936 if (!isdigit(*p)) |
937 break; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
938 if (ulong_ < ulong.max/10 || (ulong_ == ulong.max/10 && *p <= '5')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
939 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
940 ulong_ *= 10; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
941 ulong_ += *p - '0'; |
58 | 942 continue; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
943 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
944 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
945 overflow = true; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
946 while (isdigit(*++p)) {} |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
947 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
948 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
949 |
61 | 950 // The number could be a float, so check overflow below. |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
951 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
952 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
953 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
954 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
955 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
956 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
957 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
958 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
959 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
960 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
961 goto LscanReal; |
58 | 962 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
963 } |
58 | 964 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
965 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
966 error(MID.OverflowDecimalNumber); |
58 | 967 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
968 assert((isdigit(p[-1]) || p[-1] == '_') && !isdigit(*p) && *p != '_'); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
969 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
970 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
971 LscanHex: |
61 | 972 assert(digits == 0); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
973 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
974 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
975 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
976 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
977 if (!ishexad(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
978 break; |
61 | 979 ++digits; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
980 ulong_ *= 16; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
981 if (*p <= '9') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
982 ulong_ += *p - '0'; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
983 else if (*p <= 'F') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
984 ulong_ += *p - 'A' + 10; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
985 else |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
986 ulong_ += *p - 'a' + 10; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
987 } |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
988 |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
989 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
990 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
991 case '.': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
992 if (p[1] != '.') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
993 goto LscanHexReal; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
994 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
995 case 'L': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
996 if (p[1] != 'i') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
997 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
998 case 'i', 'p', 'P': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
999 goto LscanHexReal; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1000 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1001 } |
61 | 1002 if (digits == 0) |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1003 error(MID.NoDigitsInHexNumber); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1004 else if (digits > 16) |
61 | 1005 { |
1006 // Overflow: skip following digits. | |
1007 error(MID.OverflowHexNumber); | |
1008 while (ishexad(*++p)) {} | |
1009 } | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1010 goto Lfinalize; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1011 LscanHexReal: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1012 return scanHexReal(t); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1013 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1014 LscanBin: |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1015 assert(digits == 0); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1016 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1017 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1018 if (*++p == '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1019 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1020 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1021 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1022 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1023 if (*p == '1') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1024 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1025 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1026 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1027 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1028 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1029 if (*p == '_') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1030 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1031 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1032 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1033 |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1034 if (digits == 0) |
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1035 error(MID.NoDigitsInBinNumber); |
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1036 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1037 if (digits > 64) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1038 error(MID.OverflowBinaryNumber); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1039 assert((p[-1] == '0' || p[-1] == '1' || p[-1] == '_') && !(*p == '0' || *p == '1' || *p == '_')); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1040 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1041 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1042 LscanOct: |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1043 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1044 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1045 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1046 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1047 if (!isoctal(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1048 break; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1049 if (ulong_ < ulong.max/2 || (ulong_ == ulong.max/2 && *p <= '1')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1050 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1051 ulong_ *= 8; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1052 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1053 ++p; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1054 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1055 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1056 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1057 overflow = true; |
63 | 1058 while (isdigit(*++p)) {} |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1059 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1060 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1061 |
63 | 1062 bool hasDecimalDigits; |
1063 if (isdigit(*p)) | |
1064 { | |
1065 hasDecimalDigits = true; | |
1066 while (isdigit(*++p)) {} | |
1067 } | |
1068 | |
1069 // The number could be a float, so check errors below. | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1070 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1071 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1072 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1073 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1074 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1075 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1076 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1077 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1078 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1079 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1080 goto LscanReal; |
63 | 1081 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1082 } |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1083 |
63 | 1084 if (hasDecimalDigits) |
1085 error(MID.OctalNumberHasDecimals); | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1086 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1087 error(MID.OverflowOctalNumber); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1088 // goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1089 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1090 Lfinalize: |
58 | 1091 enum Suffix |
1092 { | |
1093 None = 0, | |
1094 Unsigned = 1, | |
60 | 1095 Long = 2 |
58 | 1096 } |
60 | 1097 |
58 | 1098 Suffix suffix; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1099 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1100 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1101 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1102 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1103 case 'L': |
58 | 1104 if (suffix & Suffix.Long) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1105 break; |
60 | 1106 suffix |= Suffix.Long; |
1107 ++p; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1108 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1109 case 'u', 'U': |
58 | 1110 if (suffix & Suffix.Unsigned) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1111 break; |
60 | 1112 suffix |= Suffix.Unsigned; |
1113 ++p; | |
1114 continue; | |
58 | 1115 default: |
1116 break; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1117 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1118 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1119 } |
58 | 1120 |
1121 switch (suffix) | |
1122 { | |
1123 case Suffix.None: | |
1124 if (ulong_ & 0x8000000000000000) | |
1125 { | |
1126 if (isDecimal) | |
1127 error(MID.OverflowDecimalSign); | |
1128 t.type = TOK.Uint64; | |
1129 } | |
1130 else if (ulong_ & 0xFFFFFFFF00000000) | |
1131 t.type = TOK.Int64; | |
1132 else if (ulong_ & 0x80000000) | |
1133 t.type = isDecimal ? TOK.Int64 : TOK.Uint32; | |
1134 else | |
1135 t.type = TOK.Int32; | |
1136 break; | |
1137 case Suffix.Unsigned: | |
1138 if (ulong_ & 0xFFFFFFFF00000000) | |
1139 t.type = TOK.Uint64; | |
1140 else | |
1141 t.type = TOK.Uint32; | |
1142 break; | |
1143 case Suffix.Long: | |
1144 if (ulong_ & 0x8000000000000000) | |
1145 { | |
1146 if (isDecimal) | |
1147 error(MID.OverflowDecimalSign); | |
1148 t.type = TOK.Uint64; | |
1149 } | |
1150 else | |
1151 t.type = TOK.Int64; | |
1152 break; | |
1153 case Suffix.Unsigned | Suffix.Long: | |
1154 t.type = TOK.Uint64; | |
1155 break; | |
1156 default: | |
1157 assert(0); | |
1158 } | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1159 t.ulong_ = ulong_; |
15 | 1160 t.end = p; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1161 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1162 LscanReal: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1163 scanReal(t); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1164 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1165 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1166 |
63 | 1167 /* |
1168 FloatLiteral:= Float[fFL]?i? | |
1169 Float:= DecFloat | HexFloat | |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1170 DecFloat:= ([0-9][0-9_]*[.][0-9_]*DecExponent?) | [.][0-9][0-9_]*DecExponent? | [0-9][0-9_]*DecExponent |
63 | 1171 DecExponent:= [eE][+-]?[0-9][0-9_]* |
1172 HexFloat:= 0[xX](HexDigits[.]HexDigits | [.][0-9a-zA-Z]HexDigits? | HexDigits)HexExponent | |
1173 HexExponent:= [pP][+-]?[0-9][0-9_]* | |
1174 */ | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1175 void scanReal(ref Token t) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1176 { |
63 | 1177 if (*p == '.') |
1178 // This function was called by scan() or scanNumber(). | |
1179 while (isdigit(*++p) || *p == '_') {} | |
1180 else | |
1181 { | |
1182 // This function was called by scanNumber(). | |
1183 debug switch (*p) | |
1184 { | |
1185 case 'L': | |
1186 if (p[1] != 'i') | |
1187 assert(0); | |
1188 case 'i', 'f', 'F', 'e', 'E': break; | |
1189 default: assert(0); | |
1190 } | |
1191 } | |
1192 | |
1193 // Scan exponent. | |
1194 if (*p == 'e' || *p == 'E') | |
1195 { | |
1196 ++p; | |
1197 if (*p == '-' || *p == '+') | |
1198 ++p; | |
1199 if (!isdigit(*p)) | |
1200 error(MID.FloatExponentDigitExpected); | |
1201 else | |
64 | 1202 while (isdigit(*++p) || *p == '_') {} |
63 | 1203 } |
1204 | |
1205 // Copy string to buffer ignoring underscores. | |
1206 char[] buffer; | |
1207 char* end = p; | |
1208 p = t.start; | |
1209 do | |
1210 { | |
1211 if (*p == '_') | |
1212 { | |
1213 ++p; | |
1214 continue; | |
1215 } | |
1216 buffer ~= *p; | |
1217 ++p; | |
1218 } while (p != end) | |
1219 buffer ~= 0; | |
1220 finalizeFloat(t, buffer); | |
30 | 1221 } |
1222 | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1223 void scanHexReal(ref Token t) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1224 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1225 assert(*p == '.' || *p == 'i' || *p == 'p' || *p == 'P' || (*p == 'L' && p[1] == 'i')); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1226 MID mid; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1227 if (*p == '.') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1228 while (ishexad(*++p) || *p == '_') {} |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1229 if (*p != 'p' && *p != 'P') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1230 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1231 mid = MID.HexFloatExponentRequired; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1232 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1233 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1234 // Copy mantissa to a buffer ignoring underscores. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1235 char* end = p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1236 p = t.start; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1237 char[] buffer; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1238 do |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1239 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1240 if (*p == '_') |
63 | 1241 { |
1242 ++p; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1243 continue; |
63 | 1244 } |
1245 buffer ~= *p; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1246 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1247 } while (p != end) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1248 |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1249 assert(p == end && (*p == 'p' || *p == 'P')); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1250 // Scan and copy the exponent. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1251 buffer ~= 'p'; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1252 size_t bufflen = buffer.length; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1253 while (1) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1254 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1255 if (*++p == '_') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1256 continue; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1257 if (isdigit(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1258 buffer ~= *p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1259 else |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1260 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1261 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1262 // When the buffer length hasn't changed, no digits were copied. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1263 if (bufflen == buffer.length) { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1264 mid = MID.HexFloatMissingExpDigits; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1265 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1266 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1267 buffer ~= 0; // Terminate for C functions. |
63 | 1268 finalizeFloat(t, buffer); |
1269 return; | |
1270 Lerr: | |
1271 t.type = TOK.Float32; | |
1272 t.end = p; | |
1273 error(mid); | |
1274 } | |
1275 | |
1276 void finalizeFloat(ref Token t, string buffer) | |
1277 { | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1278 // Float number is well-formed. Check suffixes and do conversion. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1279 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1280 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1281 case 'f', 'F': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1282 t.type = TOK.Float32; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1283 t.float_ = strtof(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1284 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1285 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1286 case 'L': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1287 t.type = TOK.Float80; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1288 t.real_ = strtold(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1289 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1290 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1291 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1292 t.type = TOK.Float64; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1293 t.double_ = strtod(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1294 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1295 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1296 if (*p == 'i') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1297 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1298 ++p; |
63 | 1299 t.type += 3; // Switch to imaginary counterpart. |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1300 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1301 if (getErrno == ERANGE) |
63 | 1302 error(MID.OverflowFloatNumber); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1303 t.end = p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1304 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1305 |
30 | 1306 /// Scan special token: #line Integer [Filespec] EndOfLine |
51 | 1307 // TODO: Handle case like: #line 0 #line 2 |
30 | 1308 void scanSpecialToken() |
1309 { | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1310 assert(*p == '#'); |
51 | 1311 |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1312 ++p; |
30 | 1313 MID mid; |
1314 Token t; | |
51 | 1315 uint oldloc = this.loc, newloc; |
30 | 1316 |
51 | 1317 peek(t); |
65 | 1318 if (!(this.loc == oldloc && p == t.start && t.type == TOK.Identifier && t.srcText == "line")) |
51 | 1319 { |
1320 this.loc = oldloc; // reset this.loc because we took a peek at the next token | |
1321 mid = MID.ExpectedIdentifierSTLine; | |
1322 goto Lerr; | |
1323 } | |
1324 p = t.end; // consume token | |
1325 | |
1326 peek(t); | |
1327 if (this.loc == oldloc && t.type == TOK.Number) | |
1328 { | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1329 newloc = t.uint_ - 1; |
51 | 1330 p = t.end; |
1331 } | |
1332 else | |
1333 { | |
1334 this.loc = oldloc; | |
1335 mid = MID.ExpectedNumberAfterSTLine; | |
30 | 1336 goto Lerr; |
1337 } | |
1338 | |
51 | 1339 peek(t); |
1340 if (this.loc != oldloc) | |
30 | 1341 { |
51 | 1342 this.loc = oldloc; |
1343 mid = MID.NewlineInSpecialToken; | |
1344 goto Lerr; | |
30 | 1345 } |
1346 if (t.type == TOK.String) | |
1347 { | |
51 | 1348 if (*t.start != '"') |
1349 { | |
1350 mid = MID.ExpectedNormalStringLiteral; | |
30 | 1351 goto Lerr; |
51 | 1352 } |
65 | 1353 fileName = t.srcText[1..$-1]; // contents of "..." |
30 | 1354 p = t.end; |
1355 } | |
65 | 1356 else if (t.type == TOK.Identifier && t.srcText == "__FILE__") |
30 | 1357 { |
1358 p = t.end; | |
1359 } | |
51 | 1360 /+ |
1361 peek(t); | |
1362 if (this.loc == oldloc && t.type != TOK.EOF) | |
1363 { | |
1364 mid = MID.UnterminatedSpecialToken; | |
1365 goto Lerr; | |
1366 } | |
1367 +/ | |
30 | 1368 while (1) |
1369 { | |
51 | 1370 switch (*p) |
1371 { | |
1372 case '\r': | |
1373 if (p[1] == '\n') | |
1374 ++p; | |
1375 case '\n': | |
1376 ++p; | |
30 | 1377 break; |
51 | 1378 case LS[0]: |
1379 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) | |
1380 { | |
1381 p += 2; | |
1382 break; | |
1383 } | |
1384 case 0, _Z_: | |
1385 break; | |
1386 default: | |
1387 if (isspace(*p)) { | |
1388 ++p; | |
1389 continue; | |
1390 } | |
30 | 1391 mid = MID.UnterminatedSpecialToken; |
1392 goto Lerr; | |
1393 } | |
51 | 1394 break; |
30 | 1395 } |
1396 | |
51 | 1397 this.loc = newloc; |
30 | 1398 return; |
1399 Lerr: | |
1400 error(mid); | |
15 | 1401 } |
1402 | |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1403 dchar decodeUTF8() |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1404 { |
49 | 1405 assert(*p & 128, "check for ASCII char before calling decodeUTF8()."); |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1406 size_t idx; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1407 dchar d; |
49 | 1408 try |
1409 { | |
1410 d = std.utf.decode(p[0 .. end-p], idx); | |
1411 p += idx -1; | |
1412 } | |
1413 catch (UtfException e) | |
1414 { | |
1415 error(MID.InvalidUTF8Sequence); | |
1416 // Skip to next valid utf-8 sequence | |
1417 while (UTF8stride[*++p] != 0xFF) {} | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1418 --p; |
49 | 1419 } |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1420 return d; |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1421 } |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1422 |
28 | 1423 void loadKeywords() |
1424 { | |
1425 foreach(k; keywords) | |
1426 idtable[k.str] = k; | |
1427 } | |
1428 | |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1429 void peek(ref Token t) |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1430 { |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1431 // Because peeked tokens are not stored in a linked |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1432 // list we need to switch off error reporting |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1433 // so as to avoid getting the same error more than once. |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1434 reportErrors = false; |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1435 char* save = p; |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1436 if (t.end) // For successive peeks. |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1437 { |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1438 p = t.end; |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1439 assert(text.ptr <= p && p < end); |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1440 } |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1441 scan(t); |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1442 p = save; |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1443 reportErrors = true; |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1444 } |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1445 |
66
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
1446 void error(MID id, ...) |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1447 { |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1448 if (reportErrors) |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1449 errors ~= new Information(InfoType.Lexer, id, loc, arguments(_arguments, _argptr)); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1450 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1451 |
3 | 1452 public TOK nextToken() |
1453 { | |
4 | 1454 scan(this.token); |
1455 return this.token.type; | |
1456 } | |
1457 | |
1458 Token[] getTokens() | |
1459 { | |
1460 Token[] tokens; | |
1461 while (nextToken() != TOK.EOF) | |
1462 tokens ~= this.token; | |
1463 tokens ~= this.token; | |
1464 return tokens; | |
3 | 1465 } |
48 | 1466 |
1467 private void encodeUTF8(inout char[] str, dchar d) | |
1468 { | |
1469 char[6] b; | |
1470 assert(d > 0x7F, "check for ASCII char before calling encodeUTF8()."); | |
1471 if (d < 0x800) | |
1472 { | |
1473 b[0] = 0xC0 | (d >> 6); | |
1474 b[1] = 0x80 | (d & 0x3F); | |
1475 str ~= b[0..2]; | |
1476 } | |
1477 else if (d < 0x10000) | |
1478 { | |
1479 b[0] = 0xE0 | (d >> 12); | |
1480 b[1] = 0x80 | ((d >> 6) & 0x3F); | |
1481 b[2] = 0x80 | (d & 0x3F); | |
1482 str ~= b[0..3]; | |
1483 } | |
1484 else if (d < 0x200000) | |
1485 { | |
1486 b[0] = 0xF0 | (d >> 18); | |
1487 b[1] = 0x80 | ((d >> 12) & 0x3F); | |
1488 b[2] = 0x80 | ((d >> 6) & 0x3F); | |
1489 b[3] = 0x80 | (d & 0x3F); | |
1490 str ~= b[0..4]; | |
1491 } | |
1492 else if (d < 0x4000000) | |
1493 { | |
1494 b[0] = 0xF8 | (d >> 24); | |
1495 b[1] = 0x80 | ((d >> 18) & 0x3F); | |
1496 b[2] = 0x80 | ((d >> 12) & 0x3F); | |
1497 b[3] = 0x80 | ((d >> 6) & 0x3F); | |
1498 b[4] = 0x80 | (d & 0x3F); | |
1499 str ~= b[0..5]; | |
1500 } | |
1501 else if (d < 0x80000000) | |
1502 { | |
1503 b[0] = 0xFC | (d >> 30); | |
1504 b[1] = 0x80 | ((d >> 24) & 0x3F); | |
1505 b[2] = 0x80 | ((d >> 18) & 0x3F); | |
1506 b[3] = 0x80 | ((d >> 12) & 0x3F); | |
1507 b[4] = 0x80 | ((d >> 6) & 0x3F); | |
1508 b[5] = 0x80 | (d & 0x3F); | |
1509 str ~= b[0..6]; | |
1510 } | |
1511 else | |
1512 error(MID.InvalidUnicodeCharacter); | |
1513 } | |
5 | 1514 } |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1515 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1516 unittest |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1517 { |
40 | 1518 string[] toks = [ |
1519 ">", ">=", ">>", ">>=", ">>>", ">>>=", "<", "<=", "<>", | |
1520 "<>=", "<<", "<<=", "!", "!<", "!>", "!<=", "!>=", "!<>", | |
1521 "!<>=", ".", "..", "...", "&", "&&", "&=", "+", "++", | |
1522 "+=", "-", "--", "-=", "=", "==", "~", "~=", "*", | |
1523 "*=", "/", "/=", "^", "^=", "%", "%=", "(", ")", | |
1524 "[", "]", "{", "}", ":", ";", "?", ",", "$" | |
1525 ]; | |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1526 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1527 char[] src; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1528 |
40 | 1529 foreach (op; toks) |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1530 src ~= op ~ " "; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1531 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1532 auto lx = new Lexer(src, ""); |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1533 auto tokens = lx.getTokens(); |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1534 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1535 tokens = tokens[0..$-1]; // exclude TOK.EOF |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1536 |
40 | 1537 assert(tokens.length == toks.length ); |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1538 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1539 foreach (i, t; tokens) |
40 | 1540 assert(t.span == toks[i], std.string.format("Lexed '%s' but expected '%s'", t.span, toks[i])); |
41 | 1541 } |
55 | 1542 |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1543 unittest |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1544 { |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1545 // Numbers unittest |
68
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1546 // 0L 0ULi 0_L 0_UL 0x0U 0x0p2 0_Fi 0_e2 0_F 0_i |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1547 // 0u 0U 0uL 0UL 0L 0LU 0Lu |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1548 // 0Li 0f 0F 0fi 0Fi 0i |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1549 // 0b_1_LU 0b1000u |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1550 // 0x232Lu |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1551 } |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1552 |
55 | 1553 /// ASCII character properties table. |
1554 static const int ptable[256] = [ | |
1555 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0, | |
1556 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1557 32, 0, 0x2200, 0, 0, 0, 0, 0x2700, 0, 0, 0, 0, 0, 0, 0, 0, | |
1558 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0x3f00, | |
1559 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8, | |
1560 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0x5c00, 0, 0,16, | |
1561 0, 0x70c, 0x80c,12,12,12, 0xc0c, 8, 8, 8, 8, 8, 8, 8, 0xa08, 8, | |
1562 8, 8, 0xd08, 8, 0x908, 8, 0xb08, 8, 8, 8, 8, 0, 0, 0, 0, 0, | |
1563 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1564 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1565 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1566 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1567 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1568 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1569 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1570 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 | |
1571 ]; | |
1572 | |
1573 enum CProperty | |
1574 { | |
1575 Octal = 1, | |
1576 Digit = 1<<1, | |
1577 Hex = 1<<2, | |
1578 Alpha = 1<<3, | |
1579 Underscore = 1<<4, | |
1580 Whitespace = 1<<5 | |
1581 } | |
1582 | |
1583 const uint EVMask = 0xFF00; // Bit mask for escape value | |
1584 | |
1585 private alias CProperty CP; | |
1586 int isoctal(char c) { return ptable[c] & CP.Octal; } | |
1587 int isdigit(char c) { return ptable[c] & CP.Digit; } | |
1588 int ishexad(char c) { return ptable[c] & CP.Hex; } | |
1589 int isalpha(char c) { return ptable[c] & CP.Alpha; } | |
1590 int isalnum(char c) { return ptable[c] & (CP.Alpha | CP.Digit); } | |
1591 int isidbeg(char c) { return ptable[c] & (CP.Alpha | CP.Underscore); } | |
1592 int isident(char c) { return ptable[c] & (CP.Alpha | CP.Underscore | CP.Digit); } | |
1593 int isspace(char c) { return ptable[c] & CP.Whitespace; } | |
1594 int char2ev(char c) { return ptable[c] >> 8; /*(ptable[c] & EVMask) >> 8;*/ } | |
1595 | |
1596 version(gen_ptable) | |
1597 static this() | |
1598 { | |
1599 alias ptable p; | |
1600 // Initialize character properties table. | |
1601 for (int i; i < p.length; ++i) | |
1602 { | |
1603 p[i] = 0; | |
1604 if ('0' <= i && i <= '7') | |
1605 p[i] |= CP.Octal; | |
1606 if ('0' <= i && i <= '9') | |
1607 p[i] |= CP.Digit; | |
1608 if (isdigit(i) || 'a' <= i && i <= 'f' || 'A' <= i && i <= 'F') | |
1609 p[i] |= CP.Hex; | |
1610 if ('a' <= i && i <= 'z' || 'A' <= i && i <= 'Z') | |
1611 p[i] |= CP.Alpha; | |
1612 if (i == '_') | |
1613 p[i] |= CP.Underscore; | |
1614 if (i == ' ' || i == '\t' || i == '\v' || i == '\f') | |
1615 p[i] |= CP.Whitespace; | |
1616 } | |
1617 // Store escape sequence values in second byte. | |
1618 assert(CProperty.max <= ubyte.max, "character property flags and escape value byte overlap."); | |
1619 p['\''] |= 39 << 8; | |
1620 p['"'] |= 34 << 8; | |
1621 p['?'] |= 63 << 8; | |
1622 p['\\'] |= 92 << 8; | |
1623 p['a'] |= 7 << 8; | |
1624 p['b'] |= 8 << 8; | |
1625 p['f'] |= 12 << 8; | |
1626 p['n'] |= 10 << 8; | |
1627 p['r'] |= 13 << 8; | |
1628 p['t'] |= 9 << 8; | |
1629 p['v'] |= 11 << 8; | |
1630 // Print a formatted array literal. | |
1631 char[] array = "[\n"; | |
1632 for (int i; i < p.length; ++i) | |
1633 { | |
1634 int c = p[i]; | |
1635 array ~= std.string.format(c>255?" 0x%x,":"%2d,", c, ((i+1) % 16) ? "":"\n"); | |
1636 } | |
1637 array[$-2..$] = "\n]"; | |
1638 writefln(array); | |
1639 } |