Mercurial > projects > dil
annotate trunk/src/Lexer.d @ 57:c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
author | aziz |
---|---|
date | Thu, 28 Jun 2007 12:47:05 +0000 |
parents | 63af7ddf52e1 |
children | 50bb7fc9db44 |
rev | line source |
---|---|
0 | 1 /++ |
2 Author: Aziz Köksal | |
3 License: GPL2 | |
4 +/ | |
5 module Lexer; | |
3 | 6 import Token; |
28 | 7 import Keywords; |
8 import Identifier; | |
54 | 9 import Messages; |
2 | 10 import std.stdio; |
4 | 11 import std.utf; |
12 import std.uni; | |
30 | 13 import std.conv; |
0 | 14 |
11 | 15 const char[3] LS = \u2028; |
16 const char[3] PS = \u2029; | |
17 | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
18 const dchar LSd = 0x2028; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
19 const dchar PSd = 0x2029; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
20 |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
21 const uint _Z_ = 26; /// Control+Z |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
22 |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
23 class Problem |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
24 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
25 enum Type |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
26 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
27 Lexer, |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
28 Parser, |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
29 Semantic |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
30 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
31 |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
32 MID id; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
33 Type type; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
34 uint loc; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
35 this(Type type, MID id, uint loc) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
36 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
37 this.id = id; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
38 this.type = type; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
39 this.loc = loc; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
40 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
41 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
42 |
0 | 43 class Lexer |
44 { | |
4 | 45 Token token; |
30 | 46 string text; |
4 | 47 char* p; |
48 char* end; | |
49 | |
17 | 50 uint loc = 1; /// line of code |
51 | |
30 | 52 char[] fileName; |
53 | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
54 Problem[] errors; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
55 |
28 | 56 Identifier[string] idtable; |
57 | |
30 | 58 this(string text, string fileName) |
4 | 59 { |
30 | 60 this.fileName = fileName; |
61 | |
4 | 62 this.text = text; |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
63 if (text[$-1] != 0) |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
64 { |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
65 this.text.length = this.text.length + 1; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
66 this.text[$-1] = 0; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
67 } |
4 | 68 |
69 this.p = this.text.ptr; | |
70 this.end = this.p + this.text.length; | |
28 | 71 |
72 loadKeywords(); | |
4 | 73 } |
74 | |
75 public void scan(out Token t) | |
3 | 76 { |
4 | 77 assert(p < end); |
0 | 78 |
10 | 79 uint c = *p; |
4 | 80 |
5 | 81 while(1) |
4 | 82 { |
5 | 83 t.start = p; |
16 | 84 |
5 | 85 if (c == 0) |
86 { | |
51 | 87 assert(*p == 0); |
17 | 88 ++p; |
51 | 89 assert(p == end); |
5 | 90 t.type = TOK.EOF; |
17 | 91 t.end = p; |
5 | 92 return; |
93 } | |
4 | 94 |
17 | 95 if (c == '\n') |
96 { | |
97 c = *++p; | |
98 ++loc; | |
99 continue; | |
100 } | |
101 else if (c == '\r') | |
102 { | |
103 c = *++p; | |
104 if (c != '\n') | |
105 ++loc; | |
106 continue; | |
107 } | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
108 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
109 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
110 p += 3; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
111 c = *p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
112 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
113 } |
17 | 114 |
13 | 115 if (isidbeg(c)) |
5 | 116 { |
33 | 117 if (c == 'r' && p[1] == '"' && ++p) |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
118 return scanRawStringLiteral(t); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
119 if (c == 'x' && p[1] == '"') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
120 return scanHexStringLiteral(t); |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
121 Lidentifier: |
5 | 122 do |
123 { c = *++p; } | |
49 | 124 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8())) |
28 | 125 |
5 | 126 t.end = p; |
28 | 127 |
128 string str = t.span; | |
129 Identifier* id = str in idtable; | |
130 | |
131 if (!id) | |
132 { | |
133 idtable[str] = Identifier.Identifier(TOK.Identifier, str); | |
134 id = str in idtable; | |
135 } | |
136 assert(id); | |
137 t.type = id.type; | |
5 | 138 return; |
139 } | |
140 | |
15 | 141 if (isdigit(c)) |
142 return scanNumber(t); | |
143 | |
8 | 144 if (c == '/') |
5 | 145 { |
8 | 146 c = *++p; |
14 | 147 switch(c) |
5 | 148 { |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
149 case '=': |
14 | 150 ++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
151 t.type = TOK.DivAssign; |
14 | 152 t.end = p; |
153 return; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
154 case '+': |
8 | 155 uint level = 1; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
156 while (1) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
157 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
158 c = *++p; |
42 | 159 LswitchNC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
160 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
161 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
162 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
163 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
164 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
165 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
166 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
167 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
168 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
169 error(MID.UnterminatedNestedComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
170 goto LreturnNC; |
42 | 171 default: |
172 } | |
173 | |
174 c <<= 8; | |
175 c |= *++p; | |
176 switch (c) | |
177 { | |
178 case 0x2F2B: // /+ | |
179 ++level; | |
180 continue; | |
181 case 0x2B2F: // +/ | |
182 if (--level == 0) | |
183 { | |
184 ++p; | |
185 LreturnNC: | |
186 t.type = TOK.Comment; | |
187 t.end = p; | |
188 return; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
189 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
190 continue; |
42 | 191 case 0xE280: // LS[0..1] || PS[0..1] |
192 if (p[1] == LS[2] || p[1] == PS[2]) | |
193 { | |
194 ++loc; | |
195 ++p; | |
196 } | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
197 continue; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
198 default: |
42 | 199 c &= char.max; |
200 goto LswitchNC; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
201 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
202 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
203 case '*': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
204 while (1) |
7 | 205 { |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
206 c = *++p; |
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
207 LswitchBC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
208 switch (c) |
8 | 209 { |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
210 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
211 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
212 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
213 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
214 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
215 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
216 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
217 error(MID.UnterminatedBlockComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
218 goto LreturnBC; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
219 default: |
8 | 220 } |
41 | 221 |
222 c <<= 8; | |
223 c |= *++p; | |
224 switch (c) | |
225 { | |
226 case 0x2A2F: // */ | |
227 ++p; | |
228 LreturnBC: | |
229 t.type = TOK.Comment; | |
230 t.end = p; | |
231 return; | |
232 case 0xE280: // LS[0..1] || PS[0..1] | |
233 if (p[1] == LS[2] || p[1] == PS[2]) | |
234 { | |
235 ++loc; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
236 ++p; |
41 | 237 } |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
238 continue; |
41 | 239 default: |
240 c &= char.max; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
241 goto LswitchBC; |
41 | 242 } |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
243 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
244 assert(0); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
245 case '/': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
246 while (1) |
7 | 247 { |
8 | 248 c = *++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
249 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
250 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
251 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
252 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
253 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
254 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
255 case 0, _Z_: |
11 | 256 break; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
257 case LS[0]: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
258 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
259 break; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
260 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
261 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
262 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
263 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
264 t.type = TOK.Comment; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
265 t.end = p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
266 return; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
267 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
268 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
269 t.type = TOK.Div; |
10 | 270 t.end = p; |
271 return; | |
272 } | |
5 | 273 } |
274 | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
275 switch (c) |
20 | 276 { |
48 | 277 case '\'': |
278 return scanCharacterLiteral(t); | |
279 case '`': | |
280 return scanRawStringLiteral(t); | |
281 case '"': | |
282 return scanNormalStringLiteral(t); | |
283 case '\\': | |
284 char[] buffer; | |
285 do | |
286 { | |
287 ++p; | |
288 c = scanEscapeSequence(); | |
50 | 289 if (c == 0xFFFF) |
290 break; | |
48 | 291 if (c < 128) |
292 buffer ~= c; | |
293 else | |
294 encodeUTF8(buffer, c); | |
295 } while (*p == '\\') | |
296 buffer ~= 0; | |
297 t.type = TOK.String; | |
298 t.str = buffer; | |
299 t.end = p; | |
300 return; | |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
301 case '>': /* > >= >> >>= >>> >>>= */ |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
302 c = *++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
303 switch (c) |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
304 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
305 case '=': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
306 t.type = TOK.GreaterEqual; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
307 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
308 case '>': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
309 if (p[1] == '>') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
310 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
311 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
312 if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
313 { ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
314 t.type = TOK.URShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
315 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
316 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
317 t.type = TOK.URShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
318 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
319 else if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
320 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
321 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
322 t.type = TOK.RShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
323 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
324 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
325 t.type = TOK.RShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
326 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
327 default: |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
328 t.type = TOK.Greater; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
329 goto Lcommon2; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
330 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
331 assert(0); |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
332 case '<': /* < <= <> <>= << <<= */ |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
333 c = *++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
334 switch (c) |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
335 { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
336 case '=': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
337 t.type = TOK.LessEqual; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
338 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
339 case '<': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
340 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
341 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
342 t.type = TOK.LShiftAssign; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
343 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
344 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
345 t.type = TOK.LShift; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
346 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
347 case '>': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
348 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
349 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
350 t.type = TOK.LorEorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
351 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
352 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
353 t.type = TOK.LorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
354 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
355 default: |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
356 t.type = TOK.Less; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
357 goto Lcommon2; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
358 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
359 assert(0); |
37
7f3bcb97d017
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
36
diff
changeset
|
360 case '!': /* ! !< !> !<= !>= !<> !<>= */ |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
361 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
362 switch (c) |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
363 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
364 case '<': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
365 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
366 if (c == '>') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
367 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
368 if (p[1] == '=') { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
369 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
370 t.type = TOK.Unordered; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
371 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
372 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
373 t.type = TOK.UorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
374 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
375 else if (c == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
376 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
377 t.type = TOK.UorG; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
378 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
379 else { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
380 t.type = TOK.UorGorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
381 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
382 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
383 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
384 case '>': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
385 if (p[1] == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
386 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
387 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
388 t.type = TOK.UorL; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
389 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
390 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
391 t.type = TOK.UorLorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
392 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
393 case '=': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
394 t.type = TOK.NotEqual; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
395 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
396 default: |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
397 t.type = TOK.Not; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
398 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
399 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
400 assert(0); |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
401 case '.': /* . .. ... */ |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
402 if (p[1] == '.') |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
403 { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
404 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
405 if (p[1] == '.') { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
406 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
407 t.type = TOK.Ellipses; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
408 } |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
409 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
410 t.type = TOK.Slice; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
411 } |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
412 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
413 t.type = TOK.Dot; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
414 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
415 case '|': /* | || |= */ |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
416 c = *++p; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
417 if (c == '=') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
418 t.type = TOK.OrAssign; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
419 else if (c == '|') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
420 t.type = TOK.OrLogical; |
26 | 421 else { |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
422 t.type = TOK.OrBinary; |
26 | 423 goto Lcommon2; |
424 } | |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
425 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
426 case '&': /* & && &= */ |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
427 c = *++p; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
428 if (c == '=') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
429 t.type = TOK.AndAssign; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
430 else if (c == '&') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
431 t.type = TOK.AndLogical; |
26 | 432 else { |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
433 t.type = TOK.AndBinary; |
26 | 434 goto Lcommon2; |
435 } | |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
436 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
437 case '+': /* + ++ += */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
438 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
439 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
440 t.type = TOK.PlusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
441 else if (c == '+') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
442 t.type = TOK.PlusPlus; |
26 | 443 else { |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
444 t.type = TOK.Plus; |
26 | 445 goto Lcommon2; |
446 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
447 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
448 case '-': /* - -- -= */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
449 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
450 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
451 t.type = TOK.MinusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
452 else if (c == '-') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
453 t.type = TOK.MinusMinus; |
26 | 454 else { |
455 t.type = TOK.Minus; | |
456 goto Lcommon2; | |
457 } | |
458 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
459 case '=': /* = == */ |
26 | 460 if (p[1] == '=') { |
461 ++p; | |
462 t.type = TOK.Equal; | |
463 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
464 else |
26 | 465 t.type = TOK.Assign; |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
466 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
467 case '~': /* ~ ~= */ |
27 | 468 if (p[1] == '=') { |
469 ++p; | |
470 t.type = TOK.CatAssign; | |
471 } | |
472 else | |
473 t.type = TOK.Tilde; | |
474 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
475 case '*': /* * *= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
476 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
477 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
478 t.type = TOK.MulAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
479 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
480 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
481 t.type = TOK.Mul; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
482 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
483 case '^': /* ^ ^= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
484 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
485 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
486 t.type = TOK.XorAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
487 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
488 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
489 t.type = TOK.Xor; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
490 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
491 case '%': /* % %= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
492 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
493 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
494 t.type = TOK.ModAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
495 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
496 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
497 t.type = TOK.Mod; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
498 goto Lcommon; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
499 // Single character tokens: |
20 | 500 case '(': |
501 t.type = TOK.LParen; | |
502 goto Lcommon; | |
503 case ')': | |
504 t.type = TOK.RParen; | |
505 goto Lcommon; | |
506 case '[': | |
507 t.type = TOK.LBracket; | |
508 goto Lcommon; | |
509 case ']': | |
510 t.type = TOK.RBracket; | |
511 goto Lcommon; | |
512 case '{': | |
513 t.type = TOK.LBrace; | |
514 goto Lcommon; | |
515 case '}': | |
516 t.type = TOK.RBrace; | |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
517 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
518 case ':': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
519 t.type = TOK.Colon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
520 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
521 case ';': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
522 t.type = TOK.Semicolon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
523 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
524 case '?': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
525 t.type = TOK.Question; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
526 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
527 case ',': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
528 t.type = TOK.Comma; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
529 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
530 case '$': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
531 t.type = TOK.Dollar; |
20 | 532 Lcommon: |
533 ++p; | |
26 | 534 Lcommon2: |
20 | 535 t.end = p; |
536 return; | |
30 | 537 case '#': |
538 scanSpecialToken(); | |
51 | 539 c = *p; |
540 continue; | |
20 | 541 default: |
542 } | |
543 | |
49 | 544 if (c & 128 && isUniAlpha(decodeUTF8())) |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
545 goto Lidentifier; |
5 | 546 c = *++p; |
4 | 547 } |
3 | 548 } |
4 | 549 |
30 | 550 void peek(ref Token t) |
551 { | |
552 char* tmp = p; | |
51 | 553 uint len = errors.length; |
30 | 554 scan(t); |
555 p = tmp; | |
51 | 556 if (errors.length != len) |
557 errors = errors[0..len]; | |
30 | 558 } |
559 | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
560 void scanNormalStringLiteral(ref Token t) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
561 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
562 assert(*p == '"'); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
563 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
564 char[] buffer; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
565 t.type = TOK.String; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
566 while (1) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
567 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
568 switch (*p) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
569 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
570 case '"': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
571 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
572 Lreturn: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
573 buffer ~= 0; |
49 | 574 t.str = buffer; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
575 t.pf = scanPostfix(); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
576 t.end = p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
577 return; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
578 case '\\': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
579 ++p; |
49 | 580 dchar d = scanEscapeSequence(); |
50 | 581 if (d == 0xFFFF) |
582 continue; | |
49 | 583 if (d < 128) |
584 buffer ~= d; | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
585 else |
49 | 586 encodeUTF8(buffer, d); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
587 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
588 case '\r': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
589 if (p[1] == '\n') |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
590 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
591 case '\n': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
592 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
593 ++loc; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
594 buffer ~= '\n'; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
595 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
596 case 0, _Z_: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
597 error(MID.UnterminatedString); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
598 goto Lreturn; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
599 default: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
600 if (*p & 128) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
601 { |
49 | 602 char* begin = p; |
603 dchar d = decodeUTF8(); | |
604 if (d == LSd || d == PSd) | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
605 goto case '\n'; |
49 | 606 |
607 if (d != 0xFFFF) | |
608 { | |
609 ++p; | |
610 buffer ~= begin[0 .. p - begin]; | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
611 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
612 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
613 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
614 buffer ~= *p++; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
615 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
616 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
617 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
618 |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
619 void scanCharacterLiteral(ref Token t) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
620 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
621 assert(*p == '\''); |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
622 MID id = MID.UnterminatedCharacterLiteral; |
46 | 623 ++p; |
624 switch (*p) | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
625 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
626 case '\\': |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
627 ++p; |
46 | 628 t.dchar_ = scanEscapeSequence(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
629 break; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
630 case '\'': |
46 | 631 ++p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
632 id = MID.EmptyCharacterLiteral; |
46 | 633 case '\n', '\r', 0, _Z_: |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
634 goto Lerr; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
635 default: |
46 | 636 uint c = *p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
637 if (c & 128) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
638 { |
49 | 639 c = decodeUTF8(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
640 if (c == LSd || c == PSd) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
641 goto Lerr; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
642 } |
46 | 643 t.dchar_ = c; |
644 ++p; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
645 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
646 |
46 | 647 if (*p == '\'') |
648 ++p; | |
649 else | |
650 Lerr: | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
651 error(id); |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
652 t.type = TOK.Character; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
653 t.end = p; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
654 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
655 |
33 | 656 char scanPostfix() |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
657 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
658 switch (*p) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
659 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
660 case 'c': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
661 case 'w': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
662 case 'd': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
663 return *p++; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
664 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
665 return 0; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
666 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
667 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
668 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
669 void scanRawStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
670 { |
33 | 671 uint delim = *p; |
672 assert(delim == '`' || delim == '"' && p[-1] == 'r'); | |
673 t.type = TOK.String; | |
674 char[] buffer; | |
675 uint c; | |
676 while (1) | |
677 { | |
678 c = *++p; | |
679 switch (c) | |
680 { | |
681 case '\r': | |
682 if (p[1] == '\n') | |
683 ++p; | |
684 c = '\n'; // Convert '\r' and '\r\n' to '\n' | |
685 case '\n': | |
686 ++loc; | |
52
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
687 break; |
33 | 688 case '`': |
689 case '"': | |
690 if (c == delim) | |
691 { | |
692 ++p; | |
693 t.pf = scanPostfix(); | |
694 Lreturn: | |
695 t.str = buffer ~ '\0'; | |
696 t.end = p; | |
697 return; | |
698 } | |
699 break; | |
700 case LS[0]: | |
701 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) | |
702 { | |
52
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
703 c = '\n'; |
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
704 ++p; ++p; |
33 | 705 ++loc; |
706 } | |
707 break; | |
708 case 0, _Z_: | |
709 if (delim == 'r') | |
710 error(MID.UnterminatedRawString); | |
711 else | |
712 error(MID.UnterminatedBackQuoteString); | |
713 goto Lreturn; | |
714 default: | |
715 } | |
716 buffer ~= c; // copy character to buffer | |
717 } | |
718 assert(0); | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
719 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
720 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
721 void scanHexStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
722 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
723 assert(p[0] == 'x' && p[1] == '"'); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
724 t.type = TOK.String; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
725 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
726 uint c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
727 ubyte[] buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
728 ubyte h; // hex number |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
729 uint n; // number of hex digits |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
730 |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
731 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
732 while (1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
733 { |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
734 c = *++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
735 switch (c) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
736 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
737 case '"': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
738 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
739 if (n & 1) |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
740 error(MID.OddNumberOfDigitsInHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
741 t.pf = scanPostfix(); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
742 Lreturn: |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
743 buffer ~= 0; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
744 t.str = cast(string) buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
745 t.end = p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
746 return; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
747 case '\r': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
748 if (p[1] == '\n') |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
749 ++p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
750 case '\n': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
751 ++loc; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
752 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
753 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
754 if (ishexad(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
755 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
756 if (c <= '9') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
757 c -= '0'; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
758 else if (c <= 'F') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
759 c -= 'A' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
760 else |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
761 c -= 'a' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
762 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
763 if (n & 1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
764 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
765 h <<= 4; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
766 h |= c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
767 buffer ~= h; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
768 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
769 else |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
770 h = c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
771 ++n; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
772 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
773 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
774 else if (isspace(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
775 continue; |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
776 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
777 { |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
778 ++p; ++p; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
779 ++loc; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
780 continue; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
781 } |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
782 else if (c == 0 || c == _Z_) |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
783 { |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
784 error(MID.UnterminatedHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
785 t.pf = 0; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
786 goto Lreturn; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
787 } |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
788 error(MID.NonHexCharInHexString); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
789 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
790 } |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
791 assert(0); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
792 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
793 |
45 | 794 dchar scanEscapeSequence() |
795 { | |
796 uint c = char2ev(*p); | |
46 | 797 if (c) { |
798 ++p; | |
45 | 799 return c; |
46 | 800 } |
50 | 801 c = 0xFFFF; |
45 | 802 uint digits = 2; |
803 | |
804 switch (*p) | |
805 { | |
806 case 'x': | |
807 c = 0; | |
808 while (1) | |
809 { | |
810 ++p; | |
811 if (ishexad(*p)) | |
812 { | |
813 c *= 16; | |
814 if (*p <= '9') | |
48 | 815 c += *p - '0'; |
45 | 816 else if (*p <= 'F') |
48 | 817 c += *p - 'A' + 10; |
45 | 818 else |
48 | 819 c += *p - 'a' + 10; |
820 if (!--digits) { | |
821 ++p; | |
45 | 822 break; |
48 | 823 } |
45 | 824 } |
825 else | |
826 { | |
827 error(MID.InsufficientHexDigits); | |
50 | 828 c = 0xFFFF; |
45 | 829 break; |
830 } | |
831 } | |
832 break; | |
833 case 'u': | |
834 digits = 4; | |
835 goto case 'x'; | |
836 case 'U': | |
837 digits = 8; | |
838 goto case 'x'; | |
839 default: | |
48 | 840 if (isoctal(*p)) |
45 | 841 { |
48 | 842 c = 0; |
843 c += *p - '0'; | |
844 ++p; | |
845 if (!isoctal(*p)) | |
846 return c; | |
847 c *= 8; | |
848 c += *p - '0'; | |
849 ++p; | |
850 if (!isoctal(*p)) | |
851 return c; | |
852 c *= 8; | |
853 c += *p - '0'; | |
854 ++p; | |
855 } | |
856 else if(*p == '&') | |
857 { | |
858 if (isalpha(*++p)) | |
45 | 859 { |
48 | 860 while (1) |
861 { | |
862 if (isalnum(*++p)) | |
863 continue; | |
864 if (*p == ';') { | |
865 // TODO: convert entity to unicode codepoint. | |
866 ++p; | |
867 break; | |
868 } | |
869 else { | |
870 error(MID.UnterminatedHTMLEntity); | |
871 break; | |
872 } | |
45 | 873 } |
874 } | |
48 | 875 else |
876 error(MID.InvalidBeginHTMLEntity); | |
45 | 877 } |
878 else | |
48 | 879 error(MID.UndefinedEscapeSequence); |
45 | 880 } |
881 | |
882 return c; | |
883 } | |
884 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
885 /* |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
886 IntegerLiteral:= (Dec|Hex|Bin|Oct)Suffix? |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
887 Dec:= (0|[1-9][0-9_]*) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
888 Hex:= 0[xX] HexDigits |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
889 Bin:= 0[bB][01_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
890 Oct:= 0[0-7_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
891 Suffix:= (L|[uU]|L[uU]|[uU]L)? |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
892 HexDigits:= [0-9a-zA-Z_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
893 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
894 FloatLiteral:= Float([fFL]|i|[fFL]i)? |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
895 Float:= DecFloat | HexFloat |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
896 DecFloat:= ([0-9][0-9_]*[.]([0-9_]*DecExponent?)?) | [.][0-9][0-9_]*DecExponent? | [0-9][0-9_]*DecExponent |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
897 DecExponent:= [eE][+-]?[0-9_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
898 HexFloat:= 0[xX](HexDigits[.]HexDigits | [.][0-9a-zA-Z]HexDigits? | HexDigits)HexExponent |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
899 HexExponent:= [pP][+-]?[0-9_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
900 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
901 Invalid: "0b_", "0x_", "._" |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
902 */ |
15 | 903 void scanNumber(ref Token t) |
904 { | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
905 enum Suffix |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
906 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
907 None = 0, |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
908 Unsigned = 1, |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
909 Long = 1<<1 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
910 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
911 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
912 ulong ulong_; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
913 bool overflow; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
914 size_t digits; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
915 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
916 if (*p != '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
917 goto LscanInteger; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
918 ++p; // skip zero |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
919 // check for xX bB ... |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
920 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
921 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
922 case 'x','X': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
923 goto LscanHex; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
924 case 'b','B': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
925 goto LscanBin; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
926 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
927 if (p[1] == 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
928 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
929 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
930 if (p[1] == '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
931 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
932 case 'i','f','F': // Imaginary and float literal suffix |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
933 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
934 case '_': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
935 ++p; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
936 goto LscanOct; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
937 default: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
938 if (isoctal(*p)) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
939 goto LscanOct; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
940 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
941 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
942 LscanInteger: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
943 while (isdigit(*++p)) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
944 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
945 if (*p == '_') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
946 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
947 if (ulong_ < ulong.max/10 || (ulong_ == ulong.max/10 && *p <= '5')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
948 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
949 ulong_ *= 10; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
950 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
951 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
952 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
953 overflow = true; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
954 while (isdigit(*++p)) {} |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
955 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
956 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
957 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
958 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
959 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
960 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
961 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
962 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
963 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
964 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
965 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
966 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
967 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
968 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
969 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
970 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
971 error(MID.OverflowDecimalNumber); |
15 | 972 t.type = TOK.Number; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
973 // ulong_ = toInt(t.span); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
974 assert((isdigit(p[-1]) || p[-1] == '_') && !isdigit(*p) && *p != '_'); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
975 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
976 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
977 LscanHex: |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
978 digits = 16; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
979 while (ishexad(*++p)) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
980 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
981 if (*p == '_') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
982 continue; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
983 |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
984 if (--digits == 0) |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
985 { |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
986 // Overflow: skip following digits. |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
987 overflow = true; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
988 while (ishexad(*++p)) {} |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
989 break; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
990 } |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
991 |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
992 ulong_ *= 16; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
993 if (*p <= '9') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
994 ulong_ += *p - '0'; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
995 else if (*p <= 'F') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
996 ulong_ += *p - 'A' + 10; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
997 else |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
998 ulong_ += *p - 'a' + 10; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
999 } |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1000 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1001 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1002 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1003 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1004 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1005 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1006 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1007 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1008 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1009 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1010 case 'i', 'p', 'P': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1011 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1012 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1013 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1014 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1015 LscanBin: |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1016 assert(digits == 0); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1017 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1018 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1019 if (*++p == '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1020 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1021 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1022 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1023 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1024 if (*p == '1') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1025 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1026 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1027 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1028 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1029 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1030 if (*p == '_') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1031 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1032 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1033 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1034 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1035 if (digits > 64) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1036 error(MID.OverflowBinaryNumber); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1037 assert((p[-1] == '0' || p[-1] == '1' || p[-1] == '_') && !(*p == '0' || *p == '1' || *p == '_')); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1038 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1039 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1040 LscanOct: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1041 while (isoctal(*++p)) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1042 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1043 if (*p == '_') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1044 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1045 if (ulong_ < ulong.max/2 || (ulong_ == ulong.max/2 && *p <= '1')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1046 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1047 ulong_ *= 8; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1048 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1049 ++p; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1050 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1051 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1052 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1053 overflow = true; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1054 while (isoctal(*++p)) {} |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1055 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1056 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1057 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1058 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1059 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1060 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1061 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1062 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1063 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1064 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1065 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1066 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1067 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1068 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1069 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1070 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1071 error(MID.OverflowOctalNumber); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1072 // goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1073 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1074 Lfinalize: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1075 Suffix s; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1076 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1077 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1078 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1079 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1080 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1081 if (s & Suffix.Long) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1082 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1083 s = Suffix.Long; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1084 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1085 case 'u', 'U': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1086 if (s & Suffix.Unsigned) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1087 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1088 s = Suffix.Unsigned; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1089 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1090 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1091 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1092 t.ulong_ = ulong_; |
15 | 1093 t.end = p; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1094 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1095 LscanReal: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1096 p = t.start; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1097 scanReal(t); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1098 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1099 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1100 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1101 void scanReal(ref Token t) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1102 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1103 assert(*p == '.' || isdigit(*p)); |
30 | 1104 } |
1105 | |
1106 /// Scan special token: #line Integer [Filespec] EndOfLine | |
51 | 1107 // TODO: Handle case like: #line 0 #line 2 |
30 | 1108 void scanSpecialToken() |
1109 { | |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1110 assert(*p == '#'); |
51 | 1111 |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1112 ++p; |
30 | 1113 MID mid; |
1114 Token t; | |
51 | 1115 uint oldloc = this.loc, newloc; |
30 | 1116 |
51 | 1117 peek(t); |
1118 if (!(this.loc == oldloc && p == t.start && t.type == TOK.Identifier && t.span == "line")) | |
1119 { | |
1120 this.loc = oldloc; // reset this.loc because we took a peek at the next token | |
1121 mid = MID.ExpectedIdentifierSTLine; | |
1122 goto Lerr; | |
1123 } | |
1124 p = t.end; // consume token | |
1125 | |
1126 peek(t); | |
1127 if (this.loc == oldloc && t.type == TOK.Number) | |
1128 { | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1129 newloc = t.uint_ - 1; |
51 | 1130 p = t.end; |
1131 } | |
1132 else | |
1133 { | |
1134 this.loc = oldloc; | |
1135 mid = MID.ExpectedNumberAfterSTLine; | |
30 | 1136 goto Lerr; |
1137 } | |
1138 | |
51 | 1139 peek(t); |
1140 if (this.loc != oldloc) | |
30 | 1141 { |
51 | 1142 this.loc = oldloc; |
1143 mid = MID.NewlineInSpecialToken; | |
1144 goto Lerr; | |
30 | 1145 } |
1146 if (t.type == TOK.String) | |
1147 { | |
51 | 1148 if (*t.start != '"') |
1149 { | |
1150 mid = MID.ExpectedNormalStringLiteral; | |
30 | 1151 goto Lerr; |
51 | 1152 } |
30 | 1153 fileName = t.span[1..$-1]; // contents of "..." |
1154 p = t.end; | |
1155 } | |
1156 else if (t.type == TOK.Identifier && t.span == "__FILE__") | |
1157 { | |
1158 p = t.end; | |
1159 } | |
51 | 1160 /+ |
1161 peek(t); | |
1162 if (this.loc == oldloc && t.type != TOK.EOF) | |
1163 { | |
1164 mid = MID.UnterminatedSpecialToken; | |
1165 goto Lerr; | |
1166 } | |
1167 +/ | |
30 | 1168 while (1) |
1169 { | |
51 | 1170 switch (*p) |
1171 { | |
1172 case '\r': | |
1173 if (p[1] == '\n') | |
1174 ++p; | |
1175 case '\n': | |
1176 ++p; | |
30 | 1177 break; |
51 | 1178 case LS[0]: |
1179 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) | |
1180 { | |
1181 p += 2; | |
1182 break; | |
1183 } | |
1184 case 0, _Z_: | |
1185 break; | |
1186 default: | |
1187 if (isspace(*p)) { | |
1188 ++p; | |
1189 continue; | |
1190 } | |
30 | 1191 mid = MID.UnterminatedSpecialToken; |
1192 goto Lerr; | |
1193 } | |
51 | 1194 break; |
30 | 1195 } |
1196 | |
51 | 1197 this.loc = newloc; |
30 | 1198 return; |
1199 Lerr: | |
1200 error(mid); | |
15 | 1201 } |
1202 | |
49 | 1203 uint decodeUTF8() |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1204 { |
49 | 1205 assert(*p & 128, "check for ASCII char before calling decodeUTF8()."); |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1206 size_t idx; |
49 | 1207 uint d = 0xFFFF; |
1208 try | |
1209 { | |
1210 d = std.utf.decode(p[0 .. end-p], idx); | |
1211 p += idx -1; | |
1212 } | |
1213 catch (UtfException e) | |
1214 { | |
1215 error(MID.InvalidUTF8Sequence); | |
1216 // Skip to next valid utf-8 sequence | |
1217 while (UTF8stride[*++p] != 0xFF) {} | |
1218 } | |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1219 return d; |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1220 } |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1221 |
28 | 1222 void loadKeywords() |
1223 { | |
1224 foreach(k; keywords) | |
1225 idtable[k.str] = k; | |
1226 } | |
1227 | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1228 void error(MID id) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1229 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1230 errors ~= new Problem(Problem.Type.Lexer, id, loc); |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1231 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1232 |
3 | 1233 public TOK nextToken() |
1234 { | |
4 | 1235 scan(this.token); |
1236 return this.token.type; | |
1237 } | |
1238 | |
1239 Token[] getTokens() | |
1240 { | |
1241 Token[] tokens; | |
1242 while (nextToken() != TOK.EOF) | |
1243 tokens ~= this.token; | |
1244 tokens ~= this.token; | |
1245 return tokens; | |
3 | 1246 } |
48 | 1247 |
1248 private void encodeUTF8(inout char[] str, dchar d) | |
1249 { | |
1250 char[6] b; | |
1251 assert(d > 0x7F, "check for ASCII char before calling encodeUTF8()."); | |
1252 if (d < 0x800) | |
1253 { | |
1254 b[0] = 0xC0 | (d >> 6); | |
1255 b[1] = 0x80 | (d & 0x3F); | |
1256 str ~= b[0..2]; | |
1257 } | |
1258 else if (d < 0x10000) | |
1259 { | |
1260 b[0] = 0xE0 | (d >> 12); | |
1261 b[1] = 0x80 | ((d >> 6) & 0x3F); | |
1262 b[2] = 0x80 | (d & 0x3F); | |
1263 str ~= b[0..3]; | |
1264 } | |
1265 else if (d < 0x200000) | |
1266 { | |
1267 b[0] = 0xF0 | (d >> 18); | |
1268 b[1] = 0x80 | ((d >> 12) & 0x3F); | |
1269 b[2] = 0x80 | ((d >> 6) & 0x3F); | |
1270 b[3] = 0x80 | (d & 0x3F); | |
1271 str ~= b[0..4]; | |
1272 } | |
1273 else if (d < 0x4000000) | |
1274 { | |
1275 b[0] = 0xF8 | (d >> 24); | |
1276 b[1] = 0x80 | ((d >> 18) & 0x3F); | |
1277 b[2] = 0x80 | ((d >> 12) & 0x3F); | |
1278 b[3] = 0x80 | ((d >> 6) & 0x3F); | |
1279 b[4] = 0x80 | (d & 0x3F); | |
1280 str ~= b[0..5]; | |
1281 } | |
1282 else if (d < 0x80000000) | |
1283 { | |
1284 b[0] = 0xFC | (d >> 30); | |
1285 b[1] = 0x80 | ((d >> 24) & 0x3F); | |
1286 b[2] = 0x80 | ((d >> 18) & 0x3F); | |
1287 b[3] = 0x80 | ((d >> 12) & 0x3F); | |
1288 b[4] = 0x80 | ((d >> 6) & 0x3F); | |
1289 b[5] = 0x80 | (d & 0x3F); | |
1290 str ~= b[0..6]; | |
1291 } | |
1292 else | |
1293 error(MID.InvalidUnicodeCharacter); | |
1294 } | |
5 | 1295 } |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1296 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1297 unittest |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1298 { |
40 | 1299 string[] toks = [ |
1300 ">", ">=", ">>", ">>=", ">>>", ">>>=", "<", "<=", "<>", | |
1301 "<>=", "<<", "<<=", "!", "!<", "!>", "!<=", "!>=", "!<>", | |
1302 "!<>=", ".", "..", "...", "&", "&&", "&=", "+", "++", | |
1303 "+=", "-", "--", "-=", "=", "==", "~", "~=", "*", | |
1304 "*=", "/", "/=", "^", "^=", "%", "%=", "(", ")", | |
1305 "[", "]", "{", "}", ":", ";", "?", ",", "$" | |
1306 ]; | |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1307 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1308 char[] src; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1309 |
40 | 1310 foreach (op; toks) |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1311 src ~= op ~ " "; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1312 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1313 auto lx = new Lexer(src, ""); |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1314 auto tokens = lx.getTokens(); |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1315 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1316 tokens = tokens[0..$-1]; // exclude TOK.EOF |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1317 |
40 | 1318 assert(tokens.length == toks.length ); |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1319 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
1320 foreach (i, t; tokens) |
40 | 1321 assert(t.span == toks[i], std.string.format("Lexed '%s' but expected '%s'", t.span, toks[i])); |
41 | 1322 } |
55 | 1323 |
1324 /// ASCII character properties table. | |
1325 static const int ptable[256] = [ | |
1326 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0, | |
1327 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1328 32, 0, 0x2200, 0, 0, 0, 0, 0x2700, 0, 0, 0, 0, 0, 0, 0, 0, | |
1329 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0x3f00, | |
1330 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8, | |
1331 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0x5c00, 0, 0,16, | |
1332 0, 0x70c, 0x80c,12,12,12, 0xc0c, 8, 8, 8, 8, 8, 8, 8, 0xa08, 8, | |
1333 8, 8, 0xd08, 8, 0x908, 8, 0xb08, 8, 8, 8, 8, 0, 0, 0, 0, 0, | |
1334 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1335 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1336 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1337 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1338 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1339 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1340 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
1341 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 | |
1342 ]; | |
1343 | |
1344 enum CProperty | |
1345 { | |
1346 Octal = 1, | |
1347 Digit = 1<<1, | |
1348 Hex = 1<<2, | |
1349 Alpha = 1<<3, | |
1350 Underscore = 1<<4, | |
1351 Whitespace = 1<<5 | |
1352 } | |
1353 | |
1354 const uint EVMask = 0xFF00; // Bit mask for escape value | |
1355 | |
1356 private alias CProperty CP; | |
1357 int isoctal(char c) { return ptable[c] & CP.Octal; } | |
1358 int isdigit(char c) { return ptable[c] & CP.Digit; } | |
1359 int ishexad(char c) { return ptable[c] & CP.Hex; } | |
1360 int isalpha(char c) { return ptable[c] & CP.Alpha; } | |
1361 int isalnum(char c) { return ptable[c] & (CP.Alpha | CP.Digit); } | |
1362 int isidbeg(char c) { return ptable[c] & (CP.Alpha | CP.Underscore); } | |
1363 int isident(char c) { return ptable[c] & (CP.Alpha | CP.Underscore | CP.Digit); } | |
1364 int isspace(char c) { return ptable[c] & CP.Whitespace; } | |
1365 int char2ev(char c) { return ptable[c] >> 8; /*(ptable[c] & EVMask) >> 8;*/ } | |
1366 | |
1367 version(gen_ptable) | |
1368 static this() | |
1369 { | |
1370 alias ptable p; | |
1371 // Initialize character properties table. | |
1372 for (int i; i < p.length; ++i) | |
1373 { | |
1374 p[i] = 0; | |
1375 if ('0' <= i && i <= '7') | |
1376 p[i] |= CP.Octal; | |
1377 if ('0' <= i && i <= '9') | |
1378 p[i] |= CP.Digit; | |
1379 if (isdigit(i) || 'a' <= i && i <= 'f' || 'A' <= i && i <= 'F') | |
1380 p[i] |= CP.Hex; | |
1381 if ('a' <= i && i <= 'z' || 'A' <= i && i <= 'Z') | |
1382 p[i] |= CP.Alpha; | |
1383 if (i == '_') | |
1384 p[i] |= CP.Underscore; | |
1385 if (i == ' ' || i == '\t' || i == '\v' || i == '\f') | |
1386 p[i] |= CP.Whitespace; | |
1387 } | |
1388 // Store escape sequence values in second byte. | |
1389 assert(CProperty.max <= ubyte.max, "character property flags and escape value byte overlap."); | |
1390 p['\''] |= 39 << 8; | |
1391 p['"'] |= 34 << 8; | |
1392 p['?'] |= 63 << 8; | |
1393 p['\\'] |= 92 << 8; | |
1394 p['a'] |= 7 << 8; | |
1395 p['b'] |= 8 << 8; | |
1396 p['f'] |= 12 << 8; | |
1397 p['n'] |= 10 << 8; | |
1398 p['r'] |= 13 << 8; | |
1399 p['t'] |= 9 << 8; | |
1400 p['v'] |= 11 << 8; | |
1401 // Print a formatted array literal. | |
1402 char[] array = "[\n"; | |
1403 for (int i; i < p.length; ++i) | |
1404 { | |
1405 int c = p[i]; | |
1406 array ~= std.string.format(c>255?" 0x%x,":"%2d,", c, ((i+1) % 16) ? "":"\n"); | |
1407 } | |
1408 array[$-2..$] = "\n]"; | |
1409 writefln(array); | |
1410 } |