Mercurial > projects > dil
annotate trunk/src/dil/Lexer.d @ 388:ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Fix: Token.line_num can be null which could cause a segfault in printToken().
Renamed MID.ExpectedNumberAfterSTLine to ExpectedIntegerAfterSTLine.
Fix: Added a check for TOK.Int32 and TOK.Uint32 after a number was scanned
in scanSpecialTokenSequence().
author | Aziz K?ksal <aziz.koeksal@gmail.com> |
---|---|
date | Wed, 12 Sep 2007 13:43:45 +0200 |
parents | ad0cbd1c8881 |
children | c4bfceab7246 |
rev | line source |
---|---|
0 | 1 /++ |
2 Author: Aziz Köksal | |
249 | 3 License: GPL3 |
0 | 4 +/ |
326 | 5 module dil.Lexer; |
327 | 6 import dil.Token; |
7 import dil.Information; | |
8 import dil.Keywords; | |
9 import dil.Identifier; | |
10 import dil.Messages; | |
11 import dil.HtmlEntities; | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
12 import dil.Settings; |
2 | 13 import std.stdio; |
4 | 14 import std.utf; |
15 import std.uni; | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
16 import std.c.stdlib : strtof, strtod, strtold, getErrno, ERANGE; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
17 import std.c.time : time_t, time, ctime; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
18 import std.c.string : strlen; |
66
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
19 import std.string; |
0 | 20 |
11 | 21 const char[3] LS = \u2028; |
22 const char[3] PS = \u2029; | |
23 | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
24 const dchar LSd = 0x2028; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
25 const dchar PSd = 0x2029; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
26 |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
27 const uint _Z_ = 26; /// Control+Z |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
28 |
0 | 29 class Lexer |
30 { | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
31 Token* head; /// The head of the doubly linked token list. |
344 | 32 Token* tail; /// The tail of the linked list. Set in scan(). |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
33 Token* token; /// Points to the current token in the token list. |
30 | 34 string text; |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
35 char* p; /// Points to the current character in the source text. |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
36 char* end; /// Points one character past the end of the source text. |
4 | 37 |
371 | 38 uint loc = 1; /// Actual line of code. |
39 | |
40 uint loc_old; /// Store actual line number when #line token is parsed. | |
41 uint loc_hline; /// Line number set by #line. | |
17 | 42 |
30 | 43 char[] fileName; |
44 | |
69
24db7c5522d5
- Added module Information for compiler messages like warnings, info and errors to the user.
aziz
parents:
68
diff
changeset
|
45 Information[] errors; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
46 |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
47 // bool reportErrors; |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
48 |
28 | 49 Identifier[string] idtable; |
50 | |
30 | 51 this(string text, string fileName) |
4 | 52 { |
30 | 53 this.fileName = fileName; |
54 | |
4 | 55 this.text = text; |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
56 if (text[$-1] != 0) |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
57 { |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
58 this.text.length = this.text.length + 1; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
59 this.text[$-1] = 0; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
60 } |
4 | 61 |
62 this.p = this.text.ptr; | |
63 this.end = this.p + this.text.length; | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
64 // this.reportErrors = true; |
28 | 65 loadKeywords(); |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
66 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
67 this.head = new Token; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
68 this.head.type = TOK.HEAD; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
69 this.token = this.head; |
315 | 70 scanShebang(); |
71 } | |
72 | |
344 | 73 ~this() |
74 { | |
75 auto token = head.next; | |
76 do | |
77 { | |
78 assert(token.type == TOK.EOF ? token == tail && token.next is null : 1); | |
79 delete token.prev; | |
80 token = token.next; | |
81 } while (token !is null) | |
82 delete tail; | |
83 } | |
84 | |
315 | 85 void scanShebang() |
86 { | |
87 if (*p == '#' && p[1] == '!') | |
88 { | |
89 Token* t = new Token; | |
90 t.start = p; | |
91 t.type = TOK.Shebang; | |
92 ++p; | |
93 while (1) | |
94 { | |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
95 t.end = p; |
315 | 96 switch (*++p) |
97 { | |
98 case '\r': | |
99 if (p[1] == '\n') | |
100 ++p; | |
101 case '\n': | |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
102 ++p; |
315 | 103 ++loc; |
104 break; | |
105 case 0, _Z_: | |
106 break; | |
107 default: | |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
108 if (*p & 128) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
109 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
110 auto c = decodeUTF8(); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
111 if (c == LSd || c == PSd) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
112 goto case '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
113 } |
315 | 114 continue; |
115 } | |
116 break; // Exit loop. | |
117 } | |
118 this.head.next = t; | |
119 t.prev = this.head; | |
120 } | |
4 | 121 } |
122 | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
123 void finalizeSpecialToken(ref Token t) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
124 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
125 assert(t.srcText[0..2] == "__"); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
126 switch (t.type) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
127 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
128 case TOK.FILE: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
129 t.str = this.fileName; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
130 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
131 case TOK.LINE: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
132 t.uint_ = this.loc; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
133 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
134 case TOK.DATE, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
135 TOK.TIME, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
136 TOK.TIMESTAMP: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
137 time_t time_val; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
138 time(&time_val); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
139 char* str = ctime(&time_val); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
140 char[] time_str = str[0 .. strlen(str)]; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
141 switch (t.type) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
142 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
143 case TOK.DATE: |
346
ce9e2e77743f
- Fix: include one space when slicing time_str for __DATE__.
aziz
parents:
344
diff
changeset
|
144 time_str = time_str[4..11] ~ time_str[20..24] ~ \0; break; |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
145 case TOK.TIME: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
146 time_str = time_str[11..19] ~ \0; break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
147 case TOK.TIMESTAMP: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
148 time_str = time_str[0..24] ~ \0; break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
149 default: assert(0); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
150 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
151 t.str = time_str; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
152 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
153 case TOK.VENDOR: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
154 t.str = VENDOR; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
155 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
156 case TOK.VERSION: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
157 t.uint_ = VERSION_MAJOR*1000 + VERSION_MINOR; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
158 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
159 default: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
160 assert(0); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
161 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
162 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
163 |
4 | 164 public void scan(out Token t) |
207 | 165 in |
3 | 166 { |
207 | 167 assert(text.ptr <= p && p < end); |
168 } | |
169 out | |
170 { | |
171 assert(text.ptr <= t.start && t.start < end); | |
172 assert(text.ptr < t.end && t.end <= end, std.string.format(t.type)); | |
173 } | |
174 body | |
175 { | |
10 | 176 uint c = *p; |
4 | 177 |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
178 while (1) |
4 | 179 { |
5 | 180 t.start = p; |
16 | 181 |
212 | 182 if (c == 0 || c == _Z_) |
5 | 183 { |
212 | 184 assert(*p == 0 || *p == _Z_); |
5 | 185 t.type = TOK.EOF; |
17 | 186 t.end = p; |
344 | 187 tail = &t; |
212 | 188 assert(t.start == t.end); |
5 | 189 return; |
190 } | |
4 | 191 |
17 | 192 if (c == '\n') |
193 { | |
194 c = *++p; | |
195 ++loc; | |
196 continue; | |
197 } | |
198 else if (c == '\r') | |
199 { | |
200 c = *++p; | |
201 if (c != '\n') | |
202 ++loc; | |
203 continue; | |
204 } | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
205 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
206 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
207 p += 3; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
208 c = *p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
209 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
210 } |
17 | 211 |
13 | 212 if (isidbeg(c)) |
5 | 213 { |
33 | 214 if (c == 'r' && p[1] == '"' && ++p) |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
215 return scanRawStringLiteral(t); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
216 if (c == 'x' && p[1] == '"') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
217 return scanHexStringLiteral(t); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
218 version(D2) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
219 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
220 if (c == 'q' && p[1] == '"') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
221 return scanDelimitedStringLiteral(t); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
222 if (c == 'q' && p[1] == '{') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
223 return scanTokenStringLiteral(t); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
224 } |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
225 Lidentifier: |
5 | 226 do |
227 { c = *++p; } | |
49 | 228 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8())) |
28 | 229 |
5 | 230 t.end = p; |
28 | 231 |
65 | 232 string str = t.srcText; |
28 | 233 Identifier* id = str in idtable; |
234 | |
235 if (!id) | |
236 { | |
327 | 237 idtable[str] = Identifier(TOK.Identifier, str); |
28 | 238 id = str in idtable; |
239 } | |
240 assert(id); | |
241 t.type = id.type; | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
242 if (t.isSpecialToken) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
243 finalizeSpecialToken(t); |
5 | 244 return; |
245 } | |
246 | |
15 | 247 if (isdigit(c)) |
248 return scanNumber(t); | |
249 | |
8 | 250 if (c == '/') |
5 | 251 { |
8 | 252 c = *++p; |
14 | 253 switch(c) |
5 | 254 { |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
255 case '=': |
14 | 256 ++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
257 t.type = TOK.DivAssign; |
14 | 258 t.end = p; |
259 return; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
260 case '+': |
8 | 261 uint level = 1; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
262 while (1) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
263 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
264 c = *++p; |
42 | 265 LswitchNC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
266 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
267 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
268 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
269 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
270 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
271 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
272 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
273 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
274 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
275 error(MID.UnterminatedNestedComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
276 goto LreturnNC; |
42 | 277 default: |
278 } | |
279 | |
280 c <<= 8; | |
281 c |= *++p; | |
282 switch (c) | |
283 { | |
284 case 0x2F2B: // /+ | |
285 ++level; | |
286 continue; | |
287 case 0x2B2F: // +/ | |
288 if (--level == 0) | |
289 { | |
290 ++p; | |
291 LreturnNC: | |
292 t.type = TOK.Comment; | |
293 t.end = p; | |
294 return; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
295 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
296 continue; |
42 | 297 case 0xE280: // LS[0..1] || PS[0..1] |
298 if (p[1] == LS[2] || p[1] == PS[2]) | |
299 { | |
300 ++loc; | |
301 ++p; | |
302 } | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
303 continue; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
304 default: |
42 | 305 c &= char.max; |
306 goto LswitchNC; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
307 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
308 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
309 case '*': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
310 while (1) |
7 | 311 { |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
312 c = *++p; |
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
313 LswitchBC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
314 switch (c) |
8 | 315 { |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
316 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
317 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
318 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
319 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
320 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
321 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
322 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
323 error(MID.UnterminatedBlockComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
324 goto LreturnBC; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
325 default: |
8 | 326 } |
41 | 327 |
328 c <<= 8; | |
329 c |= *++p; | |
330 switch (c) | |
331 { | |
332 case 0x2A2F: // */ | |
333 ++p; | |
334 LreturnBC: | |
335 t.type = TOK.Comment; | |
336 t.end = p; | |
337 return; | |
338 case 0xE280: // LS[0..1] || PS[0..1] | |
339 if (p[1] == LS[2] || p[1] == PS[2]) | |
340 { | |
341 ++loc; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
342 ++p; |
41 | 343 } |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
344 continue; |
41 | 345 default: |
346 c &= char.max; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
347 goto LswitchBC; |
41 | 348 } |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
349 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
350 assert(0); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
351 case '/': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
352 while (1) |
7 | 353 { |
8 | 354 c = *++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
355 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
356 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
357 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
358 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
359 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
360 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
361 case 0, _Z_: |
11 | 362 break; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
363 case LS[0]: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
364 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
365 break; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
366 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
367 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
368 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
369 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
370 t.type = TOK.Comment; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
371 t.end = p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
372 return; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
373 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
374 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
375 t.type = TOK.Div; |
10 | 376 t.end = p; |
377 return; | |
378 } | |
5 | 379 } |
380 | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
381 switch (c) |
20 | 382 { |
48 | 383 case '\'': |
384 return scanCharacterLiteral(t); | |
385 case '`': | |
386 return scanRawStringLiteral(t); | |
387 case '"': | |
388 return scanNormalStringLiteral(t); | |
389 case '\\': | |
390 char[] buffer; | |
391 do | |
392 { | |
393 c = scanEscapeSequence(); | |
394 if (c < 128) | |
395 buffer ~= c; | |
396 else | |
397 encodeUTF8(buffer, c); | |
398 } while (*p == '\\') | |
399 buffer ~= 0; | |
400 t.type = TOK.String; | |
401 t.str = buffer; | |
402 t.end = p; | |
403 return; | |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
404 case '>': /* > >= >> >>= >>> >>>= */ |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
405 c = *++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
406 switch (c) |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
407 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
408 case '=': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
409 t.type = TOK.GreaterEqual; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
410 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
411 case '>': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
412 if (p[1] == '>') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
413 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
414 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
415 if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
416 { ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
417 t.type = TOK.URShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
418 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
419 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
420 t.type = TOK.URShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
421 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
422 else if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
423 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
424 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
425 t.type = TOK.RShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
426 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
427 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
428 t.type = TOK.RShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
429 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
430 default: |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
431 t.type = TOK.Greater; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
432 goto Lcommon2; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
433 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
434 assert(0); |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
435 case '<': /* < <= <> <>= << <<= */ |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
436 c = *++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
437 switch (c) |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
438 { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
439 case '=': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
440 t.type = TOK.LessEqual; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
441 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
442 case '<': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
443 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
444 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
445 t.type = TOK.LShiftAssign; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
446 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
447 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
448 t.type = TOK.LShift; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
449 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
450 case '>': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
451 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
452 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
453 t.type = TOK.LorEorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
454 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
455 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
456 t.type = TOK.LorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
457 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
458 default: |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
459 t.type = TOK.Less; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
460 goto Lcommon2; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
461 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
462 assert(0); |
37
7f3bcb97d017
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
36
diff
changeset
|
463 case '!': /* ! !< !> !<= !>= !<> !<>= */ |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
464 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
465 switch (c) |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
466 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
467 case '<': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
468 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
469 if (c == '>') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
470 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
471 if (p[1] == '=') { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
472 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
473 t.type = TOK.Unordered; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
474 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
475 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
476 t.type = TOK.UorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
477 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
478 else if (c == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
479 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
480 t.type = TOK.UorG; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
481 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
482 else { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
483 t.type = TOK.UorGorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
484 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
485 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
486 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
487 case '>': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
488 if (p[1] == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
489 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
490 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
491 t.type = TOK.UorL; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
492 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
493 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
494 t.type = TOK.UorLorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
495 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
496 case '=': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
497 t.type = TOK.NotEqual; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
498 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
499 default: |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
500 t.type = TOK.Not; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
501 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
502 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
503 assert(0); |
63 | 504 case '.': /* . .[0-9] .. ... */ |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
505 if (p[1] == '.') |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
506 { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
507 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
508 if (p[1] == '.') { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
509 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
510 t.type = TOK.Ellipses; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
511 } |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
512 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
513 t.type = TOK.Slice; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
514 } |
63 | 515 else if (isdigit(p[1])) |
516 { | |
517 return scanReal(t); | |
518 } | |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
519 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
520 t.type = TOK.Dot; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
521 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
522 case '|': /* | || |= */ |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
523 c = *++p; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
524 if (c == '=') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
525 t.type = TOK.OrAssign; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
526 else if (c == '|') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
527 t.type = TOK.OrLogical; |
26 | 528 else { |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
529 t.type = TOK.OrBinary; |
26 | 530 goto Lcommon2; |
531 } | |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
532 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
533 case '&': /* & && &= */ |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
534 c = *++p; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
535 if (c == '=') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
536 t.type = TOK.AndAssign; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
537 else if (c == '&') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
538 t.type = TOK.AndLogical; |
26 | 539 else { |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
540 t.type = TOK.AndBinary; |
26 | 541 goto Lcommon2; |
542 } | |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
543 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
544 case '+': /* + ++ += */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
545 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
546 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
547 t.type = TOK.PlusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
548 else if (c == '+') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
549 t.type = TOK.PlusPlus; |
26 | 550 else { |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
551 t.type = TOK.Plus; |
26 | 552 goto Lcommon2; |
553 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
554 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
555 case '-': /* - -- -= */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
556 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
557 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
558 t.type = TOK.MinusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
559 else if (c == '-') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
560 t.type = TOK.MinusMinus; |
26 | 561 else { |
562 t.type = TOK.Minus; | |
563 goto Lcommon2; | |
564 } | |
565 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
566 case '=': /* = == */ |
26 | 567 if (p[1] == '=') { |
568 ++p; | |
569 t.type = TOK.Equal; | |
570 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
571 else |
26 | 572 t.type = TOK.Assign; |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
573 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
574 case '~': /* ~ ~= */ |
27 | 575 if (p[1] == '=') { |
576 ++p; | |
577 t.type = TOK.CatAssign; | |
578 } | |
579 else | |
580 t.type = TOK.Tilde; | |
581 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
582 case '*': /* * *= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
583 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
584 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
585 t.type = TOK.MulAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
586 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
587 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
588 t.type = TOK.Mul; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
589 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
590 case '^': /* ^ ^= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
591 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
592 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
593 t.type = TOK.XorAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
594 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
595 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
596 t.type = TOK.Xor; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
597 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
598 case '%': /* % %= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
599 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
600 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
601 t.type = TOK.ModAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
602 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
603 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
604 t.type = TOK.Mod; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
605 goto Lcommon; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
606 // Single character tokens: |
20 | 607 case '(': |
608 t.type = TOK.LParen; | |
609 goto Lcommon; | |
610 case ')': | |
611 t.type = TOK.RParen; | |
612 goto Lcommon; | |
613 case '[': | |
614 t.type = TOK.LBracket; | |
615 goto Lcommon; | |
616 case ']': | |
617 t.type = TOK.RBracket; | |
618 goto Lcommon; | |
619 case '{': | |
620 t.type = TOK.LBrace; | |
621 goto Lcommon; | |
622 case '}': | |
623 t.type = TOK.RBrace; | |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
624 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
625 case ':': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
626 t.type = TOK.Colon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
627 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
628 case ';': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
629 t.type = TOK.Semicolon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
630 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
631 case '?': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
632 t.type = TOK.Question; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
633 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
634 case ',': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
635 t.type = TOK.Comma; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
636 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
637 case '$': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
638 t.type = TOK.Dollar; |
20 | 639 Lcommon: |
640 ++p; | |
26 | 641 Lcommon2: |
20 | 642 t.end = p; |
643 return; | |
30 | 644 case '#': |
360
b6a3755eba94
- Renamed scanSpecialToken() to scanSpecialTokenSequence().
aziz
parents:
350
diff
changeset
|
645 return scanSpecialTokenSequence(t); |
20 | 646 default: |
647 } | |
648 | |
49 | 649 if (c & 128 && isUniAlpha(decodeUTF8())) |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
650 goto Lidentifier; |
5 | 651 c = *++p; |
4 | 652 } |
3 | 653 } |
4 | 654 |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
655 void scanNormalStringLiteral(ref Token t) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
656 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
657 assert(*p == '"'); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
658 char[] buffer; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
659 t.type = TOK.String; |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
660 uint c; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
661 while (1) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
662 { |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
663 c = *++p; |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
664 switch (c) |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
665 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
666 case '"': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
667 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
668 Lreturn: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
669 buffer ~= 0; |
49 | 670 t.str = buffer; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
671 t.pf = scanPostfix(); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
672 t.end = p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
673 return; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
674 case '\\': |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
675 c = scanEscapeSequence(); |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
676 --p; |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
677 if (c & 128) |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
678 encodeUTF8(buffer, c); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
679 else |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
680 break; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
681 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
682 case '\r': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
683 if (p[1] == '\n') |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
684 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
685 case '\n': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
686 ++loc; |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
687 c = '\n'; // Convert EndOfLine to \n. |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
688 break; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
689 case 0, _Z_: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
690 error(MID.UnterminatedString); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
691 goto Lreturn; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
692 default: |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
693 if (c & 128) |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
694 { |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
695 // char* begin = p; |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
696 c = decodeUTF8(); |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
697 if (c == LSd || c == PSd) |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
698 goto case '\n'; |
49 | 699 |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
700 // We don't copy per pointer because we might include |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
701 // invalid, skipped utf-8 sequences. See decodeUTF8(). |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
702 // ++p; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
703 // buffer ~= begin[0 .. p - begin]; |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
704 encodeUTF8(buffer, c); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
705 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
706 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
707 } |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
708 // Copy ASCII character. |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
709 buffer ~= c; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
710 } |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
711 assert(0); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
712 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
713 |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
714 void scanCharacterLiteral(ref Token t) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
715 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
716 assert(*p == '\''); |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
717 MID id = MID.UnterminatedCharacterLiteral; |
46 | 718 ++p; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
719 TOK type = TOK.CharLiteral; |
46 | 720 switch (*p) |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
721 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
722 case '\\': |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
723 switch (p[1]) |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
724 { |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
725 case 'u': |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
726 type = TOK.WCharLiteral; break; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
727 case 'U': |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
728 type = TOK.DCharLiteral; break; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
729 default: |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
730 } |
46 | 731 t.dchar_ = scanEscapeSequence(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
732 break; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
733 case '\'': |
46 | 734 ++p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
735 id = MID.EmptyCharacterLiteral; |
46 | 736 case '\n', '\r', 0, _Z_: |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
737 goto Lerr; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
738 default: |
46 | 739 uint c = *p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
740 if (c & 128) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
741 { |
49 | 742 c = decodeUTF8(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
743 if (c == LSd || c == PSd) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
744 goto Lerr; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
745 if (c <= 0xFFFF) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
746 type = TOK.WCharLiteral; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
747 else |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
748 type = TOK.DCharLiteral; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
749 } |
46 | 750 t.dchar_ = c; |
751 ++p; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
752 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
753 |
46 | 754 if (*p == '\'') |
755 ++p; | |
756 else | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
757 Lerr: |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
758 error(id); |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
759 t.type = type; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
760 t.end = p; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
761 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
762 |
33 | 763 char scanPostfix() |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
764 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
765 switch (*p) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
766 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
767 case 'c': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
768 case 'w': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
769 case 'd': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
770 return *p++; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
771 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
772 return 0; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
773 } |
112 | 774 assert(0); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
775 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
776 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
777 void scanRawStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
778 { |
33 | 779 uint delim = *p; |
780 assert(delim == '`' || delim == '"' && p[-1] == 'r'); | |
781 t.type = TOK.String; | |
782 char[] buffer; | |
783 uint c; | |
784 while (1) | |
785 { | |
786 c = *++p; | |
787 switch (c) | |
788 { | |
789 case '\r': | |
790 if (p[1] == '\n') | |
791 ++p; | |
792 case '\n': | |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
793 c = '\n'; // Convert EndOfLine ('\r','\r\n','\n',LS,PS) to '\n' |
33 | 794 ++loc; |
52
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
795 break; |
33 | 796 case '`': |
797 case '"': | |
798 if (c == delim) | |
799 { | |
800 ++p; | |
801 t.pf = scanPostfix(); | |
802 Lreturn: | |
803 t.str = buffer ~ '\0'; | |
804 t.end = p; | |
805 return; | |
806 } | |
807 break; | |
808 case 0, _Z_: | |
809 if (delim == 'r') | |
810 error(MID.UnterminatedRawString); | |
811 else | |
812 error(MID.UnterminatedBackQuoteString); | |
813 goto Lreturn; | |
814 default: | |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
815 if (c & 128) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
816 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
817 c = decodeUTF8(); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
818 if (c == LSd || c == PSd) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
819 goto case '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
820 encodeUTF8(buffer, c); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
821 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
822 } |
33 | 823 } |
824 buffer ~= c; // copy character to buffer | |
825 } | |
826 assert(0); | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
827 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
828 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
829 void scanHexStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
830 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
831 assert(p[0] == 'x' && p[1] == '"'); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
832 t.type = TOK.String; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
833 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
834 uint c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
835 ubyte[] buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
836 ubyte h; // hex number |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
837 uint n; // number of hex digits |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
838 |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
839 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
840 while (1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
841 { |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
842 c = *++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
843 switch (c) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
844 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
845 case '"': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
846 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
847 if (n & 1) |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
848 error(MID.OddNumberOfDigitsInHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
849 t.pf = scanPostfix(); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
850 Lreturn: |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
851 buffer ~= 0; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
852 t.str = cast(string) buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
853 t.end = p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
854 return; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
855 case '\r': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
856 if (p[1] == '\n') |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
857 ++p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
858 case '\n': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
859 ++loc; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
860 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
861 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
862 if (ishexad(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
863 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
864 if (c <= '9') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
865 c -= '0'; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
866 else if (c <= 'F') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
867 c -= 'A' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
868 else |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
869 c -= 'a' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
870 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
871 if (n & 1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
872 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
873 h <<= 4; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
874 h |= c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
875 buffer ~= h; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
876 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
877 else |
112 | 878 h = cast(ubyte)c; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
879 ++n; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
880 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
881 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
882 else if (isspace(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
883 continue; |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
884 |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
885 if (c & 128) |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
886 { |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
887 c = decodeUTF8(); |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
888 if (c == LSd || c == PSd) |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
889 { |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
890 ++p; ++p; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
891 ++loc; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
892 continue; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
893 } |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
894 } |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
895 else if (c == 0 || c == _Z_) |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
896 { |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
897 error(MID.UnterminatedHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
898 t.pf = 0; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
899 goto Lreturn; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
900 } |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
901 error(MID.NonHexCharInHexString, cast(dchar)c); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
902 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
903 } |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
904 assert(0); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
905 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
906 |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
907 version(D2) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
908 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
909 void scanDelimitedStringLiteral(ref Token t) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
910 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
911 assert(p[0] == 'q' && p[1] == '"'); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
912 t.type = TOK.String; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
913 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
914 char[] buffer; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
915 dchar opening_delim, // 0 if no nested delimiter or '[', '(', '<', '{' |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
916 closing_delim; // Will be ']', ')', '>', '}', any other character |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
917 // or the first, decoded character of an identifier. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
918 char[] str_delim; // Identifier delimiter |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
919 uint level = 1; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
920 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
921 ++p; ++p; // Skip q" |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
922 uint c = *p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
923 switch (c) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
924 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
925 case '(': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
926 opening_delim = c; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
927 closing_delim = ')'; // *p + 1 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
928 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
929 case '[', '<', '{': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
930 opening_delim = c; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
931 closing_delim = c + 2; // Get to closing counterpart. Feature of ASCII table. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
932 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
933 default: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
934 char* begin = p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
935 closing_delim = c; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
936 // TODO: What to do about newlines? Skip or accept as delimiter? |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
937 // TODO: Check for non-printable characters? |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
938 if (c & 128) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
939 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
940 closing_delim = decodeUTF8(); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
941 if (!isUniAlpha(c)) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
942 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
943 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
944 else if (!isidbeg(c)) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
945 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
946 // Parse identifier + newline |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
947 do |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
948 { c = *++p; } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
949 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8())) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
950 // Store identifier |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
951 str_delim = begin[0..p-begin]; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
952 // Scan newline |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
953 switch (*p) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
954 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
955 case '\r': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
956 if (p[1] == '\n') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
957 ++p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
958 case '\n': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
959 ++loc; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
960 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
961 case LS[0]: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
962 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
963 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
964 ++p; ++p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
965 ++loc; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
966 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
967 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
968 // goto default; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
969 default: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
970 // TODO: error(MID.ExpectedNewlineAfterIdentDelim); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
971 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
972 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
973 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
974 bool checkStringDelim(char* p) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
975 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
976 assert(str_delim.length != 0); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
977 if (end-p >= str_delim.length && // Check remaining length. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
978 p[0..str_delim.length] == str_delim) // Compare. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
979 return true; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
980 return false; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
981 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
982 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
983 while (1) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
984 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
985 c = *++p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
986 switch (c) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
987 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
988 case '\r': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
989 if (p[1] == '\n') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
990 ++p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
991 case '\n': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
992 c = '\n'; // Convert EndOfLine ('\r','\r\n','\n',LS,PS) to '\n' |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
993 ++loc; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
994 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
995 case 0, _Z_: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
996 // error(MID.UnterminatedDelimitedString); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
997 goto Lreturn3; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
998 default: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
999 if (c & 128) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1000 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1001 auto begin = p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1002 c = decodeUTF8(); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1003 if (c == LSd || c == PSd) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1004 goto case '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1005 if (c == closing_delim) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1006 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1007 if (str_delim.length && checkStringDelim(begin)) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1008 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1009 p = begin + str_delim.length; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1010 goto Lreturn2; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1011 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1012 assert(level == 1); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1013 --level; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1014 goto Lreturn; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1015 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1016 encodeUTF8(buffer, c); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1017 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1018 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1019 else |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1020 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1021 if (c == opening_delim) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1022 ++level; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1023 else if (c == closing_delim) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1024 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1025 if (str_delim.length && checkStringDelim(p)) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1026 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1027 p += str_delim.length; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1028 goto Lreturn2; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1029 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1030 if (--level == 0) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1031 goto Lreturn; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1032 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1033 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1034 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1035 buffer ~= c; // copy character to buffer |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1036 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1037 Lreturn: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1038 assert(*p == closing_delim); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1039 assert(level == 0); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1040 ++p; // Skip closing delimiter. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1041 Lreturn2: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1042 if (*p == '"') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1043 ++p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1044 // else |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1045 // TODO: error(MID.ExpectedDblQuoteAfterDelim, str_delim.length ? str_delim : p[-1]); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1046 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1047 t.pf = scanPostfix(); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1048 Lreturn3: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1049 t.str = buffer ~ '\0'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1050 t.end = p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1051 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1052 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1053 void scanTokenStringLiteral(ref Token t) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1054 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1055 assert(p[0] == 'q' && p[1] == '{'); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1056 t.type = TOK.String; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1057 // Copy members that might be changed by subsequent tokens. Like #line for example. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1058 auto loc_old = this.loc_old; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1059 auto loc_hline = this.loc_hline; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1060 auto filePath = this.fileName; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1061 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1062 uint loc = this.loc; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1063 uint level = 1; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1064 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1065 ++p; ++p; // Skip q{ |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1066 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1067 auto prev_t = &t; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1068 Token* token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1069 while (1) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1070 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1071 token = new Token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1072 scan(*token); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1073 // Save the tokens in a doubly linked list. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1074 // Could be useful for various tools. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1075 token.prev = prev_t; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1076 prev_t.next = token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1077 prev_t = token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1078 switch (token.type) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1079 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1080 case TOK.LBrace: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1081 ++level; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1082 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1083 case TOK.RBrace: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1084 if (--level == 0) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1085 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1086 t.tok_str = t.next; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1087 t.next = null; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1088 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1089 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1090 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1091 case TOK.EOF: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1092 // TODO: error(MID.UnterminatedTokenString); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1093 t.tok_str = t.next; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1094 t.next = token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1095 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1096 default: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1097 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1098 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1099 break; // Exit loop. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1100 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1101 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1102 assert(token.type == TOK.RBrace || token.type == TOK.EOF); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1103 assert(token.type == TOK.RBrace && t.next is null || |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1104 token.type == TOK.EOF && t.next !is null); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1105 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1106 char[] buffer; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1107 // token points to } or EOF |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1108 if (token.type == TOK.EOF) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1109 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1110 t.end = token.start; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1111 buffer = t.srcText[2..$].dup ~ '\0'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1112 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1113 else |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1114 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1115 // Assign to buffer before scanPostfix(). |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1116 t.end = p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1117 buffer = t.srcText[2..$-1].dup ~ '\0'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1118 t.pf = scanPostfix(); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1119 t.end = p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1120 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1121 // Convert EndOfLines to '\n' |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1122 if (loc != this.loc) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1123 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1124 assert(buffer[$-1] == '\0'); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1125 uint i, j; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1126 for (; i < buffer.length; ++i) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1127 switch (buffer[i]) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1128 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1129 case '\r': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1130 if (buffer[i+1] == '\n') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1131 ++i; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1132 case '\n': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1133 buffer[j++] = '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1134 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1135 case LS[0]: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1136 auto b = buffer[i..$]; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1137 if (b[1] == LS[1] && (b[2] == LS[2] || b[2] == PS[2])) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1138 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1139 ++i; ++i; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1140 goto case '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1141 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1142 // goto default; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1143 default: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1144 buffer[j++] = buffer[i]; // Copy character |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1145 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1146 buffer.length = j; // Adjust length |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1147 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1148 assert(buffer[$-1] == '\0'); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1149 t.str = buffer; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1150 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1151 // Restore possibly changed members. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1152 this.loc_old = loc_old; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1153 this.loc_hline = loc_hline; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1154 this.fileName = filePath; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1155 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1156 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1157 |
45 | 1158 dchar scanEscapeSequence() |
1159 { | |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1160 assert(*p == '\\'); |
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1161 ++p; |
45 | 1162 uint c = char2ev(*p); |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1163 if (c) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1164 { |
46 | 1165 ++p; |
45 | 1166 return c; |
46 | 1167 } |
45 | 1168 uint digits = 2; |
1169 | |
1170 switch (*p) | |
1171 { | |
1172 case 'x': | |
1173 c = 0; | |
1174 while (1) | |
1175 { | |
1176 ++p; | |
1177 if (ishexad(*p)) | |
1178 { | |
1179 c *= 16; | |
1180 if (*p <= '9') | |
48 | 1181 c += *p - '0'; |
45 | 1182 else if (*p <= 'F') |
48 | 1183 c += *p - 'A' + 10; |
45 | 1184 else |
48 | 1185 c += *p - 'a' + 10; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1186 |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1187 if (!--digits) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1188 { |
48 | 1189 ++p; |
45 | 1190 break; |
48 | 1191 } |
45 | 1192 } |
1193 else | |
1194 { | |
1195 error(MID.InsufficientHexDigits); | |
1196 break; | |
1197 } | |
1198 } | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1199 if (!isValidDchar(c)) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1200 error(MID.InvalidUnicodeCharacter); |
45 | 1201 break; |
1202 case 'u': | |
1203 digits = 4; | |
1204 goto case 'x'; | |
1205 case 'U': | |
1206 digits = 8; | |
1207 goto case 'x'; | |
1208 default: | |
48 | 1209 if (isoctal(*p)) |
45 | 1210 { |
48 | 1211 c = 0; |
1212 c += *p - '0'; | |
1213 ++p; | |
1214 if (!isoctal(*p)) | |
1215 return c; | |
1216 c *= 8; | |
1217 c += *p - '0'; | |
1218 ++p; | |
1219 if (!isoctal(*p)) | |
1220 return c; | |
1221 c *= 8; | |
1222 c += *p - '0'; | |
1223 ++p; | |
1224 } | |
1225 else if(*p == '&') | |
1226 { | |
1227 if (isalpha(*++p)) | |
45 | 1228 { |
272
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1229 auto begin = p; |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1230 while (isalnum(*++p)) |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1231 {} |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1232 |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1233 if (*p == ';') |
48 | 1234 { |
272
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1235 c = entity2Unicode(begin[0..p - begin]); |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1236 ++p; // Skip ; |
272
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1237 if (c == 0xFFFF) |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1238 error(MID.UndefinedHTMLEntity, (begin-1)[0..p-(begin-1)]); |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1239 } |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1240 else |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1241 error(MID.UnterminatedHTMLEntity); |
45 | 1242 } |
48 | 1243 else |
1244 error(MID.InvalidBeginHTMLEntity); | |
45 | 1245 } |
1246 else | |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1247 { |
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1248 dchar d = *p; |
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1249 char[] str = `\`; |
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1250 if (d & 128) |
387
ad0cbd1c8881
Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
386
diff
changeset
|
1251 encodeUTF8(str, decodeUTF8()); |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1252 else |
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1253 str ~= d; |
387
ad0cbd1c8881
Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
386
diff
changeset
|
1254 ++p; |
ad0cbd1c8881
Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
386
diff
changeset
|
1255 // TODO: check for unprintable character? |
ad0cbd1c8881
Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
386
diff
changeset
|
1256 error(MID.UndefinedEscapeSequence, str); |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1257 } |
45 | 1258 } |
1259 | |
1260 return c; | |
1261 } | |
1262 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1263 /* |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1264 IntegerLiteral:= (Dec|Hex|Bin|Oct)Suffix? |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1265 Dec:= (0|[1-9][0-9_]*) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1266 Hex:= 0[xX] HexDigits |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1267 Bin:= 0[bB][01_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1268 Oct:= 0[0-7_]+ |
68
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1269 Suffix:= (L[uU]?|[uU]L?) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1270 HexDigits:= [0-9a-zA-Z_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1271 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1272 Invalid: "0b_", "0x_", "._" |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1273 */ |
15 | 1274 void scanNumber(ref Token t) |
1275 { | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1276 ulong ulong_; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1277 bool overflow; |
58 | 1278 bool isDecimal; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1279 size_t digits; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1280 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1281 if (*p != '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1282 goto LscanInteger; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1283 ++p; // skip zero |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1284 // check for xX bB ... |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1285 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1286 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1287 case 'x','X': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1288 goto LscanHex; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1289 case 'b','B': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1290 goto LscanBin; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1291 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1292 if (p[1] == 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1293 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1294 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1295 if (p[1] == '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1296 break; |
58 | 1297 case 'i','f','F', 'e', 'E': // Imaginary and float literal suffix |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1298 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1299 default: |
323 | 1300 if (*p == '_' || isoctal(*p)) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1301 goto LscanOct; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1302 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1303 |
323 | 1304 // Number 0 |
1305 assert(p[-1] == '0'); | |
1306 assert(ulong_ == 0); | |
58 | 1307 isDecimal = true; |
1308 goto Lfinalize; | |
1309 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1310 LscanInteger: |
323 | 1311 assert(*p != 0 && isdigit(*p)); |
58 | 1312 isDecimal = true; |
323 | 1313 goto Lenter_loop_int; |
63 | 1314 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1315 { |
63 | 1316 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1317 continue; |
63 | 1318 if (!isdigit(*p)) |
1319 break; | |
323 | 1320 Lenter_loop_int: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1321 if (ulong_ < ulong.max/10 || (ulong_ == ulong.max/10 && *p <= '5')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1322 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1323 ulong_ *= 10; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1324 ulong_ += *p - '0'; |
58 | 1325 continue; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1326 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1327 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1328 overflow = true; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1329 while (isdigit(*++p)) {} |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1330 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1331 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1332 |
61 | 1333 // The number could be a float, so check overflow below. |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1334 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1335 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1336 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1337 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1338 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1339 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1340 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1341 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1342 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1343 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1344 goto LscanReal; |
58 | 1345 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1346 } |
58 | 1347 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1348 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1349 error(MID.OverflowDecimalNumber); |
58 | 1350 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1351 assert((isdigit(p[-1]) || p[-1] == '_') && !isdigit(*p) && *p != '_'); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1352 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1353 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1354 LscanHex: |
61 | 1355 assert(digits == 0); |
323 | 1356 assert(*p == 'x'); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1357 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1358 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1359 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1360 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1361 if (!ishexad(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1362 break; |
61 | 1363 ++digits; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1364 ulong_ *= 16; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1365 if (*p <= '9') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1366 ulong_ += *p - '0'; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1367 else if (*p <= 'F') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1368 ulong_ += *p - 'A' + 10; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1369 else |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1370 ulong_ += *p - 'a' + 10; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1371 } |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1372 |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1373 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1374 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1375 case '.': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1376 if (p[1] != '.') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1377 goto LscanHexReal; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1378 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1379 case 'L': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1380 if (p[1] != 'i') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1381 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1382 case 'i', 'p', 'P': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1383 goto LscanHexReal; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1384 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1385 } |
61 | 1386 if (digits == 0) |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1387 error(MID.NoDigitsInHexNumber); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1388 else if (digits > 16) |
61 | 1389 { |
1390 // Overflow: skip following digits. | |
1391 error(MID.OverflowHexNumber); | |
1392 while (ishexad(*++p)) {} | |
1393 } | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1394 goto Lfinalize; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1395 LscanHexReal: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1396 return scanHexReal(t); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1397 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1398 LscanBin: |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1399 assert(digits == 0); |
323 | 1400 assert(*p == 'b'); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1401 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1402 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1403 if (*++p == '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1404 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1405 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1406 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1407 } |
367
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1408 else if (*p == '1') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1409 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1410 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1411 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1412 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1413 } |
367
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1414 else if (*p == '_') |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1415 continue; |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1416 else |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1417 break; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1418 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1419 |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1420 if (digits == 0) |
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1421 error(MID.NoDigitsInBinNumber); |
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1422 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1423 if (digits > 64) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1424 error(MID.OverflowBinaryNumber); |
367
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1425 assert(p[-1] == '0' || p[-1] == '1' || p[-1] == '_', p[-1] ~ ""); |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1426 assert( !(*p == '0' || *p == '1' || *p == '_') ); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1427 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1428 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1429 LscanOct: |
323 | 1430 assert(*p == '_' || isoctal(*p)); |
1431 if (*p != '_') | |
1432 goto Lenter_loop_oct; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1433 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1434 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1435 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1436 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1437 if (!isoctal(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1438 break; |
323 | 1439 Lenter_loop_oct: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1440 if (ulong_ < ulong.max/2 || (ulong_ == ulong.max/2 && *p <= '1')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1441 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1442 ulong_ *= 8; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1443 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1444 ++p; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1445 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1446 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1447 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1448 overflow = true; |
63 | 1449 while (isdigit(*++p)) {} |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1450 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1451 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1452 |
63 | 1453 bool hasDecimalDigits; |
1454 if (isdigit(*p)) | |
1455 { | |
1456 hasDecimalDigits = true; | |
1457 while (isdigit(*++p)) {} | |
1458 } | |
1459 | |
1460 // The number could be a float, so check errors below. | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1461 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1462 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1463 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1464 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1465 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1466 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1467 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1468 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1469 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1470 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1471 goto LscanReal; |
63 | 1472 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1473 } |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1474 |
63 | 1475 if (hasDecimalDigits) |
1476 error(MID.OctalNumberHasDecimals); | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1477 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1478 error(MID.OverflowOctalNumber); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1479 // goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1480 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1481 Lfinalize: |
58 | 1482 enum Suffix |
1483 { | |
1484 None = 0, | |
1485 Unsigned = 1, | |
60 | 1486 Long = 2 |
58 | 1487 } |
60 | 1488 |
58 | 1489 Suffix suffix; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1490 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1491 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1492 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1493 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1494 case 'L': |
58 | 1495 if (suffix & Suffix.Long) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1496 break; |
60 | 1497 suffix |= Suffix.Long; |
1498 ++p; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1499 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1500 case 'u', 'U': |
58 | 1501 if (suffix & Suffix.Unsigned) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1502 break; |
60 | 1503 suffix |= Suffix.Unsigned; |
1504 ++p; | |
1505 continue; | |
58 | 1506 default: |
1507 break; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1508 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1509 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1510 } |
58 | 1511 |
1512 switch (suffix) | |
1513 { | |
1514 case Suffix.None: | |
1515 if (ulong_ & 0x8000000000000000) | |
1516 { | |
1517 if (isDecimal) | |
1518 error(MID.OverflowDecimalSign); | |
1519 t.type = TOK.Uint64; | |
1520 } | |
1521 else if (ulong_ & 0xFFFFFFFF00000000) | |
1522 t.type = TOK.Int64; | |
1523 else if (ulong_ & 0x80000000) | |
1524 t.type = isDecimal ? TOK.Int64 : TOK.Uint32; | |
1525 else | |
1526 t.type = TOK.Int32; | |
1527 break; | |
1528 case Suffix.Unsigned: | |
1529 if (ulong_ & 0xFFFFFFFF00000000) | |
1530 t.type = TOK.Uint64; | |
1531 else | |
1532 t.type = TOK.Uint32; | |
1533 break; | |
1534 case Suffix.Long: | |
1535 if (ulong_ & 0x8000000000000000) | |
1536 { | |
1537 if (isDecimal) | |
1538 error(MID.OverflowDecimalSign); | |
1539 t.type = TOK.Uint64; | |
1540 } | |
1541 else | |
1542 t.type = TOK.Int64; | |
1543 break; | |
1544 case Suffix.Unsigned | Suffix.Long: | |
1545 t.type = TOK.Uint64; | |
1546 break; | |
1547 default: | |
1548 assert(0); | |
1549 } | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1550 t.ulong_ = ulong_; |
15 | 1551 t.end = p; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1552 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1553 LscanReal: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1554 scanReal(t); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1555 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1556 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1557 |
63 | 1558 /* |
1559 FloatLiteral:= Float[fFL]?i? | |
1560 Float:= DecFloat | HexFloat | |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1561 DecFloat:= ([0-9][0-9_]*[.][0-9_]*DecExponent?) | [.][0-9][0-9_]*DecExponent? | [0-9][0-9_]*DecExponent |
63 | 1562 DecExponent:= [eE][+-]?[0-9][0-9_]* |
1563 HexFloat:= 0[xX](HexDigits[.]HexDigits | [.][0-9a-zA-Z]HexDigits? | HexDigits)HexExponent | |
1564 HexExponent:= [pP][+-]?[0-9][0-9_]* | |
1565 */ | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1566 void scanReal(ref Token t) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1567 { |
63 | 1568 if (*p == '.') |
1569 // This function was called by scan() or scanNumber(). | |
1570 while (isdigit(*++p) || *p == '_') {} | |
1571 else | |
1572 { | |
1573 // This function was called by scanNumber(). | |
1574 debug switch (*p) | |
1575 { | |
1576 case 'L': | |
1577 if (p[1] != 'i') | |
1578 assert(0); | |
1579 case 'i', 'f', 'F', 'e', 'E': break; | |
1580 default: assert(0); | |
1581 } | |
1582 } | |
1583 | |
1584 // Scan exponent. | |
1585 if (*p == 'e' || *p == 'E') | |
1586 { | |
1587 ++p; | |
1588 if (*p == '-' || *p == '+') | |
1589 ++p; | |
1590 if (!isdigit(*p)) | |
1591 error(MID.FloatExponentDigitExpected); | |
1592 else | |
64 | 1593 while (isdigit(*++p) || *p == '_') {} |
63 | 1594 } |
1595 | |
1596 // Copy string to buffer ignoring underscores. | |
1597 char[] buffer; | |
1598 char* end = p; | |
1599 p = t.start; | |
1600 do | |
1601 { | |
1602 if (*p == '_') | |
1603 { | |
1604 ++p; | |
1605 continue; | |
1606 } | |
1607 buffer ~= *p; | |
1608 ++p; | |
1609 } while (p != end) | |
1610 buffer ~= 0; | |
1611 finalizeFloat(t, buffer); | |
30 | 1612 } |
1613 | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1614 void scanHexReal(ref Token t) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1615 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1616 assert(*p == '.' || *p == 'i' || *p == 'p' || *p == 'P' || (*p == 'L' && p[1] == 'i')); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1617 MID mid; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1618 if (*p == '.') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1619 while (ishexad(*++p) || *p == '_') {} |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1620 if (*p != 'p' && *p != 'P') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1621 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1622 mid = MID.HexFloatExponentRequired; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1623 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1624 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1625 // Copy mantissa to a buffer ignoring underscores. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1626 char* end = p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1627 p = t.start; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1628 char[] buffer; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1629 do |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1630 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1631 if (*p == '_') |
63 | 1632 { |
1633 ++p; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1634 continue; |
63 | 1635 } |
1636 buffer ~= *p; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1637 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1638 } while (p != end) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1639 |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1640 assert(p == end && (*p == 'p' || *p == 'P')); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1641 // Scan and copy the exponent. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1642 buffer ~= 'p'; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1643 size_t bufflen = buffer.length; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1644 while (1) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1645 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1646 if (*++p == '_') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1647 continue; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1648 if (isdigit(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1649 buffer ~= *p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1650 else |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1651 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1652 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1653 // When the buffer length hasn't changed, no digits were copied. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1654 if (bufflen == buffer.length) { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1655 mid = MID.HexFloatMissingExpDigits; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1656 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1657 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1658 buffer ~= 0; // Terminate for C functions. |
63 | 1659 finalizeFloat(t, buffer); |
1660 return; | |
1661 Lerr: | |
1662 t.type = TOK.Float32; | |
1663 t.end = p; | |
1664 error(mid); | |
1665 } | |
1666 | |
1667 void finalizeFloat(ref Token t, string buffer) | |
1668 { | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1669 // Float number is well-formed. Check suffixes and do conversion. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1670 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1671 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1672 case 'f', 'F': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1673 t.type = TOK.Float32; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1674 t.float_ = strtof(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1675 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1676 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1677 case 'L': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1678 t.type = TOK.Float80; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1679 t.real_ = strtold(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1680 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1681 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1682 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1683 t.type = TOK.Float64; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1684 t.double_ = strtod(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1685 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1686 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1687 if (*p == 'i') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1688 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1689 ++p; |
63 | 1690 t.type += 3; // Switch to imaginary counterpart. |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1691 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1692 if (getErrno == ERANGE) |
63 | 1693 error(MID.OverflowFloatNumber); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1694 t.end = p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1695 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1696 |
30 | 1697 /// Scan special token: #line Integer [Filespec] EndOfLine |
360
b6a3755eba94
- Renamed scanSpecialToken() to scanSpecialTokenSequence().
aziz
parents:
350
diff
changeset
|
1698 void scanSpecialTokenSequence(ref Token t) |
30 | 1699 { |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1700 assert(*p == '#'); |
51 | 1701 |
323 | 1702 t.type = TOK.HashLine; |
1703 | |
30 | 1704 MID mid; |
1705 | |
323 | 1706 ++p; |
1707 if (p[0] != 'l' || p[1] != 'i' || p[2] != 'n' || p[3] != 'e') | |
51 | 1708 { |
370 | 1709 mid = MID.ExpectedIdentifierSTLine; |
51 | 1710 goto Lerr; |
1711 } | |
323 | 1712 p += 3; |
1713 | |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1714 // TODO: #line58"path/file" is legal. Require spaces? |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1715 // State.Space could be used for that purpose. |
323 | 1716 enum State |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1717 { /+Space,+/ Integer, Filespec, End } |
51 | 1718 |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1719 State state = State.Integer; |
323 | 1720 |
1721 Loop: | |
1722 while (1) | |
51 | 1723 { |
323 | 1724 switch (*++p) |
1725 { | |
1726 case '\r': | |
1727 if (p[1] == '\n') | |
1728 ++p; | |
1729 case '\n', 0, _Z_: | |
1730 break Loop; | |
1731 case LS[0]: | |
1732 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) | |
1733 { | |
1734 ++p; ++p; | |
1735 break Loop; | |
1736 } | |
1737 goto default; | |
1738 default: | |
1739 if (isspace(*p)) | |
1740 continue; | |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1741 if (state == State.Integer) |
323 | 1742 { |
1743 if (!isdigit(*p)) | |
1744 { | |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1745 mid = MID.ExpectedIntegerAfterSTLine; |
323 | 1746 goto Lerr; |
1747 } | |
1748 t.line_num = new Token; | |
1749 scan(*t.line_num); | |
1750 --p; | |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1751 if (t.line_num.type != TOK.Int32 && t.line_num.type != TOK.Uint32) |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1752 { |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1753 mid = MID.ExpectedIntegerAfterSTLine; |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1754 goto Lerr; |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1755 } |
323 | 1756 state = State.Filespec; |
1757 } | |
1758 else if (state == State.Filespec) | |
1759 { | |
1760 if (*p != '"') | |
1761 { | |
1762 mid = MID.ExpectedFilespec; | |
1763 goto Lerr; | |
1764 } | |
1765 t.line_filespec = new Token; | |
1766 t.line_filespec.start = p; | |
1767 t.line_filespec.type = TOK.Filespec; | |
1768 while (1) | |
1769 { | |
1770 switch (*++p) | |
1771 { | |
1772 case '"': | |
1773 break; | |
1774 case LS[0]: | |
1775 if (!(p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))) | |
1776 goto default; | |
1777 case '\r', '\n', 0, _Z_: | |
1778 mid = MID.UnterminatedFilespec; | |
1779 t.line_filespec.end = p; | |
1780 goto Lerr; | |
1781 default: | |
1782 if (*p & 128) | |
1783 decodeUTF8(); | |
1784 continue; | |
1785 } | |
1786 break; // Exit loop. | |
1787 } | |
1788 auto start = t.line_filespec.start +1; // +1 skips '"' | |
1789 t.line_filespec.str = start[0 .. p - start]; | |
1790 t.line_filespec.end = p + 1; | |
1791 state = State.End; | |
1792 } | |
1793 else/+ if (state == State.End)+/ | |
1794 { | |
1795 mid = MID.UnterminatedSpecialToken; | |
1796 goto Lerr; | |
1797 } | |
1798 } | |
51 | 1799 } |
323 | 1800 |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1801 if (state == State.Integer) |
51 | 1802 { |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1803 mid = MID.ExpectedIntegerAfterSTLine; |
30 | 1804 goto Lerr; |
1805 } | |
1806 | |
371 | 1807 this.loc_old = this.loc; |
1808 this.loc_hline = t.line_num.uint_ - 1; | |
323 | 1809 if (t.line_filespec) |
1810 this.fileName = t.line_filespec.str; | |
1811 t.end = p; | |
30 | 1812 |
1813 return; | |
1814 Lerr: | |
323 | 1815 t.end = p; |
30 | 1816 error(mid); |
15 | 1817 } |
1818 | |
371 | 1819 uint errorLoc() |
1820 { | |
1821 // ∆loc + line_num_of(#line) | |
1822 return this.loc - this.loc_old + this.loc_hline; | |
1823 } | |
1824 | |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1825 dchar decodeUTF8() |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1826 { |
49 | 1827 assert(*p & 128, "check for ASCII char before calling decodeUTF8()."); |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1828 size_t idx; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1829 dchar d; |
49 | 1830 try |
1831 { | |
1832 d = std.utf.decode(p[0 .. end-p], idx); | |
1833 p += idx -1; | |
1834 } | |
1835 catch (UtfException e) | |
1836 { | |
1837 error(MID.InvalidUTF8Sequence); | |
1838 // Skip to next valid utf-8 sequence | |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1839 while (p < end && UTF8stride[*++p] != 0xFF) {} |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1840 --p; |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1841 assert(p < end); |
49 | 1842 } |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1843 return d; |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1844 } |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1845 |
28 | 1846 void loadKeywords() |
1847 { | |
1848 foreach(k; keywords) | |
1849 idtable[k.str] = k; | |
1850 } | |
382
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1851 /+ // Not needed anymore because tokens are stored in a linked list. |
105 | 1852 struct State |
1853 { | |
1854 Lexer lexer; | |
209 | 1855 Token token; |
105 | 1856 char* scanPointer; |
209 | 1857 int loc; |
212 | 1858 string fileName; |
105 | 1859 size_t errorLen; |
212 | 1860 static State opCall(Lexer lx) |
105 | 1861 { |
1862 State s; | |
212 | 1863 s.lexer = lx; |
1864 s.token = lx.token; | |
1865 s.scanPointer = lx.p; | |
1866 s.loc = lx.loc; | |
1867 s.fileName = lx.fileName; | |
1868 s.errorLen = lx.errors.length; | |
105 | 1869 return s; |
1870 } | |
1871 void restore() | |
1872 { | |
1873 lexer.p = scanPointer; | |
209 | 1874 lexer.token = token; |
1875 lexer.loc = loc; | |
212 | 1876 lexer.fileName = fileName; |
105 | 1877 lexer.errors = lexer.errors[0..errorLen]; |
1878 } | |
1879 } | |
1880 | |
1881 State getState() | |
1882 { | |
212 | 1883 return State(this); |
105 | 1884 } |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1885 +/ |
105 | 1886 |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1887 private void scanNext(ref Token* t) |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1888 { |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1889 assert(t !is null); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1890 if (t.next) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1891 t = t.next; |
344 | 1892 else if (t != this.tail) |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1893 { |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1894 Token* new_t = new Token; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1895 scan(*new_t); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1896 new_t.prev = t; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1897 t.next = new_t; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1898 t = new_t; |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1899 } |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1900 } |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1901 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1902 void peek(ref Token* t) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1903 { |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1904 scanNext(t); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1905 } |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1906 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1907 TOK nextToken() |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1908 { |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1909 scanNext(this.token); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1910 return this.token.type; |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1911 } |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1912 |
66
8e84db78ad55
- Added support for variadic arguments in error messages.
aziz
parents:
65
diff
changeset
|
1913 void error(MID id, ...) |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1914 { |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1915 // if (reportErrors) |
371 | 1916 errors ~= new Information(InfoType.Lexer, id, this.errorLoc, arguments(_arguments, _argptr)); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1917 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1918 |
207 | 1919 unittest |
1920 { | |
382
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1921 writefln("Testing method Lexer.peek()"); |
207 | 1922 string sourceText = "unittest { }"; |
1923 auto lx = new Lexer(sourceText, null); | |
1924 | |
350 | 1925 Token* next = lx.head; |
207 | 1926 lx.peek(next); |
350 | 1927 assert(next.type == TOK.Unittest); |
207 | 1928 lx.peek(next); |
350 | 1929 assert(next.type == TOK.LBrace); |
207 | 1930 lx.peek(next); |
350 | 1931 assert(next.type == TOK.RBrace); |
207 | 1932 lx.peek(next); |
350 | 1933 assert(next.type == TOK.EOF); |
207 | 1934 } |
1935 | |
306 | 1936 Token* getTokens() |
4 | 1937 { |
306 | 1938 while (nextToken() != TOK.EOF) |
1939 {} | |
1940 return head; | |
3 | 1941 } |
48 | 1942 |
365 | 1943 static bool isNonReservedIdentifier(char[] ident) |
1944 { | |
382
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1945 if (ident.length == 0) |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1946 return false; |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1947 |
365 | 1948 static Identifier[string] reserved_ids_table; |
1949 if (reserved_ids_table is null) | |
1950 foreach(k; keywords) | |
1951 reserved_ids_table[k.str] = k; | |
1952 | |
382
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1953 size_t idx = 1; // Index to the 2nd character in ident. |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1954 dchar isFirstCharUniAlpha() |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1955 { |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1956 idx = 0; |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1957 // NB: decode() could throw an Exception which would be |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1958 // caught by the next try-catch-block. |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1959 return isUniAlpha(std.utf.decode(ident, idx)); |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1960 } |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1961 |
365 | 1962 try |
382
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1963 { |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1964 if (isidbeg(ident[0]) || |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1965 ident[0] & 128 && isFirstCharUniAlpha()) |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1966 { |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1967 foreach (dchar c; ident[idx..$]) |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1968 if (!isident(c) && !isUniAlpha(c)) |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1969 return false; |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1970 } |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1971 } |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1972 catch (Exception) |
365 | 1973 return false; |
1974 | |
1975 return !(ident in reserved_ids_table); | |
1976 } | |
1977 | |
48 | 1978 private void encodeUTF8(inout char[] str, dchar d) |
1979 { | |
1980 char[6] b; | |
1981 assert(d > 0x7F, "check for ASCII char before calling encodeUTF8()."); | |
1982 if (d < 0x800) | |
1983 { | |
1984 b[0] = 0xC0 | (d >> 6); | |
1985 b[1] = 0x80 | (d & 0x3F); | |
1986 str ~= b[0..2]; | |
1987 } | |
1988 else if (d < 0x10000) | |
1989 { | |
1990 b[0] = 0xE0 | (d >> 12); | |
1991 b[1] = 0x80 | ((d >> 6) & 0x3F); | |
1992 b[2] = 0x80 | (d & 0x3F); | |
1993 str ~= b[0..3]; | |
1994 } | |
1995 else if (d < 0x200000) | |
1996 { | |
1997 b[0] = 0xF0 | (d >> 18); | |
1998 b[1] = 0x80 | ((d >> 12) & 0x3F); | |
1999 b[2] = 0x80 | ((d >> 6) & 0x3F); | |
2000 b[3] = 0x80 | (d & 0x3F); | |
2001 str ~= b[0..4]; | |
2002 } | |
2003 else if (d < 0x4000000) | |
2004 { | |
2005 b[0] = 0xF8 | (d >> 24); | |
2006 b[1] = 0x80 | ((d >> 18) & 0x3F); | |
2007 b[2] = 0x80 | ((d >> 12) & 0x3F); | |
2008 b[3] = 0x80 | ((d >> 6) & 0x3F); | |
2009 b[4] = 0x80 | (d & 0x3F); | |
2010 str ~= b[0..5]; | |
2011 } | |
2012 else if (d < 0x80000000) | |
2013 { | |
2014 b[0] = 0xFC | (d >> 30); | |
2015 b[1] = 0x80 | ((d >> 24) & 0x3F); | |
2016 b[2] = 0x80 | ((d >> 18) & 0x3F); | |
2017 b[3] = 0x80 | ((d >> 12) & 0x3F); | |
2018 b[4] = 0x80 | ((d >> 6) & 0x3F); | |
2019 b[5] = 0x80 | (d & 0x3F); | |
2020 str ~= b[0..6]; | |
2021 } | |
2022 else | |
2023 error(MID.InvalidUnicodeCharacter); | |
2024 } | |
5 | 2025 } |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2026 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2027 unittest |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2028 { |
350 | 2029 writefln("Testing Lexer."); |
40 | 2030 string[] toks = [ |
2031 ">", ">=", ">>", ">>=", ">>>", ">>>=", "<", "<=", "<>", | |
2032 "<>=", "<<", "<<=", "!", "!<", "!>", "!<=", "!>=", "!<>", | |
2033 "!<>=", ".", "..", "...", "&", "&&", "&=", "+", "++", | |
2034 "+=", "-", "--", "-=", "=", "==", "~", "~=", "*", | |
2035 "*=", "/", "/=", "^", "^=", "%", "%=", "(", ")", | |
2036 "[", "]", "{", "}", ":", ";", "?", ",", "$" | |
2037 ]; | |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2038 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2039 char[] src; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2040 |
40 | 2041 foreach (op; toks) |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2042 src ~= op ~ " "; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2043 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2044 auto lx = new Lexer(src, ""); |
350 | 2045 auto token = lx.getTokens(); |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2046 |
350 | 2047 uint i; |
2048 assert(token == lx.head); | |
2049 token = token.next; | |
2050 do | |
2051 { | |
2052 assert(i < toks.length); | |
2053 assert(token.srcText == toks[i], std.string.format("Scanned '%s' but expected '%s'", token.srcText, toks[i])); | |
2054 ++i; | |
2055 token = token.next; | |
2056 } while (token.type != TOK.EOF) | |
41 | 2057 } |
55 | 2058 |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2059 unittest |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2060 { |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2061 // Numbers unittest |
68
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2062 // 0L 0ULi 0_L 0_UL 0x0U 0x0p2 0_Fi 0_e2 0_F 0_i |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2063 // 0u 0U 0uL 0UL 0L 0LU 0Lu |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2064 // 0Li 0f 0F 0fi 0Fi 0i |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2065 // 0b_1_LU 0b1000u |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2066 // 0x232Lu |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2067 } |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2068 |
55 | 2069 /// ASCII character properties table. |
2070 static const int ptable[256] = [ | |
2071 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0, | |
2072 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2073 32, 0, 0x2200, 0, 0, 0, 0, 0x2700, 0, 0, 0, 0, 0, 0, 0, 0, | |
2074 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0x3f00, | |
2075 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8, | |
2076 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0x5c00, 0, 0,16, | |
2077 0, 0x70c, 0x80c,12,12,12, 0xc0c, 8, 8, 8, 8, 8, 8, 8, 0xa08, 8, | |
2078 8, 8, 0xd08, 8, 0x908, 8, 0xb08, 8, 8, 8, 8, 0, 0, 0, 0, 0, | |
2079 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2080 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2081 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2082 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2083 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2084 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2085 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2086 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 | |
2087 ]; | |
2088 | |
2089 enum CProperty | |
2090 { | |
2091 Octal = 1, | |
2092 Digit = 1<<1, | |
2093 Hex = 1<<2, | |
2094 Alpha = 1<<3, | |
2095 Underscore = 1<<4, | |
2096 Whitespace = 1<<5 | |
2097 } | |
2098 | |
2099 const uint EVMask = 0xFF00; // Bit mask for escape value | |
2100 | |
2101 private alias CProperty CP; | |
2102 int isoctal(char c) { return ptable[c] & CP.Octal; } | |
2103 int isdigit(char c) { return ptable[c] & CP.Digit; } | |
2104 int ishexad(char c) { return ptable[c] & CP.Hex; } | |
2105 int isalpha(char c) { return ptable[c] & CP.Alpha; } | |
2106 int isalnum(char c) { return ptable[c] & (CP.Alpha | CP.Digit); } | |
2107 int isidbeg(char c) { return ptable[c] & (CP.Alpha | CP.Underscore); } | |
2108 int isident(char c) { return ptable[c] & (CP.Alpha | CP.Underscore | CP.Digit); } | |
2109 int isspace(char c) { return ptable[c] & CP.Whitespace; } | |
2110 int char2ev(char c) { return ptable[c] >> 8; /*(ptable[c] & EVMask) >> 8;*/ } | |
2111 | |
2112 version(gen_ptable) | |
2113 static this() | |
2114 { | |
2115 alias ptable p; | |
2116 // Initialize character properties table. | |
2117 for (int i; i < p.length; ++i) | |
2118 { | |
2119 p[i] = 0; | |
2120 if ('0' <= i && i <= '7') | |
2121 p[i] |= CP.Octal; | |
2122 if ('0' <= i && i <= '9') | |
2123 p[i] |= CP.Digit; | |
2124 if (isdigit(i) || 'a' <= i && i <= 'f' || 'A' <= i && i <= 'F') | |
2125 p[i] |= CP.Hex; | |
2126 if ('a' <= i && i <= 'z' || 'A' <= i && i <= 'Z') | |
2127 p[i] |= CP.Alpha; | |
2128 if (i == '_') | |
2129 p[i] |= CP.Underscore; | |
2130 if (i == ' ' || i == '\t' || i == '\v' || i == '\f') | |
2131 p[i] |= CP.Whitespace; | |
2132 } | |
2133 // Store escape sequence values in second byte. | |
2134 assert(CProperty.max <= ubyte.max, "character property flags and escape value byte overlap."); | |
2135 p['\''] |= 39 << 8; | |
2136 p['"'] |= 34 << 8; | |
2137 p['?'] |= 63 << 8; | |
2138 p['\\'] |= 92 << 8; | |
2139 p['a'] |= 7 << 8; | |
2140 p['b'] |= 8 << 8; | |
2141 p['f'] |= 12 << 8; | |
2142 p['n'] |= 10 << 8; | |
2143 p['r'] |= 13 << 8; | |
2144 p['t'] |= 9 << 8; | |
2145 p['v'] |= 11 << 8; | |
2146 // Print a formatted array literal. | |
2147 char[] array = "[\n"; | |
2148 for (int i; i < p.length; ++i) | |
2149 { | |
2150 int c = p[i]; | |
2151 array ~= std.string.format(c>255?" 0x%x,":"%2d,", c, ((i+1) % 16) ? "":"\n"); | |
2152 } | |
2153 array[$-2..$] = "\n]"; | |
2154 writefln(array); | |
2155 } |