Mercurial > projects > dil
annotate trunk/src/dil/Lexer.d @ 392:bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
In scanDelimitedStringLiteral:
- leading newlines are skipped now.
- passed c instead of closing_delim to isUniAlpha().
- the matching string delimiter must follow a newline.
- corrected if-statements and an assert().
Added inTokenString member:
- in scanTokenStringLiteral() this member is increment at the start
and decremented at the end.
- in scanSpecialTokenSequence() this member is checked and #line is not
evaluated if it is not zero.
Added missing break to case 'L' in scanNumber().
In scanReal():
- changed debug switch to a delegate inside an assert().
- removing underscores from buffer more efficiently.
Made some other trivial changes and fixes to other modules.
author | Aziz K?ksal <aziz.koeksal@gmail.com> |
---|---|
date | Mon, 17 Sep 2007 20:35:08 +0200 |
parents | 33b566df6af4 |
children | fce1e6133dac |
rev | line source |
---|---|
0 | 1 /++ |
2 Author: Aziz Köksal | |
249 | 3 License: GPL3 |
0 | 4 +/ |
326 | 5 module dil.Lexer; |
327 | 6 import dil.Token; |
7 import dil.Information; | |
8 import dil.Keywords; | |
9 import dil.Identifier; | |
10 import dil.Messages; | |
11 import dil.HtmlEntities; | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
12 import dil.Settings; |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
13 import tango.stdc.stdlib : strtof, strtod, strtold; |
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
14 import tango.stdc.errno : errno, ERANGE; |
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
15 import tango.stdc.time : time_t, time, ctime; |
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
16 import tango.stdc.string : strlen; |
4 | 17 import std.utf; |
18 import std.uni; | |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
19 import common; |
0 | 20 |
11 | 21 const char[3] LS = \u2028; |
22 const char[3] PS = \u2029; | |
23 | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
24 const dchar LSd = 0x2028; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
25 const dchar PSd = 0x2029; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
26 |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
27 const uint _Z_ = 26; /// Control+Z |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
28 |
0 | 29 class Lexer |
30 { | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
31 Token* head; /// The head of the doubly linked token list. |
344 | 32 Token* tail; /// The tail of the linked list. Set in scan(). |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
33 Token* token; /// Points to the current token in the token list. |
30 | 34 string text; |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
35 char* p; /// Points to the current character in the source text. |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
36 char* end; /// Points one character past the end of the source text. |
4 | 37 |
371 | 38 uint loc = 1; /// Actual line of code. |
39 | |
40 uint loc_old; /// Store actual line number when #line token is parsed. | |
41 uint loc_hline; /// Line number set by #line. | |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
42 private uint inTokenString; // > 0 if inside q{ } |
17 | 43 |
30 | 44 char[] fileName; |
45 | |
69
24db7c5522d5
- Added module Information for compiler messages like warnings, info and errors to the user.
aziz
parents:
68
diff
changeset
|
46 Information[] errors; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
47 |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
48 // bool reportErrors; |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
49 |
28 | 50 Identifier[string] idtable; |
51 | |
30 | 52 this(string text, string fileName) |
4 | 53 { |
30 | 54 this.fileName = fileName; |
55 | |
4 | 56 this.text = text; |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
57 if (text[$-1] != 0) |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
58 { |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
59 this.text.length = this.text.length + 1; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
60 this.text[$-1] = 0; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
61 } |
4 | 62 |
63 this.p = this.text.ptr; | |
64 this.end = this.p + this.text.length; | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
65 // this.reportErrors = true; |
28 | 66 loadKeywords(); |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
67 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
68 this.head = new Token; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
69 this.head.type = TOK.HEAD; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
70 this.token = this.head; |
315 | 71 scanShebang(); |
72 } | |
73 | |
344 | 74 ~this() |
75 { | |
76 auto token = head.next; | |
77 do | |
78 { | |
79 assert(token.type == TOK.EOF ? token == tail && token.next is null : 1); | |
80 delete token.prev; | |
81 token = token.next; | |
82 } while (token !is null) | |
83 delete tail; | |
84 } | |
85 | |
315 | 86 void scanShebang() |
87 { | |
88 if (*p == '#' && p[1] == '!') | |
89 { | |
90 Token* t = new Token; | |
91 t.start = p; | |
92 t.type = TOK.Shebang; | |
93 ++p; | |
94 while (1) | |
95 { | |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
96 t.end = p; |
315 | 97 switch (*++p) |
98 { | |
99 case '\r': | |
100 if (p[1] == '\n') | |
101 ++p; | |
102 case '\n': | |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
103 ++p; |
315 | 104 ++loc; |
105 break; | |
106 case 0, _Z_: | |
107 break; | |
108 default: | |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
109 if (*p & 128) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
110 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
111 auto c = decodeUTF8(); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
112 if (c == LSd || c == PSd) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
113 goto case '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
114 } |
315 | 115 continue; |
116 } | |
117 break; // Exit loop. | |
118 } | |
119 this.head.next = t; | |
120 t.prev = this.head; | |
121 } | |
4 | 122 } |
123 | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
124 void finalizeSpecialToken(ref Token t) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
125 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
126 assert(t.srcText[0..2] == "__"); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
127 switch (t.type) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
128 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
129 case TOK.FILE: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
130 t.str = this.fileName; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
131 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
132 case TOK.LINE: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
133 t.uint_ = this.loc; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
134 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
135 case TOK.DATE, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
136 TOK.TIME, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
137 TOK.TIMESTAMP: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
138 time_t time_val; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
139 time(&time_val); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
140 char* str = ctime(&time_val); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
141 char[] time_str = str[0 .. strlen(str)]; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
142 switch (t.type) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
143 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
144 case TOK.DATE: |
346
ce9e2e77743f
- Fix: include one space when slicing time_str for __DATE__.
aziz
parents:
344
diff
changeset
|
145 time_str = time_str[4..11] ~ time_str[20..24] ~ \0; break; |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
146 case TOK.TIME: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
147 time_str = time_str[11..19] ~ \0; break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
148 case TOK.TIMESTAMP: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
149 time_str = time_str[0..24] ~ \0; break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
150 default: assert(0); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
151 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
152 t.str = time_str; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
153 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
154 case TOK.VENDOR: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
155 t.str = VENDOR; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
156 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
157 case TOK.VERSION: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
158 t.uint_ = VERSION_MAJOR*1000 + VERSION_MINOR; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
159 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
160 default: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
161 assert(0); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
162 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
163 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
164 |
4 | 165 public void scan(out Token t) |
207 | 166 in |
3 | 167 { |
207 | 168 assert(text.ptr <= p && p < end); |
169 } | |
170 out | |
171 { | |
172 assert(text.ptr <= t.start && t.start < end); | |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
173 assert(text.ptr < t.end && t.end <= end, Token.toString(t.type)); |
207 | 174 } |
175 body | |
176 { | |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
177 // Scan whitespace. |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
178 auto pws = p; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
179 while (1) |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
180 { |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
181 switch (*p) |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
182 { |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
183 case '\r': |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
184 if (p[1] == '\n') |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
185 ++p; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
186 case '\n': |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
187 ++p; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
188 ++loc; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
189 continue; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
190 case LS[0]: |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
191 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
192 { |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
193 p += 3; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
194 ++loc; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
195 continue; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
196 } |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
197 // goto default; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
198 default: |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
199 if (!isspace(*p)) |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
200 break; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
201 ++p; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
202 continue; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
203 } |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
204 break; // Exit loop. |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
205 } |
4 | 206 |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
207 if (p != pws) |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
208 t.ws = pws; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
209 |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
210 // Scan token. |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
211 uint c = *p; |
4 | 212 { |
5 | 213 t.start = p; |
16 | 214 |
212 | 215 if (c == 0 || c == _Z_) |
5 | 216 { |
212 | 217 assert(*p == 0 || *p == _Z_); |
5 | 218 t.type = TOK.EOF; |
17 | 219 t.end = p; |
344 | 220 tail = &t; |
212 | 221 assert(t.start == t.end); |
5 | 222 return; |
223 } | |
4 | 224 |
13 | 225 if (isidbeg(c)) |
5 | 226 { |
33 | 227 if (c == 'r' && p[1] == '"' && ++p) |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
228 return scanRawStringLiteral(t); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
229 if (c == 'x' && p[1] == '"') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
230 return scanHexStringLiteral(t); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
231 version(D2) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
232 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
233 if (c == 'q' && p[1] == '"') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
234 return scanDelimitedStringLiteral(t); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
235 if (c == 'q' && p[1] == '{') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
236 return scanTokenStringLiteral(t); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
237 } |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
238 Lidentifier: |
5 | 239 do |
240 { c = *++p; } | |
49 | 241 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8())) |
28 | 242 |
5 | 243 t.end = p; |
28 | 244 |
65 | 245 string str = t.srcText; |
28 | 246 Identifier* id = str in idtable; |
247 | |
248 if (!id) | |
249 { | |
327 | 250 idtable[str] = Identifier(TOK.Identifier, str); |
28 | 251 id = str in idtable; |
252 } | |
253 assert(id); | |
254 t.type = id.type; | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
255 if (t.isSpecialToken) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
256 finalizeSpecialToken(t); |
5 | 257 return; |
258 } | |
259 | |
15 | 260 if (isdigit(c)) |
261 return scanNumber(t); | |
262 | |
8 | 263 if (c == '/') |
5 | 264 { |
8 | 265 c = *++p; |
14 | 266 switch(c) |
5 | 267 { |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
268 case '=': |
14 | 269 ++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
270 t.type = TOK.DivAssign; |
14 | 271 t.end = p; |
272 return; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
273 case '+': |
8 | 274 uint level = 1; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
275 while (1) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
276 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
277 c = *++p; |
42 | 278 LswitchNC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
279 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
280 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
281 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
282 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
283 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
284 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
285 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
286 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
287 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
288 error(MID.UnterminatedNestedComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
289 goto LreturnNC; |
42 | 290 default: |
291 } | |
292 | |
293 c <<= 8; | |
294 c |= *++p; | |
295 switch (c) | |
296 { | |
297 case 0x2F2B: // /+ | |
298 ++level; | |
299 continue; | |
300 case 0x2B2F: // +/ | |
301 if (--level == 0) | |
302 { | |
303 ++p; | |
304 LreturnNC: | |
305 t.type = TOK.Comment; | |
306 t.end = p; | |
307 return; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
308 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
309 continue; |
42 | 310 case 0xE280: // LS[0..1] || PS[0..1] |
311 if (p[1] == LS[2] || p[1] == PS[2]) | |
312 { | |
313 ++loc; | |
314 ++p; | |
315 } | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
316 continue; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
317 default: |
42 | 318 c &= char.max; |
319 goto LswitchNC; | |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
320 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
321 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
322 case '*': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
323 while (1) |
7 | 324 { |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
325 c = *++p; |
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
326 LswitchBC: // only jumped to from default case of next switch(c) |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
327 switch (c) |
8 | 328 { |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
329 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
330 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
331 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
332 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
333 ++loc; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
334 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
335 case 0, _Z_: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
336 error(MID.UnterminatedBlockComment); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
337 goto LreturnBC; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
338 default: |
8 | 339 } |
41 | 340 |
341 c <<= 8; | |
342 c |= *++p; | |
343 switch (c) | |
344 { | |
345 case 0x2A2F: // */ | |
346 ++p; | |
347 LreturnBC: | |
348 t.type = TOK.Comment; | |
349 t.end = p; | |
350 return; | |
351 case 0xE280: // LS[0..1] || PS[0..1] | |
352 if (p[1] == LS[2] || p[1] == PS[2]) | |
353 { | |
354 ++loc; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
355 ++p; |
41 | 356 } |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
357 continue; |
41 | 358 default: |
359 c &= char.max; | |
43
1845c23dd056
- Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents:
42
diff
changeset
|
360 goto LswitchBC; |
41 | 361 } |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
362 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
363 assert(0); |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
364 case '/': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
365 while (1) |
7 | 366 { |
8 | 367 c = *++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
368 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
369 { |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
370 case '\r': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
371 if (p[1] == '\n') |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
372 ++p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
373 case '\n': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
374 case 0, _Z_: |
11 | 375 break; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
376 case LS[0]: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
377 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
378 break; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
379 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
380 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
381 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
382 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
383 t.type = TOK.Comment; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
384 t.end = p; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
385 return; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
386 } |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
387 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
388 t.type = TOK.Div; |
10 | 389 t.end = p; |
390 return; | |
391 } | |
5 | 392 } |
393 | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
394 switch (c) |
20 | 395 { |
48 | 396 case '\'': |
397 return scanCharacterLiteral(t); | |
398 case '`': | |
399 return scanRawStringLiteral(t); | |
400 case '"': | |
401 return scanNormalStringLiteral(t); | |
402 case '\\': | |
403 char[] buffer; | |
404 do | |
405 { | |
406 c = scanEscapeSequence(); | |
407 if (c < 128) | |
408 buffer ~= c; | |
409 else | |
410 encodeUTF8(buffer, c); | |
411 } while (*p == '\\') | |
412 buffer ~= 0; | |
413 t.type = TOK.String; | |
414 t.str = buffer; | |
415 t.end = p; | |
416 return; | |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
417 case '>': /* > >= >> >>= >>> >>>= */ |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
418 c = *++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
419 switch (c) |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
420 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
421 case '=': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
422 t.type = TOK.GreaterEqual; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
423 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
424 case '>': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
425 if (p[1] == '>') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
426 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
427 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
428 if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
429 { ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
430 t.type = TOK.URShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
431 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
432 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
433 t.type = TOK.URShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
434 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
435 else if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
436 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
437 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
438 t.type = TOK.RShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
439 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
440 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
441 t.type = TOK.RShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
442 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
443 default: |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
444 t.type = TOK.Greater; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
445 goto Lcommon2; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
446 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
447 assert(0); |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
448 case '<': /* < <= <> <>= << <<= */ |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
449 c = *++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
450 switch (c) |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
451 { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
452 case '=': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
453 t.type = TOK.LessEqual; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
454 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
455 case '<': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
456 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
457 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
458 t.type = TOK.LShiftAssign; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
459 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
460 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
461 t.type = TOK.LShift; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
462 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
463 case '>': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
464 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
465 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
466 t.type = TOK.LorEorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
467 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
468 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
469 t.type = TOK.LorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
470 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
471 default: |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
472 t.type = TOK.Less; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
473 goto Lcommon2; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
474 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
475 assert(0); |
37
7f3bcb97d017
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
36
diff
changeset
|
476 case '!': /* ! !< !> !<= !>= !<> !<>= */ |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
477 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
478 switch (c) |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
479 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
480 case '<': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
481 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
482 if (c == '>') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
483 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
484 if (p[1] == '=') { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
485 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
486 t.type = TOK.Unordered; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
487 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
488 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
489 t.type = TOK.UorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
490 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
491 else if (c == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
492 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
493 t.type = TOK.UorG; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
494 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
495 else { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
496 t.type = TOK.UorGorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
497 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
498 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
499 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
500 case '>': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
501 if (p[1] == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
502 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
503 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
504 t.type = TOK.UorL; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
505 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
506 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
507 t.type = TOK.UorLorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
508 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
509 case '=': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
510 t.type = TOK.NotEqual; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
511 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
512 default: |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
513 t.type = TOK.Not; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
514 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
515 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
516 assert(0); |
63 | 517 case '.': /* . .[0-9] .. ... */ |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
518 if (p[1] == '.') |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
519 { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
520 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
521 if (p[1] == '.') { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
522 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
523 t.type = TOK.Ellipses; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
524 } |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
525 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
526 t.type = TOK.Slice; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
527 } |
63 | 528 else if (isdigit(p[1])) |
529 { | |
530 return scanReal(t); | |
531 } | |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
532 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
533 t.type = TOK.Dot; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
534 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
535 case '|': /* | || |= */ |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
536 c = *++p; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
537 if (c == '=') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
538 t.type = TOK.OrAssign; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
539 else if (c == '|') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
540 t.type = TOK.OrLogical; |
26 | 541 else { |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
542 t.type = TOK.OrBinary; |
26 | 543 goto Lcommon2; |
544 } | |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
545 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
546 case '&': /* & && &= */ |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
547 c = *++p; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
548 if (c == '=') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
549 t.type = TOK.AndAssign; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
550 else if (c == '&') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
551 t.type = TOK.AndLogical; |
26 | 552 else { |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
553 t.type = TOK.AndBinary; |
26 | 554 goto Lcommon2; |
555 } | |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
556 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
557 case '+': /* + ++ += */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
558 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
559 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
560 t.type = TOK.PlusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
561 else if (c == '+') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
562 t.type = TOK.PlusPlus; |
26 | 563 else { |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
564 t.type = TOK.Plus; |
26 | 565 goto Lcommon2; |
566 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
567 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
568 case '-': /* - -- -= */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
569 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
570 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
571 t.type = TOK.MinusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
572 else if (c == '-') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
573 t.type = TOK.MinusMinus; |
26 | 574 else { |
575 t.type = TOK.Minus; | |
576 goto Lcommon2; | |
577 } | |
578 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
579 case '=': /* = == */ |
26 | 580 if (p[1] == '=') { |
581 ++p; | |
582 t.type = TOK.Equal; | |
583 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
584 else |
26 | 585 t.type = TOK.Assign; |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
586 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
587 case '~': /* ~ ~= */ |
27 | 588 if (p[1] == '=') { |
589 ++p; | |
590 t.type = TOK.CatAssign; | |
591 } | |
592 else | |
593 t.type = TOK.Tilde; | |
594 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
595 case '*': /* * *= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
596 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
597 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
598 t.type = TOK.MulAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
599 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
600 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
601 t.type = TOK.Mul; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
602 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
603 case '^': /* ^ ^= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
604 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
605 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
606 t.type = TOK.XorAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
607 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
608 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
609 t.type = TOK.Xor; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
610 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
611 case '%': /* % %= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
612 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
613 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
614 t.type = TOK.ModAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
615 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
616 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
617 t.type = TOK.Mod; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
618 goto Lcommon; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
619 // Single character tokens: |
20 | 620 case '(': |
621 t.type = TOK.LParen; | |
622 goto Lcommon; | |
623 case ')': | |
624 t.type = TOK.RParen; | |
625 goto Lcommon; | |
626 case '[': | |
627 t.type = TOK.LBracket; | |
628 goto Lcommon; | |
629 case ']': | |
630 t.type = TOK.RBracket; | |
631 goto Lcommon; | |
632 case '{': | |
633 t.type = TOK.LBrace; | |
634 goto Lcommon; | |
635 case '}': | |
636 t.type = TOK.RBrace; | |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
637 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
638 case ':': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
639 t.type = TOK.Colon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
640 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
641 case ';': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
642 t.type = TOK.Semicolon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
643 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
644 case '?': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
645 t.type = TOK.Question; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
646 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
647 case ',': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
648 t.type = TOK.Comma; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
649 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
650 case '$': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
651 t.type = TOK.Dollar; |
20 | 652 Lcommon: |
653 ++p; | |
26 | 654 Lcommon2: |
20 | 655 t.end = p; |
656 return; | |
30 | 657 case '#': |
360
b6a3755eba94
- Renamed scanSpecialToken() to scanSpecialTokenSequence().
aziz
parents:
350
diff
changeset
|
658 return scanSpecialTokenSequence(t); |
20 | 659 default: |
660 } | |
661 | |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
662 if (c & 128) |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
663 { |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
664 c = decodeUTF8(); |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
665 if (isUniAlpha(c)) |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
666 goto Lidentifier; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
667 } |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
668 |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
669 error(MID.IllegalCharacter, cast(dchar)c); |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
670 |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
671 ++p; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
672 t.type = TOK.Illegal; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
673 t.dchar_ = c; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
674 t.end = p; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
675 return; |
4 | 676 } |
3 | 677 } |
4 | 678 |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
679 void scanNormalStringLiteral(ref Token t) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
680 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
681 assert(*p == '"'); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
682 char[] buffer; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
683 t.type = TOK.String; |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
684 uint c; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
685 while (1) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
686 { |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
687 c = *++p; |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
688 switch (c) |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
689 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
690 case '"': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
691 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
692 Lreturn: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
693 buffer ~= 0; |
49 | 694 t.str = buffer; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
695 t.pf = scanPostfix(); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
696 t.end = p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
697 return; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
698 case '\\': |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
699 c = scanEscapeSequence(); |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
700 --p; |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
701 if (c & 128) |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
702 encodeUTF8(buffer, c); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
703 else |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
704 break; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
705 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
706 case '\r': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
707 if (p[1] == '\n') |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
708 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
709 case '\n': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
710 ++loc; |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
711 c = '\n'; // Convert EndOfLine to \n. |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
712 break; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
713 case 0, _Z_: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
714 error(MID.UnterminatedString); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
715 goto Lreturn; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
716 default: |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
717 if (c & 128) |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
718 { |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
719 // char* begin = p; |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
720 c = decodeUTF8(); |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
721 if (c == LSd || c == PSd) |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
722 goto case '\n'; |
49 | 723 |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
724 // We don't copy per pointer because we might include |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
725 // invalid, skipped utf-8 sequences. See decodeUTF8(). |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
726 // ++p; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
727 // buffer ~= begin[0 .. p - begin]; |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
728 encodeUTF8(buffer, c); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
729 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
730 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
731 } |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
732 // Copy ASCII character. |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
733 buffer ~= c; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
734 } |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
735 assert(0); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
736 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
737 |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
738 void scanCharacterLiteral(ref Token t) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
739 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
740 assert(*p == '\''); |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
741 MID id = MID.UnterminatedCharacterLiteral; |
46 | 742 ++p; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
743 TOK type = TOK.CharLiteral; |
46 | 744 switch (*p) |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
745 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
746 case '\\': |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
747 switch (p[1]) |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
748 { |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
749 case 'u': |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
750 type = TOK.WCharLiteral; break; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
751 case 'U': |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
752 type = TOK.DCharLiteral; break; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
753 default: |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
754 } |
46 | 755 t.dchar_ = scanEscapeSequence(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
756 break; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
757 case '\'': |
46 | 758 ++p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
759 id = MID.EmptyCharacterLiteral; |
46 | 760 case '\n', '\r', 0, _Z_: |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
761 goto Lerr; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
762 default: |
46 | 763 uint c = *p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
764 if (c & 128) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
765 { |
49 | 766 c = decodeUTF8(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
767 if (c == LSd || c == PSd) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
768 goto Lerr; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
769 if (c <= 0xFFFF) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
770 type = TOK.WCharLiteral; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
771 else |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
772 type = TOK.DCharLiteral; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
773 } |
46 | 774 t.dchar_ = c; |
775 ++p; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
776 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
777 |
46 | 778 if (*p == '\'') |
779 ++p; | |
780 else | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
781 Lerr: |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
782 error(id); |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
783 t.type = type; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
784 t.end = p; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
785 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
786 |
33 | 787 char scanPostfix() |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
788 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
789 switch (*p) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
790 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
791 case 'c': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
792 case 'w': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
793 case 'd': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
794 return *p++; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
795 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
796 return 0; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
797 } |
112 | 798 assert(0); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
799 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
800 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
801 void scanRawStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
802 { |
33 | 803 uint delim = *p; |
804 assert(delim == '`' || delim == '"' && p[-1] == 'r'); | |
805 t.type = TOK.String; | |
806 char[] buffer; | |
807 uint c; | |
808 while (1) | |
809 { | |
810 c = *++p; | |
811 switch (c) | |
812 { | |
813 case '\r': | |
814 if (p[1] == '\n') | |
815 ++p; | |
816 case '\n': | |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
817 c = '\n'; // Convert EndOfLine ('\r','\r\n','\n',LS,PS) to '\n' |
33 | 818 ++loc; |
52
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
819 break; |
33 | 820 case '`': |
821 case '"': | |
822 if (c == delim) | |
823 { | |
824 ++p; | |
825 t.pf = scanPostfix(); | |
826 Lreturn: | |
827 t.str = buffer ~ '\0'; | |
828 t.end = p; | |
829 return; | |
830 } | |
831 break; | |
832 case 0, _Z_: | |
833 if (delim == 'r') | |
834 error(MID.UnterminatedRawString); | |
835 else | |
836 error(MID.UnterminatedBackQuoteString); | |
837 goto Lreturn; | |
838 default: | |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
839 if (c & 128) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
840 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
841 c = decodeUTF8(); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
842 if (c == LSd || c == PSd) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
843 goto case '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
844 encodeUTF8(buffer, c); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
845 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
846 } |
33 | 847 } |
848 buffer ~= c; // copy character to buffer | |
849 } | |
850 assert(0); | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
851 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
852 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
853 void scanHexStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
854 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
855 assert(p[0] == 'x' && p[1] == '"'); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
856 t.type = TOK.String; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
857 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
858 uint c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
859 ubyte[] buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
860 ubyte h; // hex number |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
861 uint n; // number of hex digits |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
862 |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
863 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
864 while (1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
865 { |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
866 c = *++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
867 switch (c) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
868 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
869 case '"': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
870 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
871 if (n & 1) |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
872 error(MID.OddNumberOfDigitsInHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
873 t.pf = scanPostfix(); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
874 Lreturn: |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
875 buffer ~= 0; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
876 t.str = cast(string) buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
877 t.end = p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
878 return; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
879 case '\r': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
880 if (p[1] == '\n') |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
881 ++p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
882 case '\n': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
883 ++loc; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
884 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
885 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
886 if (ishexad(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
887 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
888 if (c <= '9') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
889 c -= '0'; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
890 else if (c <= 'F') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
891 c -= 'A' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
892 else |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
893 c -= 'a' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
894 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
895 if (n & 1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
896 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
897 h <<= 4; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
898 h |= c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
899 buffer ~= h; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
900 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
901 else |
112 | 902 h = cast(ubyte)c; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
903 ++n; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
904 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
905 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
906 else if (isspace(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
907 continue; |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
908 |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
909 if (c & 128) |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
910 { |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
911 c = decodeUTF8(); |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
912 if (c == LSd || c == PSd) |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
913 { |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
914 ++p; ++p; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
915 ++loc; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
916 continue; |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
917 } |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
918 } |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
919 else if (c == 0 || c == _Z_) |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
920 { |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
921 error(MID.UnterminatedHexString); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
922 t.pf = 0; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
923 goto Lreturn; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
924 } |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
925 error(MID.NonHexCharInHexString, cast(dchar)c); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
926 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
927 } |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
928 assert(0); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
929 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
930 |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
931 version(D2) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
932 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
933 void scanDelimitedStringLiteral(ref Token t) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
934 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
935 assert(p[0] == 'q' && p[1] == '"'); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
936 t.type = TOK.String; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
937 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
938 char[] buffer; |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
939 dchar opening_delim = 0, // 0 if no nested delimiter or '[', '(', '<', '{' |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
940 closing_delim; // Will be ']', ')', '>', '}, |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
941 // the first character of an identifier or |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
942 // any other Unicode/ASCII character. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
943 char[] str_delim; // Identifier delimiter. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
944 uint level = 1; // Counter for nestable delimiters. |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
945 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
946 ++p; ++p; // Skip q" |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
947 uint c = *p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
948 switch (c) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
949 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
950 case '(': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
951 opening_delim = c; |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
952 closing_delim = ')'; // c + 1 |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
953 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
954 case '[', '<', '{': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
955 opening_delim = c; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
956 closing_delim = c + 2; // Get to closing counterpart. Feature of ASCII table. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
957 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
958 default: |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
959 dchar scanNewline() |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
960 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
961 switch (*p) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
962 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
963 case '\r': |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
964 if (p[1] == '\n') |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
965 ++p; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
966 case '\n': |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
967 ++p; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
968 ++loc; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
969 return '\n'; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
970 case LS[0]: |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
971 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
972 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
973 ++p; ++p; ++p; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
974 ++loc; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
975 return '\n'; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
976 } |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
977 default: |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
978 } |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
979 return 0; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
980 } |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
981 |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
982 // Skip leading newlines: |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
983 while (scanNewline() != 0){} |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
984 assert(*p != '\n' && *p != '\r'); |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
985 assert(!(*p == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))); |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
986 |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
987 char* begin = p; |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
988 c = *p; |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
989 closing_delim = c; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
990 // TODO: Check for non-printable characters? |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
991 if (c & 128) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
992 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
993 closing_delim = decodeUTF8(); |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
994 if (!isUniAlpha(closing_delim)) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
995 break; // Not an identifier. |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
996 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
997 else if (!isidbeg(c)) |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
998 break; // Not an identifier. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
999 |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1000 // Parse Identifier + EndOfLine |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1001 do |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1002 { c = *++p; } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1003 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8())) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1004 // Store identifier |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1005 str_delim = begin[0..p-begin]; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1006 // Scan newline |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1007 if (scanNewline() == '\n') |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1008 --p; // Go back one because of "c = *++p;" in main loop. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1009 else |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1010 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1011 // TODO: error(MID.ExpectedNewlineAfterIdentDelim); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1012 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1013 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1014 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1015 bool checkStringDelim(char* p) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1016 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1017 assert(str_delim.length != 0); |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1018 if (buffer[$-1] == '\n' && // Last character copied to buffer must be '\n'. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1019 end-p >= str_delim.length && // Check remaining length. |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1020 p[0..str_delim.length] == str_delim) // Compare. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1021 return true; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1022 return false; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1023 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1024 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1025 while (1) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1026 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1027 c = *++p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1028 switch (c) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1029 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1030 case '\r': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1031 if (p[1] == '\n') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1032 ++p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1033 case '\n': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1034 c = '\n'; // Convert EndOfLine ('\r','\r\n','\n',LS,PS) to '\n' |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1035 ++loc; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1036 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1037 case 0, _Z_: |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1038 // TODO: error(MID.UnterminatedDelimitedString); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1039 goto Lreturn3; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1040 default: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1041 if (c & 128) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1042 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1043 auto begin = p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1044 c = decodeUTF8(); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1045 if (c == LSd || c == PSd) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1046 goto case '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1047 if (c == closing_delim) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1048 { |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1049 if (str_delim.length) |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1050 { |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1051 if (checkStringDelim(begin)) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1052 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1053 p = begin + str_delim.length; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1054 goto Lreturn2; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1055 } |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1056 } |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1057 else |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1058 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1059 assert(level == 1); |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1060 --level; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1061 goto Lreturn; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1062 } |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1063 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1064 encodeUTF8(buffer, c); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1065 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1066 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1067 else |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1068 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1069 if (c == opening_delim) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1070 ++level; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1071 else if (c == closing_delim) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1072 { |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1073 if (str_delim.length) |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1074 { |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1075 if (checkStringDelim(p)) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1076 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1077 p += str_delim.length; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1078 goto Lreturn2; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1079 } |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1080 } |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1081 else if (--level == 0) |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1082 goto Lreturn; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1083 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1084 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1085 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1086 buffer ~= c; // copy character to buffer |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1087 } |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1088 Lreturn: // Character delimiter. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1089 assert(c == closing_delim); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1090 assert(level == 0); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1091 ++p; // Skip closing delimiter. |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1092 Lreturn2: // String delimiter. |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1093 if (*p == '"') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1094 ++p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1095 // else |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1096 // TODO: error(MID.ExpectedDblQuoteAfterDelim, str_delim.length ? str_delim : p[-1]); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1097 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1098 t.pf = scanPostfix(); |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1099 Lreturn3: // Error. |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1100 t.str = buffer ~ '\0'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1101 t.end = p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1102 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1103 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1104 void scanTokenStringLiteral(ref Token t) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1105 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1106 assert(p[0] == 'q' && p[1] == '{'); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1107 t.type = TOK.String; |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1108 |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1109 // A guard against changes to particular members: |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1110 // this.loc_old, this.loc_hline and this.fileName |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1111 ++inTokenString; |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1112 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1113 uint loc = this.loc; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1114 uint level = 1; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1115 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1116 ++p; ++p; // Skip q{ |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1117 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1118 auto prev_t = &t; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1119 Token* token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1120 while (1) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1121 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1122 token = new Token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1123 scan(*token); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1124 // Save the tokens in a doubly linked list. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1125 // Could be useful for various tools. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1126 token.prev = prev_t; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1127 prev_t.next = token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1128 prev_t = token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1129 switch (token.type) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1130 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1131 case TOK.LBrace: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1132 ++level; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1133 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1134 case TOK.RBrace: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1135 if (--level == 0) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1136 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1137 t.tok_str = t.next; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1138 t.next = null; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1139 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1140 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1141 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1142 case TOK.EOF: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1143 // TODO: error(MID.UnterminatedTokenString); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1144 t.tok_str = t.next; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1145 t.next = token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1146 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1147 default: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1148 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1149 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1150 break; // Exit loop. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1151 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1152 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1153 assert(token.type == TOK.RBrace || token.type == TOK.EOF); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1154 assert(token.type == TOK.RBrace && t.next is null || |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1155 token.type == TOK.EOF && t.next !is null); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1156 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1157 char[] buffer; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1158 // token points to } or EOF |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1159 if (token.type == TOK.EOF) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1160 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1161 t.end = token.start; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1162 buffer = t.srcText[2..$].dup ~ '\0'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1163 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1164 else |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1165 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1166 // Assign to buffer before scanPostfix(). |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1167 t.end = p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1168 buffer = t.srcText[2..$-1].dup ~ '\0'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1169 t.pf = scanPostfix(); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1170 t.end = p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1171 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1172 // Convert EndOfLines to '\n' |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1173 if (loc != this.loc) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1174 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1175 assert(buffer[$-1] == '\0'); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1176 uint i, j; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1177 for (; i < buffer.length; ++i) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1178 switch (buffer[i]) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1179 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1180 case '\r': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1181 if (buffer[i+1] == '\n') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1182 ++i; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1183 case '\n': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1184 buffer[j++] = '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1185 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1186 case LS[0]: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1187 auto b = buffer[i..$]; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1188 if (b[1] == LS[1] && (b[2] == LS[2] || b[2] == PS[2])) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1189 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1190 ++i; ++i; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1191 goto case '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1192 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1193 // goto default; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1194 default: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1195 buffer[j++] = buffer[i]; // Copy character |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1196 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1197 buffer.length = j; // Adjust length |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1198 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1199 assert(buffer[$-1] == '\0'); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1200 t.str = buffer; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1201 |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1202 --inTokenString; |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1203 } |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1204 } // version(D2) |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1205 |
45 | 1206 dchar scanEscapeSequence() |
1207 { | |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1208 assert(*p == '\\'); |
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1209 ++p; |
45 | 1210 uint c = char2ev(*p); |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1211 if (c) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1212 { |
46 | 1213 ++p; |
45 | 1214 return c; |
46 | 1215 } |
45 | 1216 uint digits = 2; |
1217 | |
1218 switch (*p) | |
1219 { | |
1220 case 'x': | |
1221 c = 0; | |
1222 while (1) | |
1223 { | |
1224 ++p; | |
1225 if (ishexad(*p)) | |
1226 { | |
1227 c *= 16; | |
1228 if (*p <= '9') | |
48 | 1229 c += *p - '0'; |
45 | 1230 else if (*p <= 'F') |
48 | 1231 c += *p - 'A' + 10; |
45 | 1232 else |
48 | 1233 c += *p - 'a' + 10; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1234 |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1235 if (!--digits) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1236 { |
48 | 1237 ++p; |
45 | 1238 break; |
48 | 1239 } |
45 | 1240 } |
1241 else | |
1242 { | |
1243 error(MID.InsufficientHexDigits); | |
1244 break; | |
1245 } | |
1246 } | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1247 if (!isValidDchar(c)) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1248 error(MID.InvalidUnicodeCharacter); |
45 | 1249 break; |
1250 case 'u': | |
1251 digits = 4; | |
1252 goto case 'x'; | |
1253 case 'U': | |
1254 digits = 8; | |
1255 goto case 'x'; | |
1256 default: | |
48 | 1257 if (isoctal(*p)) |
45 | 1258 { |
48 | 1259 c = 0; |
1260 c += *p - '0'; | |
1261 ++p; | |
1262 if (!isoctal(*p)) | |
1263 return c; | |
1264 c *= 8; | |
1265 c += *p - '0'; | |
1266 ++p; | |
1267 if (!isoctal(*p)) | |
1268 return c; | |
1269 c *= 8; | |
1270 c += *p - '0'; | |
1271 ++p; | |
1272 } | |
1273 else if(*p == '&') | |
1274 { | |
1275 if (isalpha(*++p)) | |
45 | 1276 { |
272
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1277 auto begin = p; |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1278 while (isalnum(*++p)) |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1279 {} |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1280 |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1281 if (*p == ';') |
48 | 1282 { |
272
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1283 c = entity2Unicode(begin[0..p - begin]); |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1284 ++p; // Skip ; |
272
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1285 if (c == 0xFFFF) |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1286 error(MID.UndefinedHTMLEntity, (begin-1)[0..p-(begin-1)]); |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1287 } |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1288 else |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1289 error(MID.UnterminatedHTMLEntity); |
45 | 1290 } |
48 | 1291 else |
1292 error(MID.InvalidBeginHTMLEntity); | |
45 | 1293 } |
1294 else | |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1295 { |
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1296 dchar d = *p; |
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1297 char[] str = `\`; |
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1298 if (d & 128) |
387
ad0cbd1c8881
Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
386
diff
changeset
|
1299 encodeUTF8(str, decodeUTF8()); |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1300 else |
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1301 str ~= d; |
387
ad0cbd1c8881
Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
386
diff
changeset
|
1302 ++p; |
ad0cbd1c8881
Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
386
diff
changeset
|
1303 // TODO: check for unprintable character? |
ad0cbd1c8881
Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
386
diff
changeset
|
1304 error(MID.UndefinedEscapeSequence, str); |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1305 } |
45 | 1306 } |
1307 | |
1308 return c; | |
1309 } | |
1310 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1311 /* |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1312 IntegerLiteral:= (Dec|Hex|Bin|Oct)Suffix? |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1313 Dec:= (0|[1-9][0-9_]*) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1314 Hex:= 0[xX] HexDigits |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1315 Bin:= 0[bB][01_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1316 Oct:= 0[0-7_]+ |
68
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1317 Suffix:= (L[uU]?|[uU]L?) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1318 HexDigits:= [0-9a-zA-Z_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1319 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1320 Invalid: "0b_", "0x_", "._" |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1321 */ |
15 | 1322 void scanNumber(ref Token t) |
1323 { | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1324 ulong ulong_; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1325 bool overflow; |
58 | 1326 bool isDecimal; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1327 size_t digits; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1328 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1329 if (*p != '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1330 goto LscanInteger; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1331 ++p; // skip zero |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1332 // check for xX bB ... |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1333 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1334 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1335 case 'x','X': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1336 goto LscanHex; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1337 case 'b','B': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1338 goto LscanBin; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1339 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1340 if (p[1] == 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1341 goto LscanReal; |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1342 break; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1343 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1344 if (p[1] == '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1345 break; |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1346 case 'i','f','F', // Imaginary and float literal suffixes. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1347 'e', 'E': // Float exponent. |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1348 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1349 default: |
323 | 1350 if (*p == '_' || isoctal(*p)) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1351 goto LscanOct; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1352 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1353 |
323 | 1354 // Number 0 |
1355 assert(p[-1] == '0'); | |
1356 assert(ulong_ == 0); | |
58 | 1357 isDecimal = true; |
1358 goto Lfinalize; | |
1359 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1360 LscanInteger: |
323 | 1361 assert(*p != 0 && isdigit(*p)); |
58 | 1362 isDecimal = true; |
323 | 1363 goto Lenter_loop_int; |
63 | 1364 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1365 { |
63 | 1366 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1367 continue; |
63 | 1368 if (!isdigit(*p)) |
1369 break; | |
323 | 1370 Lenter_loop_int: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1371 if (ulong_ < ulong.max/10 || (ulong_ == ulong.max/10 && *p <= '5')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1372 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1373 ulong_ *= 10; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1374 ulong_ += *p - '0'; |
58 | 1375 continue; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1376 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1377 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1378 overflow = true; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1379 while (isdigit(*++p)) {} |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1380 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1381 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1382 |
61 | 1383 // The number could be a float, so check overflow below. |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1384 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1385 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1386 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1387 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1388 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1389 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1390 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1391 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1392 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1393 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1394 goto LscanReal; |
58 | 1395 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1396 } |
58 | 1397 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1398 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1399 error(MID.OverflowDecimalNumber); |
58 | 1400 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1401 assert((isdigit(p[-1]) || p[-1] == '_') && !isdigit(*p) && *p != '_'); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1402 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1403 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1404 LscanHex: |
61 | 1405 assert(digits == 0); |
323 | 1406 assert(*p == 'x'); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1407 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1408 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1409 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1410 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1411 if (!ishexad(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1412 break; |
61 | 1413 ++digits; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1414 ulong_ *= 16; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1415 if (*p <= '9') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1416 ulong_ += *p - '0'; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1417 else if (*p <= 'F') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1418 ulong_ += *p - 'A' + 10; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1419 else |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1420 ulong_ += *p - 'a' + 10; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1421 } |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1422 |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1423 assert(ishexad(p[-1]) || p[-1] == '_' || p[-1] == 'x'); |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1424 assert(!ishexad(*p) && *p != '_'); |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1425 |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1426 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1427 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1428 case '.': |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1429 if (p[1] == '.') |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1430 break; |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1431 case 'p', 'P': |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1432 return scanHexReal(t); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1433 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1434 } |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1435 |
61 | 1436 if (digits == 0) |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1437 error(MID.NoDigitsInHexNumber); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1438 else if (digits > 16) |
61 | 1439 error(MID.OverflowHexNumber); |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1440 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1441 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1442 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1443 LscanBin: |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1444 assert(digits == 0); |
323 | 1445 assert(*p == 'b'); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1446 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1447 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1448 if (*++p == '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1449 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1450 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1451 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1452 } |
367
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1453 else if (*p == '1') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1454 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1455 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1456 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1457 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1458 } |
367
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1459 else if (*p == '_') |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1460 continue; |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1461 else |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1462 break; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1463 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1464 |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1465 if (digits == 0) |
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1466 error(MID.NoDigitsInBinNumber); |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1467 else if (digits > 64) |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1468 error(MID.OverflowBinaryNumber); |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
1469 |
367
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1470 assert(p[-1] == '0' || p[-1] == '1' || p[-1] == '_', p[-1] ~ ""); |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
1471 assert( !(*p == '0' || *p == '1' || *p == '_') ); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1472 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1473 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1474 LscanOct: |
323 | 1475 assert(*p == '_' || isoctal(*p)); |
1476 if (*p != '_') | |
1477 goto Lenter_loop_oct; | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1478 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1479 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1480 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1481 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1482 if (!isoctal(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1483 break; |
323 | 1484 Lenter_loop_oct: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1485 if (ulong_ < ulong.max/2 || (ulong_ == ulong.max/2 && *p <= '1')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1486 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1487 ulong_ *= 8; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1488 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1489 ++p; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1490 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1491 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1492 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1493 overflow = true; |
63 | 1494 while (isdigit(*++p)) {} |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1495 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1496 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1497 |
63 | 1498 bool hasDecimalDigits; |
1499 if (isdigit(*p)) | |
1500 { | |
1501 hasDecimalDigits = true; | |
1502 while (isdigit(*++p)) {} | |
1503 } | |
1504 | |
1505 // The number could be a float, so check errors below. | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1506 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1507 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1508 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1509 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1510 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1511 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1512 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1513 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1514 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1515 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1516 goto LscanReal; |
63 | 1517 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1518 } |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1519 |
63 | 1520 if (hasDecimalDigits) |
1521 error(MID.OctalNumberHasDecimals); | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1522 if (overflow) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1523 error(MID.OverflowOctalNumber); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1524 // goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1525 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1526 Lfinalize: |
58 | 1527 enum Suffix |
1528 { | |
1529 None = 0, | |
1530 Unsigned = 1, | |
60 | 1531 Long = 2 |
58 | 1532 } |
60 | 1533 |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1534 // Scan optional suffix: L, Lu, LU, u, uL, U or UL. |
58 | 1535 Suffix suffix; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1536 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1537 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1538 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1539 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1540 case 'L': |
58 | 1541 if (suffix & Suffix.Long) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1542 break; |
60 | 1543 suffix |= Suffix.Long; |
1544 ++p; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1545 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1546 case 'u', 'U': |
58 | 1547 if (suffix & Suffix.Unsigned) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1548 break; |
60 | 1549 suffix |= Suffix.Unsigned; |
1550 ++p; | |
1551 continue; | |
58 | 1552 default: |
1553 break; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1554 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1555 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1556 } |
58 | 1557 |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1558 // Determine type of Integer. |
58 | 1559 switch (suffix) |
1560 { | |
1561 case Suffix.None: | |
1562 if (ulong_ & 0x8000000000000000) | |
1563 { | |
1564 if (isDecimal) | |
1565 error(MID.OverflowDecimalSign); | |
1566 t.type = TOK.Uint64; | |
1567 } | |
1568 else if (ulong_ & 0xFFFFFFFF00000000) | |
1569 t.type = TOK.Int64; | |
1570 else if (ulong_ & 0x80000000) | |
1571 t.type = isDecimal ? TOK.Int64 : TOK.Uint32; | |
1572 else | |
1573 t.type = TOK.Int32; | |
1574 break; | |
1575 case Suffix.Unsigned: | |
1576 if (ulong_ & 0xFFFFFFFF00000000) | |
1577 t.type = TOK.Uint64; | |
1578 else | |
1579 t.type = TOK.Uint32; | |
1580 break; | |
1581 case Suffix.Long: | |
1582 if (ulong_ & 0x8000000000000000) | |
1583 { | |
1584 if (isDecimal) | |
1585 error(MID.OverflowDecimalSign); | |
1586 t.type = TOK.Uint64; | |
1587 } | |
1588 else | |
1589 t.type = TOK.Int64; | |
1590 break; | |
1591 case Suffix.Unsigned | Suffix.Long: | |
1592 t.type = TOK.Uint64; | |
1593 break; | |
1594 default: | |
1595 assert(0); | |
1596 } | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1597 t.ulong_ = ulong_; |
15 | 1598 t.end = p; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1599 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1600 LscanReal: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1601 scanReal(t); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1602 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1603 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1604 |
63 | 1605 /* |
1606 FloatLiteral:= Float[fFL]?i? | |
1607 Float:= DecFloat | HexFloat | |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1608 DecFloat:= ([0-9][0-9_]*[.][0-9_]*DecExponent?) | [.][0-9][0-9_]*DecExponent? | [0-9][0-9_]*DecExponent |
63 | 1609 DecExponent:= [eE][+-]?[0-9][0-9_]* |
1610 HexFloat:= 0[xX](HexDigits[.]HexDigits | [.][0-9a-zA-Z]HexDigits? | HexDigits)HexExponent | |
1611 HexExponent:= [pP][+-]?[0-9][0-9_]* | |
1612 */ | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1613 void scanReal(ref Token t) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1614 { |
63 | 1615 if (*p == '.') |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1616 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1617 assert(p[1] != '.'); |
63 | 1618 // This function was called by scan() or scanNumber(). |
1619 while (isdigit(*++p) || *p == '_') {} | |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1620 } |
63 | 1621 else |
1622 // This function was called by scanNumber(). | |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1623 assert(delegate (){ |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1624 switch (*p) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1625 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1626 case 'L': |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1627 if (p[1] != 'i') |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1628 return false; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1629 case 'i', 'f', 'F', 'e', 'E': |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1630 return true; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1631 default: |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1632 } |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1633 return false; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1634 }() |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1635 ); |
63 | 1636 |
1637 // Scan exponent. | |
1638 if (*p == 'e' || *p == 'E') | |
1639 { | |
1640 ++p; | |
1641 if (*p == '-' || *p == '+') | |
1642 ++p; | |
1643 if (!isdigit(*p)) | |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1644 error(MID.FloatExpMustStartWithDigit); |
63 | 1645 else |
64 | 1646 while (isdigit(*++p) || *p == '_') {} |
63 | 1647 } |
1648 | |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1649 // Copy whole number and remove underscores from buffer. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1650 char[] buffer = t.start[0..p-t.start].dup; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1651 uint j; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1652 foreach (c; buffer) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1653 if (c != '_') |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1654 buffer[j++] = c; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1655 buffer.length = j; // Adjust length. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1656 buffer ~= 0; // Terminate for C functions. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1657 |
63 | 1658 finalizeFloat(t, buffer); |
30 | 1659 } |
1660 | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1661 void scanHexReal(ref Token t) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1662 { |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1663 assert(*p == '.' || *p == 'p' || *p == 'P'); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1664 MID mid; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1665 if (*p == '.') |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1666 while (ishexad(*++p) || *p == '_') |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1667 {} |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1668 // Decimal exponent is required. |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1669 if (*p != 'p' && *p != 'P') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1670 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1671 mid = MID.HexFloatExponentRequired; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1672 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1673 } |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1674 // Scan exponent |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1675 assert(*p == 'p' || *p == 'P'); |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1676 if (!isdigit(*++p)) |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1677 { |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1678 mid = MID.HexFloatExpMustStartWithDigit; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1679 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1680 } |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1681 while (isdigit(*++p) || *p == '_') |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1682 {} |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1683 // Copy whole number and remove underscores from buffer. |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1684 char[] buffer = t.start[0..p-t.start].dup; |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1685 uint j; |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1686 foreach (c; buffer) |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1687 if (c != '_') |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1688 buffer[j++] = c; |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1689 buffer.length = j; // Adjust length. |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1690 buffer ~= 0; // Terminate for C functions. |
63 | 1691 finalizeFloat(t, buffer); |
1692 return; | |
1693 Lerr: | |
1694 t.type = TOK.Float32; | |
1695 t.end = p; | |
1696 error(mid); | |
1697 } | |
1698 | |
1699 void finalizeFloat(ref Token t, string buffer) | |
1700 { | |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1701 assert(buffer[$-1] == 0); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1702 // Float number is well-formed. Check suffixes and do conversion. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1703 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1704 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1705 case 'f', 'F': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1706 t.type = TOK.Float32; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1707 t.float_ = strtof(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1708 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1709 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1710 case 'L': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1711 t.type = TOK.Float80; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1712 t.real_ = strtold(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1713 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1714 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1715 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1716 t.type = TOK.Float64; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1717 t.double_ = strtod(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1718 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1719 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1720 if (*p == 'i') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1721 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1722 ++p; |
63 | 1723 t.type += 3; // Switch to imaginary counterpart. |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1724 } |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
1725 if (errno() == ERANGE) |
63 | 1726 error(MID.OverflowFloatNumber); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1727 t.end = p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1728 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
1729 |
30 | 1730 /// Scan special token: #line Integer [Filespec] EndOfLine |
360
b6a3755eba94
- Renamed scanSpecialToken() to scanSpecialTokenSequence().
aziz
parents:
350
diff
changeset
|
1731 void scanSpecialTokenSequence(ref Token t) |
30 | 1732 { |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1733 assert(*p == '#'); |
51 | 1734 |
323 | 1735 t.type = TOK.HashLine; |
1736 | |
30 | 1737 MID mid; |
1738 | |
323 | 1739 ++p; |
1740 if (p[0] != 'l' || p[1] != 'i' || p[2] != 'n' || p[3] != 'e') | |
51 | 1741 { |
370 | 1742 mid = MID.ExpectedIdentifierSTLine; |
51 | 1743 goto Lerr; |
1744 } | |
323 | 1745 p += 3; |
1746 | |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1747 // TODO: #line58"path/file" is legal. Require spaces? |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1748 // State.Space could be used for that purpose. |
323 | 1749 enum State |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1750 { /+Space,+/ Integer, Filespec, End } |
51 | 1751 |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1752 State state = State.Integer; |
323 | 1753 |
1754 Loop: | |
1755 while (1) | |
51 | 1756 { |
323 | 1757 switch (*++p) |
1758 { | |
1759 case '\r': | |
1760 if (p[1] == '\n') | |
1761 ++p; | |
1762 case '\n', 0, _Z_: | |
1763 break Loop; | |
1764 case LS[0]: | |
1765 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) | |
1766 { | |
1767 ++p; ++p; | |
1768 break Loop; | |
1769 } | |
1770 goto default; | |
1771 default: | |
1772 if (isspace(*p)) | |
1773 continue; | |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1774 if (state == State.Integer) |
323 | 1775 { |
1776 if (!isdigit(*p)) | |
1777 { | |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1778 mid = MID.ExpectedIntegerAfterSTLine; |
323 | 1779 goto Lerr; |
1780 } | |
1781 t.line_num = new Token; | |
1782 scan(*t.line_num); | |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1783 if (t.line_num.type != TOK.Int32 && t.line_num.type != TOK.Uint32) |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1784 { |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1785 mid = MID.ExpectedIntegerAfterSTLine; |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1786 goto Lerr; |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1787 } |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
1788 --p; // Go one back because scan() advanced p past the integer. |
323 | 1789 state = State.Filespec; |
1790 } | |
1791 else if (state == State.Filespec) | |
1792 { | |
1793 if (*p != '"') | |
1794 { | |
1795 mid = MID.ExpectedFilespec; | |
1796 goto Lerr; | |
1797 } | |
1798 t.line_filespec = new Token; | |
1799 t.line_filespec.start = p; | |
1800 t.line_filespec.type = TOK.Filespec; | |
1801 while (1) | |
1802 { | |
1803 switch (*++p) | |
1804 { | |
1805 case '"': | |
1806 break; | |
1807 case LS[0]: | |
1808 if (!(p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))) | |
1809 goto default; | |
1810 case '\r', '\n', 0, _Z_: | |
1811 mid = MID.UnterminatedFilespec; | |
1812 t.line_filespec.end = p; | |
1813 goto Lerr; | |
1814 default: | |
1815 if (*p & 128) | |
1816 decodeUTF8(); | |
1817 continue; | |
1818 } | |
1819 break; // Exit loop. | |
1820 } | |
1821 auto start = t.line_filespec.start +1; // +1 skips '"' | |
1822 t.line_filespec.str = start[0 .. p - start]; | |
1823 t.line_filespec.end = p + 1; | |
1824 state = State.End; | |
1825 } | |
1826 else/+ if (state == State.End)+/ | |
1827 { | |
1828 mid = MID.UnterminatedSpecialToken; | |
1829 goto Lerr; | |
1830 } | |
1831 } | |
51 | 1832 } |
323 | 1833 |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1834 if (state == State.Integer) |
51 | 1835 { |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
1836 mid = MID.ExpectedIntegerAfterSTLine; |
30 | 1837 goto Lerr; |
1838 } | |
1839 | |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1840 // Evaluate #line only when not in token string. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1841 if (!inTokenString) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1842 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1843 this.loc_old = this.loc; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1844 this.loc_hline = t.line_num.uint_ - 1; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1845 if (t.line_filespec) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1846 this.fileName = t.line_filespec.str; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1847 } |
323 | 1848 t.end = p; |
30 | 1849 |
1850 return; | |
1851 Lerr: | |
323 | 1852 t.end = p; |
30 | 1853 error(mid); |
15 | 1854 } |
1855 | |
371 | 1856 uint errorLoc() |
1857 { | |
1858 // ∆loc + line_num_of(#line) | |
1859 return this.loc - this.loc_old + this.loc_hline; | |
1860 } | |
1861 | |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1862 dchar decodeUTF8() |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1863 { |
49 | 1864 assert(*p & 128, "check for ASCII char before calling decodeUTF8()."); |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1865 size_t idx; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1866 dchar d; |
49 | 1867 try |
1868 { | |
1869 d = std.utf.decode(p[0 .. end-p], idx); | |
1870 p += idx -1; | |
1871 } | |
1872 catch (UtfException e) | |
1873 { | |
1874 error(MID.InvalidUTF8Sequence); | |
1875 // Skip to next valid utf-8 sequence | |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1876 while (p < end && UTF8stride[*++p] != 0xFF) {} |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1877 --p; |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1878 assert(p < end); |
49 | 1879 } |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1880 return d; |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1881 } |
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
1882 |
28 | 1883 void loadKeywords() |
1884 { | |
1885 foreach(k; keywords) | |
1886 idtable[k.str] = k; | |
1887 } | |
382
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1888 /+ // Not needed anymore because tokens are stored in a linked list. |
105 | 1889 struct State |
1890 { | |
1891 Lexer lexer; | |
209 | 1892 Token token; |
105 | 1893 char* scanPointer; |
209 | 1894 int loc; |
212 | 1895 string fileName; |
105 | 1896 size_t errorLen; |
212 | 1897 static State opCall(Lexer lx) |
105 | 1898 { |
1899 State s; | |
212 | 1900 s.lexer = lx; |
1901 s.token = lx.token; | |
1902 s.scanPointer = lx.p; | |
1903 s.loc = lx.loc; | |
1904 s.fileName = lx.fileName; | |
1905 s.errorLen = lx.errors.length; | |
105 | 1906 return s; |
1907 } | |
1908 void restore() | |
1909 { | |
1910 lexer.p = scanPointer; | |
209 | 1911 lexer.token = token; |
1912 lexer.loc = loc; | |
212 | 1913 lexer.fileName = fileName; |
105 | 1914 lexer.errors = lexer.errors[0..errorLen]; |
1915 } | |
1916 } | |
1917 | |
1918 State getState() | |
1919 { | |
212 | 1920 return State(this); |
105 | 1921 } |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1922 +/ |
105 | 1923 |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1924 private void scanNext(ref Token* t) |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1925 { |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1926 assert(t !is null); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1927 if (t.next) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1928 t = t.next; |
344 | 1929 else if (t != this.tail) |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1930 { |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1931 Token* new_t = new Token; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1932 scan(*new_t); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1933 new_t.prev = t; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1934 t.next = new_t; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1935 t = new_t; |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1936 } |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1937 } |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1938 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1939 void peek(ref Token* t) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1940 { |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1941 scanNext(t); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1942 } |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1943 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1944 TOK nextToken() |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1945 { |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1946 scanNext(this.token); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1947 return this.token.type; |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1948 } |
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
1949 |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
1950 void error(MID mid, ...) |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1951 { |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
1952 // if (reportErrors) |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
1953 errors ~= new Information(InfoType.Lexer, mid, this.errorLoc, Format(_arguments, _argptr, GetMsg(mid))); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1954 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1955 |
207 | 1956 unittest |
1957 { | |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
1958 Stdout("Testing method Lexer.peek()\n"); |
207 | 1959 string sourceText = "unittest { }"; |
1960 auto lx = new Lexer(sourceText, null); | |
1961 | |
350 | 1962 Token* next = lx.head; |
207 | 1963 lx.peek(next); |
350 | 1964 assert(next.type == TOK.Unittest); |
207 | 1965 lx.peek(next); |
350 | 1966 assert(next.type == TOK.LBrace); |
207 | 1967 lx.peek(next); |
350 | 1968 assert(next.type == TOK.RBrace); |
207 | 1969 lx.peek(next); |
350 | 1970 assert(next.type == TOK.EOF); |
207 | 1971 } |
1972 | |
306 | 1973 Token* getTokens() |
4 | 1974 { |
306 | 1975 while (nextToken() != TOK.EOF) |
1976 {} | |
1977 return head; | |
3 | 1978 } |
48 | 1979 |
365 | 1980 static bool isNonReservedIdentifier(char[] ident) |
1981 { | |
382
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1982 if (ident.length == 0) |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1983 return false; |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1984 |
365 | 1985 static Identifier[string] reserved_ids_table; |
1986 if (reserved_ids_table is null) | |
1987 foreach(k; keywords) | |
1988 reserved_ids_table[k.str] = k; | |
1989 | |
382
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1990 size_t idx = 1; // Index to the 2nd character in ident. |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1991 dchar isFirstCharUniAlpha() |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1992 { |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1993 idx = 0; |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1994 // NB: decode() could throw an Exception which would be |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1995 // caught by the next try-catch-block. |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1996 return isUniAlpha(std.utf.decode(ident, idx)); |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1997 } |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
1998 |
365 | 1999 try |
382
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2000 { |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2001 if (isidbeg(ident[0]) || |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2002 ident[0] & 128 && isFirstCharUniAlpha()) |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2003 { |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2004 foreach (dchar c; ident[idx..$]) |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2005 if (!isident(c) && !isUniAlpha(c)) |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2006 return false; |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2007 } |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2008 } |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2009 catch (Exception) |
365 | 2010 return false; |
2011 | |
2012 return !(ident in reserved_ids_table); | |
2013 } | |
2014 | |
48 | 2015 private void encodeUTF8(inout char[] str, dchar d) |
2016 { | |
2017 char[6] b; | |
2018 assert(d > 0x7F, "check for ASCII char before calling encodeUTF8()."); | |
2019 if (d < 0x800) | |
2020 { | |
2021 b[0] = 0xC0 | (d >> 6); | |
2022 b[1] = 0x80 | (d & 0x3F); | |
2023 str ~= b[0..2]; | |
2024 } | |
2025 else if (d < 0x10000) | |
2026 { | |
2027 b[0] = 0xE0 | (d >> 12); | |
2028 b[1] = 0x80 | ((d >> 6) & 0x3F); | |
2029 b[2] = 0x80 | (d & 0x3F); | |
2030 str ~= b[0..3]; | |
2031 } | |
2032 else if (d < 0x200000) | |
2033 { | |
2034 b[0] = 0xF0 | (d >> 18); | |
2035 b[1] = 0x80 | ((d >> 12) & 0x3F); | |
2036 b[2] = 0x80 | ((d >> 6) & 0x3F); | |
2037 b[3] = 0x80 | (d & 0x3F); | |
2038 str ~= b[0..4]; | |
2039 } | |
2040 else if (d < 0x4000000) | |
2041 { | |
2042 b[0] = 0xF8 | (d >> 24); | |
2043 b[1] = 0x80 | ((d >> 18) & 0x3F); | |
2044 b[2] = 0x80 | ((d >> 12) & 0x3F); | |
2045 b[3] = 0x80 | ((d >> 6) & 0x3F); | |
2046 b[4] = 0x80 | (d & 0x3F); | |
2047 str ~= b[0..5]; | |
2048 } | |
2049 else if (d < 0x80000000) | |
2050 { | |
2051 b[0] = 0xFC | (d >> 30); | |
2052 b[1] = 0x80 | ((d >> 24) & 0x3F); | |
2053 b[2] = 0x80 | ((d >> 18) & 0x3F); | |
2054 b[3] = 0x80 | ((d >> 12) & 0x3F); | |
2055 b[4] = 0x80 | ((d >> 6) & 0x3F); | |
2056 b[5] = 0x80 | (d & 0x3F); | |
2057 str ~= b[0..6]; | |
2058 } | |
2059 else | |
2060 error(MID.InvalidUnicodeCharacter); | |
2061 } | |
5 | 2062 } |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2063 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2064 unittest |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2065 { |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
2066 Stdout("Testing Lexer.\n"); |
40 | 2067 string[] toks = [ |
2068 ">", ">=", ">>", ">>=", ">>>", ">>>=", "<", "<=", "<>", | |
2069 "<>=", "<<", "<<=", "!", "!<", "!>", "!<=", "!>=", "!<>", | |
2070 "!<>=", ".", "..", "...", "&", "&&", "&=", "+", "++", | |
2071 "+=", "-", "--", "-=", "=", "==", "~", "~=", "*", | |
2072 "*=", "/", "/=", "^", "^=", "%", "%=", "(", ")", | |
2073 "[", "]", "{", "}", ":", ";", "?", ",", "$" | |
2074 ]; | |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2075 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2076 char[] src; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2077 |
40 | 2078 foreach (op; toks) |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2079 src ~= op ~ " "; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2080 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2081 auto lx = new Lexer(src, ""); |
350 | 2082 auto token = lx.getTokens(); |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2083 |
350 | 2084 uint i; |
2085 assert(token == lx.head); | |
2086 token = token.next; | |
2087 do | |
2088 { | |
2089 assert(i < toks.length); | |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
2090 assert(token.srcText == toks[i], Format("Scanned '{0}' but expected '{1}'", token.srcText, toks[i])); |
350 | 2091 ++i; |
2092 token = token.next; | |
2093 } while (token.type != TOK.EOF) | |
41 | 2094 } |
55 | 2095 |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2096 unittest |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2097 { |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2098 // Numbers unittest |
68
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2099 // 0L 0ULi 0_L 0_UL 0x0U 0x0p2 0_Fi 0_e2 0_F 0_i |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2100 // 0u 0U 0uL 0UL 0L 0LU 0Lu |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2101 // 0Li 0f 0F 0fi 0Fi 0i |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2102 // 0b_1_LU 0b1000u |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2103 // 0x232Lu |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2104 } |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2105 |
55 | 2106 /// ASCII character properties table. |
2107 static const int ptable[256] = [ | |
2108 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0, | |
2109 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2110 32, 0, 0x2200, 0, 0, 0, 0, 0x2700, 0, 0, 0, 0, 0, 0, 0, 0, | |
2111 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0x3f00, | |
2112 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8, | |
2113 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0x5c00, 0, 0,16, | |
2114 0, 0x70c, 0x80c,12,12,12, 0xc0c, 8, 8, 8, 8, 8, 8, 8, 0xa08, 8, | |
2115 8, 8, 0xd08, 8, 0x908, 8, 0xb08, 8, 8, 8, 8, 0, 0, 0, 0, 0, | |
2116 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2117 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2118 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2119 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2120 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2121 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2122 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2123 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 | |
2124 ]; | |
2125 | |
2126 enum CProperty | |
2127 { | |
2128 Octal = 1, | |
2129 Digit = 1<<1, | |
2130 Hex = 1<<2, | |
2131 Alpha = 1<<3, | |
2132 Underscore = 1<<4, | |
2133 Whitespace = 1<<5 | |
2134 } | |
2135 | |
2136 const uint EVMask = 0xFF00; // Bit mask for escape value | |
2137 | |
2138 private alias CProperty CP; | |
2139 int isoctal(char c) { return ptable[c] & CP.Octal; } | |
2140 int isdigit(char c) { return ptable[c] & CP.Digit; } | |
2141 int ishexad(char c) { return ptable[c] & CP.Hex; } | |
2142 int isalpha(char c) { return ptable[c] & CP.Alpha; } | |
2143 int isalnum(char c) { return ptable[c] & (CP.Alpha | CP.Digit); } | |
2144 int isidbeg(char c) { return ptable[c] & (CP.Alpha | CP.Underscore); } | |
2145 int isident(char c) { return ptable[c] & (CP.Alpha | CP.Underscore | CP.Digit); } | |
2146 int isspace(char c) { return ptable[c] & CP.Whitespace; } | |
2147 int char2ev(char c) { return ptable[c] >> 8; /*(ptable[c] & EVMask) >> 8;*/ } | |
2148 | |
2149 version(gen_ptable) | |
2150 static this() | |
2151 { | |
2152 alias ptable p; | |
2153 // Initialize character properties table. | |
2154 for (int i; i < p.length; ++i) | |
2155 { | |
2156 p[i] = 0; | |
2157 if ('0' <= i && i <= '7') | |
2158 p[i] |= CP.Octal; | |
2159 if ('0' <= i && i <= '9') | |
2160 p[i] |= CP.Digit; | |
2161 if (isdigit(i) || 'a' <= i && i <= 'f' || 'A' <= i && i <= 'F') | |
2162 p[i] |= CP.Hex; | |
2163 if ('a' <= i && i <= 'z' || 'A' <= i && i <= 'Z') | |
2164 p[i] |= CP.Alpha; | |
2165 if (i == '_') | |
2166 p[i] |= CP.Underscore; | |
2167 if (i == ' ' || i == '\t' || i == '\v' || i == '\f') | |
2168 p[i] |= CP.Whitespace; | |
2169 } | |
2170 // Store escape sequence values in second byte. | |
2171 assert(CProperty.max <= ubyte.max, "character property flags and escape value byte overlap."); | |
2172 p['\''] |= 39 << 8; | |
2173 p['"'] |= 34 << 8; | |
2174 p['?'] |= 63 << 8; | |
2175 p['\\'] |= 92 << 8; | |
2176 p['a'] |= 7 << 8; | |
2177 p['b'] |= 8 << 8; | |
2178 p['f'] |= 12 << 8; | |
2179 p['n'] |= 10 << 8; | |
2180 p['r'] |= 13 << 8; | |
2181 p['t'] |= 9 << 8; | |
2182 p['v'] |= 11 << 8; | |
2183 // Print a formatted array literal. | |
2184 char[] array = "[\n"; | |
2185 for (int i; i < p.length; ++i) | |
2186 { | |
2187 int c = p[i]; | |
2188 array ~= std.string.format(c>255?" 0x%x,":"%2d,", c, ((i+1) % 16) ? "":"\n"); | |
2189 } | |
2190 array[$-2..$] = "\n]"; | |
2191 writefln(array); | |
2192 } |