Mercurial > projects > dil
annotate trunk/src/dil/Lexer.d @ 433:063cd016c913
Fixed some issues in scanEscapeSequence().
author | Aziz K?ksal <aziz.koeksal@gmail.com> |
---|---|
date | Thu, 04 Oct 2007 22:31:59 +0200 |
parents | 3ead178e0662 |
children | cac9c8d637ad |
rev | line source |
---|---|
0 | 1 /++ |
2 Author: Aziz Köksal | |
249 | 3 License: GPL3 |
0 | 4 +/ |
326 | 5 module dil.Lexer; |
327 | 6 import dil.Token; |
7 import dil.Information; | |
8 import dil.Keywords; | |
9 import dil.Identifier; | |
10 import dil.Messages; | |
11 import dil.HtmlEntities; | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
12 import dil.Settings; |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
13 import tango.stdc.stdlib : strtof, strtod, strtold; |
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
14 import tango.stdc.errno : errno, ERANGE; |
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
15 import tango.stdc.time : time_t, time, ctime; |
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
16 import tango.stdc.string : strlen; |
4 | 17 import std.utf; |
18 import std.uni; | |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
19 import common; |
0 | 20 |
430
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
21 const char[3] LS = \u2028; /// Line separator. |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
22 const char[3] PS = \u2029; /// Paragraph separator. |
11 | 23 |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
24 const dchar LSd = 0x2028; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
25 const dchar PSd = 0x2029; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
26 |
430
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
27 /// U+FFFD = �. Used to replace invalid Unicode characters. |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
28 const dchar REPLACEMENT_CHAR = '\uFFFD'; |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
29 |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
30 const uint _Z_ = 26; /// Control+Z |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
31 |
0 | 32 class Lexer |
33 { | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
34 Token* head; /// The head of the doubly linked token list. |
344 | 35 Token* tail; /// The tail of the linked list. Set in scan(). |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
36 Token* token; /// Points to the current token in the token list. |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
37 string text; /// The source text. |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
38 char[] filePath; /// Path to the source file. |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
39 char* p; /// Points to the current character in the source text. |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
40 char* end; /// Points one character past the end of the source text. |
4 | 41 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
42 // Members used to generate error messages: |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
43 Information[] errors; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
44 char* lineBegin; /// Always points to the beginning of the current line. |
371 | 45 uint loc = 1; /// Actual line of code. |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
46 uint loc_old; /// Store actual line number when #line token is scanned. |
371 | 47 uint loc_hline; /// Line number set by #line. |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
48 uint inTokenString; /// > 0 if inside q{ } |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
49 Location errorLoc; |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
50 |
28 | 51 Identifier[string] idtable; |
52 | |
412
fb31af0fda73
Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
411
diff
changeset
|
53 version(token2LocTable) |
fb31af0fda73
Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
411
diff
changeset
|
54 /// Maps every token that starts a new line to a Location. |
fb31af0fda73
Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
411
diff
changeset
|
55 Location[Token*] token2LocTable; |
fb31af0fda73
Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
411
diff
changeset
|
56 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
57 this(string text, string filePath) |
4 | 58 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
59 this.filePath = filePath; |
30 | 60 |
4 | 61 this.text = text; |
397
c99f8aeb7b4a
Empty source files are handled correctly now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
396
diff
changeset
|
62 if (text.length == 0 || text[$-1] != 0) |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
63 { |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
64 this.text.length = this.text.length + 1; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
65 this.text[$-1] = 0; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
66 } |
4 | 67 |
68 this.p = this.text.ptr; | |
69 this.end = this.p + this.text.length; | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
70 this.lineBegin = this.p; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
71 this.errorLoc = new Location(filePath, 1, this.lineBegin, this.lineBegin); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
72 loadKeywords(this.idtable); |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
73 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
74 this.head = new Token; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
75 this.head.type = TOK.HEAD; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
76 this.token = this.head; |
315 | 77 scanShebang(); |
412
fb31af0fda73
Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
411
diff
changeset
|
78 version(token2LocTable) |
fb31af0fda73
Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
411
diff
changeset
|
79 { |
fb31af0fda73
Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
411
diff
changeset
|
80 // Add first token to table. |
fb31af0fda73
Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
411
diff
changeset
|
81 auto firstToken = this.head; |
fb31af0fda73
Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
411
diff
changeset
|
82 peek(firstToken); |
420
ce644d724d87
Made Location a class and extended it with new members.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
419
diff
changeset
|
83 token2LocTable[firstToken] = new Location(1, null); |
412
fb31af0fda73
Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
411
diff
changeset
|
84 } |
315 | 85 } |
86 | |
344 | 87 ~this() |
88 { | |
89 auto token = head.next; | |
418
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
90 while (token !is null) |
344 | 91 { |
92 assert(token.type == TOK.EOF ? token == tail && token.next is null : 1); | |
93 delete token.prev; | |
94 token = token.next; | |
418
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
95 } |
344 | 96 delete tail; |
97 } | |
98 | |
315 | 99 void scanShebang() |
100 { | |
101 if (*p == '#' && p[1] == '!') | |
102 { | |
103 Token* t = new Token; | |
104 t.start = p; | |
105 t.type = TOK.Shebang; | |
106 ++p; | |
403
e7228859d865
Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
402
diff
changeset
|
107 assert(*p == '!'); |
315 | 108 while (1) |
109 { | |
403
e7228859d865
Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
402
diff
changeset
|
110 t.end = ++p; |
e7228859d865
Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
402
diff
changeset
|
111 switch (*p) |
315 | 112 { |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
113 case '\r', '\n', 0, _Z_: |
315 | 114 break; |
418
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
115 case LS[0]: |
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
116 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
117 break; |
315 | 118 default: |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
119 if (*p & 128) |
418
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
120 decodeUTF8(); |
315 | 121 continue; |
122 } | |
123 break; // Exit loop. | |
124 } | |
403
e7228859d865
Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
402
diff
changeset
|
125 // Reset p. The newline will be scanned as whitespace in scan(). |
e7228859d865
Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
402
diff
changeset
|
126 p = t.end; |
315 | 127 this.head.next = t; |
128 t.prev = this.head; | |
129 } | |
4 | 130 } |
131 | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
132 void finalizeSpecialToken(ref Token t) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
133 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
134 assert(t.srcText[0..2] == "__"); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
135 switch (t.type) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
136 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
137 case TOK.FILE: |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
138 t.str = this.errorLoc.filePath; |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
139 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
140 case TOK.LINE: |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
141 t.uint_ = this.errorLineNum(this.loc); |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
142 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
143 case TOK.DATE, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
144 TOK.TIME, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
145 TOK.TIMESTAMP: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
146 time_t time_val; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
147 time(&time_val); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
148 char* str = ctime(&time_val); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
149 char[] time_str = str[0 .. strlen(str)]; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
150 switch (t.type) |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
151 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
152 case TOK.DATE: |
346
ce9e2e77743f
- Fix: include one space when slicing time_str for __DATE__.
aziz
parents:
344
diff
changeset
|
153 time_str = time_str[4..11] ~ time_str[20..24] ~ \0; break; |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
154 case TOK.TIME: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
155 time_str = time_str[11..19] ~ \0; break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
156 case TOK.TIMESTAMP: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
157 time_str = time_str[0..24] ~ \0; break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
158 default: assert(0); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
159 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
160 t.str = time_str; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
161 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
162 case TOK.VENDOR: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
163 t.str = VENDOR; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
164 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
165 case TOK.VERSION: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
166 t.uint_ = VERSION_MAJOR*1000 + VERSION_MINOR; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
167 break; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
168 default: |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
169 assert(0); |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
170 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
171 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
172 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
173 void setLineBegin(char* p) |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
174 { |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
175 // Check that we can look behind one character. |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
176 assert((p-1) >= text.ptr && p < end); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
177 // Check that previous character is a newline. |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
178 assert(p[-1] == '\n' || p[-1] == '\r' || |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
179 p[-1] == LS[2] || p[-1] == PS[2]); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
180 this.lineBegin = p; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
181 } |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
182 |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
183 private void scanNext(ref Token* t) |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
184 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
185 assert(t !is null); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
186 if (t.next) |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
187 t = t.next; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
188 else if (t != this.tail) |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
189 { |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
190 Token* new_t = new Token; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
191 scan(*new_t); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
192 new_t.prev = t; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
193 t.next = new_t; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
194 t = new_t; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
195 } |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
196 } |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
197 |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
198 void peek(ref Token* t) |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
199 { |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
200 scanNext(t); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
201 } |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
202 |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
203 TOK nextToken() |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
204 { |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
205 scanNext(this.token); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
206 return this.token.type; |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
207 } |
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
208 |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
209 public void scan_(out Token t) |
207 | 210 in |
3 | 211 { |
207 | 212 assert(text.ptr <= p && p < end); |
213 } | |
214 out | |
215 { | |
397
c99f8aeb7b4a
Empty source files are handled correctly now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
396
diff
changeset
|
216 assert(text.ptr <= t.start && t.start < end, Token.toString(t.type)); |
c99f8aeb7b4a
Empty source files are handled correctly now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
396
diff
changeset
|
217 assert(text.ptr <= t.end && t.end <= end, Token.toString(t.type)); |
207 | 218 } |
219 body | |
220 { | |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
221 // Scan whitespace. |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
222 auto pws = p; |
418
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
223 auto old_loc = this.loc; |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
224 while (1) |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
225 { |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
226 switch (*p) |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
227 { |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
228 case '\r': |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
229 if (p[1] == '\n') |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
230 ++p; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
231 case '\n': |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
232 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]); |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
233 ++p; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
234 ++loc; |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
235 setLineBegin(p); |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
236 continue; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
237 case LS[0]: |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
238 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
239 { |
412
fb31af0fda73
Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
411
diff
changeset
|
240 ++p; ++p; |
fb31af0fda73
Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
411
diff
changeset
|
241 goto case '\n'; |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
242 } |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
243 // goto default; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
244 default: |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
245 if (!isspace(*p)) |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
246 break; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
247 ++p; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
248 continue; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
249 } |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
250 break; // Exit loop. |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
251 } |
4 | 252 |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
253 if (p != pws) |
418
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
254 { |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
255 t.ws = pws; |
418
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
256 if (old_loc != this.loc) |
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
257 version(token2LocTable) |
420
ce644d724d87
Made Location a class and extended it with new members.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
419
diff
changeset
|
258 token2LocTable[&t] = new Location(loc, null); |
418
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
259 } |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
260 |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
261 // Scan token. |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
262 uint c = *p; |
4 | 263 { |
5 | 264 t.start = p; |
16 | 265 |
13 | 266 if (isidbeg(c)) |
5 | 267 { |
33 | 268 if (c == 'r' && p[1] == '"' && ++p) |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
269 return scanRawStringLiteral(t); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
270 if (c == 'x' && p[1] == '"') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
271 return scanHexStringLiteral(t); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
272 version(D2) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
273 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
274 if (c == 'q' && p[1] == '"') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
275 return scanDelimitedStringLiteral(t); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
276 if (c == 'q' && p[1] == '{') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
277 return scanTokenStringLiteral(t); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
278 } |
12
0989206cf73c
- Added code to decode Unicode characters in identifiers.
aziz
parents:
11
diff
changeset
|
279 Lidentifier: |
5 | 280 do |
281 { c = *++p; } | |
49 | 282 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8())) |
28 | 283 |
5 | 284 t.end = p; |
28 | 285 |
65 | 286 string str = t.srcText; |
28 | 287 Identifier* id = str in idtable; |
288 | |
289 if (!id) | |
290 { | |
327 | 291 idtable[str] = Identifier(TOK.Identifier, str); |
28 | 292 id = str in idtable; |
293 } | |
294 assert(id); | |
295 t.type = id.type; | |
411 | 296 if (t.type == TOK.Identifier) |
297 return; | |
298 if (t.type == TOK.EOF) | |
299 { | |
300 t.type = TOK.EOF; | |
301 t.end = p; | |
302 tail = &t; | |
303 assert(t.srcText == "__EOF__"); | |
304 } | |
305 else if (t.isSpecialToken) | |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
327
diff
changeset
|
306 finalizeSpecialToken(t); |
5 | 307 return; |
308 } | |
309 | |
15 | 310 if (isdigit(c)) |
311 return scanNumber(t); | |
312 | |
8 | 313 if (c == '/') |
5 | 314 { |
8 | 315 c = *++p; |
14 | 316 switch(c) |
5 | 317 { |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
318 case '=': |
14 | 319 ++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
320 t.type = TOK.DivAssign; |
14 | 321 t.end = p; |
322 return; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
323 case '+': |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
324 return scanNestedComment(t); |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
325 case '*': |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
326 return scanBlockComment(t); |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
327 case '/': |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
328 while (1) |
7 | 329 { |
8 | 330 c = *++p; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
331 switch (c) |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
332 { |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
333 case '\r', '\n', 0, _Z_: |
11 | 334 break; |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
335 case LS[0]: |
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
336 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
337 break; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
338 default: |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
339 if (c & 128) |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
340 decodeUTF8(); |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
341 continue; |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
342 } |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
343 break; // Exit loop. |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
344 } |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
345 t.type = TOK.Comment; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
346 t.end = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
347 return; |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
348 default: |
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
349 t.type = TOK.Div; |
10 | 350 t.end = p; |
351 return; | |
352 } | |
5 | 353 } |
354 | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
355 switch (c) |
20 | 356 { |
48 | 357 case '\'': |
358 return scanCharacterLiteral(t); | |
359 case '`': | |
360 return scanRawStringLiteral(t); | |
361 case '"': | |
362 return scanNormalStringLiteral(t); | |
363 case '\\': | |
364 char[] buffer; | |
365 do | |
366 { | |
367 c = scanEscapeSequence(); | |
368 if (c < 128) | |
369 buffer ~= c; | |
370 else | |
371 encodeUTF8(buffer, c); | |
372 } while (*p == '\\') | |
373 buffer ~= 0; | |
374 t.type = TOK.String; | |
375 t.str = buffer; | |
376 t.end = p; | |
377 return; | |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
378 case '>': /* > >= >> >>= >>> >>>= */ |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
379 c = *++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
380 switch (c) |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
381 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
382 case '=': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
383 t.type = TOK.GreaterEqual; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
384 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
385 case '>': |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
386 if (p[1] == '>') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
387 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
388 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
389 if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
390 { ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
391 t.type = TOK.URShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
392 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
393 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
394 t.type = TOK.URShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
395 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
396 else if (p[1] == '=') |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
397 { |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
398 ++p; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
399 t.type = TOK.RShiftAssign; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
400 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
401 else |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
402 t.type = TOK.RShift; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
403 goto Lcommon; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
404 default: |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
405 t.type = TOK.Greater; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
406 goto Lcommon2; |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
407 } |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
408 assert(0); |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
409 case '<': /* < <= <> <>= << <<= */ |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
410 c = *++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
411 switch (c) |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
412 { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
413 case '=': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
414 t.type = TOK.LessEqual; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
415 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
416 case '<': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
417 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
418 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
419 t.type = TOK.LShiftAssign; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
420 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
421 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
422 t.type = TOK.LShift; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
423 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
424 case '>': |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
425 if (p[1] == '=') { |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
426 ++p; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
427 t.type = TOK.LorEorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
428 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
429 else |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
430 t.type = TOK.LorG; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
431 goto Lcommon; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
432 default: |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
37
diff
changeset
|
433 t.type = TOK.Less; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
434 goto Lcommon2; |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
435 } |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
436 assert(0); |
37
7f3bcb97d017
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
36
diff
changeset
|
437 case '!': /* ! !< !> !<= !>= !<> !<>= */ |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
438 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
439 switch (c) |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
440 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
441 case '<': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
442 c = *++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
443 if (c == '>') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
444 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
445 if (p[1] == '=') { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
446 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
447 t.type = TOK.Unordered; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
448 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
449 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
450 t.type = TOK.UorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
451 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
452 else if (c == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
453 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
454 t.type = TOK.UorG; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
455 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
456 else { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
457 t.type = TOK.UorGorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
458 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
459 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
460 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
461 case '>': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
462 if (p[1] == '=') |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
463 { |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
464 ++p; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
465 t.type = TOK.UorL; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
466 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
467 else |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
468 t.type = TOK.UorLorE; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
469 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
470 case '=': |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
471 t.type = TOK.NotEqual; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
472 goto Lcommon; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
473 default: |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
474 t.type = TOK.Not; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
475 goto Lcommon2; |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
476 } |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
34
diff
changeset
|
477 assert(0); |
63 | 478 case '.': /* . .[0-9] .. ... */ |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
479 if (p[1] == '.') |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
480 { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
481 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
482 if (p[1] == '.') { |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
483 ++p; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
484 t.type = TOK.Ellipses; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
485 } |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
486 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
487 t.type = TOK.Slice; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
488 } |
63 | 489 else if (isdigit(p[1])) |
490 { | |
491 return scanReal(t); | |
492 } | |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
493 else |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
494 t.type = TOK.Dot; |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
495 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
496 case '|': /* | || |= */ |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
497 c = *++p; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
498 if (c == '=') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
499 t.type = TOK.OrAssign; |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
500 else if (c == '|') |
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
501 t.type = TOK.OrLogical; |
26 | 502 else { |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
503 t.type = TOK.OrBinary; |
26 | 504 goto Lcommon2; |
505 } | |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
506 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
507 case '&': /* & && &= */ |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
508 c = *++p; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
509 if (c == '=') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
510 t.type = TOK.AndAssign; |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
511 else if (c == '&') |
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
512 t.type = TOK.AndLogical; |
26 | 513 else { |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
514 t.type = TOK.AndBinary; |
26 | 515 goto Lcommon2; |
516 } | |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
517 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
518 case '+': /* + ++ += */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
519 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
520 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
521 t.type = TOK.PlusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
522 else if (c == '+') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
523 t.type = TOK.PlusPlus; |
26 | 524 else { |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
525 t.type = TOK.Plus; |
26 | 526 goto Lcommon2; |
527 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
528 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
529 case '-': /* - -- -= */ |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
530 c = *++p; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
531 if (c == '=') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
532 t.type = TOK.MinusAssign; |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
533 else if (c == '-') |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
534 t.type = TOK.MinusMinus; |
26 | 535 else { |
536 t.type = TOK.Minus; | |
537 goto Lcommon2; | |
538 } | |
539 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
540 case '=': /* = == */ |
26 | 541 if (p[1] == '=') { |
542 ++p; | |
543 t.type = TOK.Equal; | |
544 } | |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
545 else |
26 | 546 t.type = TOK.Assign; |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
547 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
548 case '~': /* ~ ~= */ |
27 | 549 if (p[1] == '=') { |
550 ++p; | |
551 t.type = TOK.CatAssign; | |
552 } | |
553 else | |
554 t.type = TOK.Tilde; | |
555 goto Lcommon; | |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
556 case '*': /* * *= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
557 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
558 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
559 t.type = TOK.MulAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
560 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
561 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
562 t.type = TOK.Mul; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
563 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
564 case '^': /* ^ ^= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
565 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
566 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
567 t.type = TOK.XorAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
568 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
569 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
570 t.type = TOK.Xor; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
571 goto Lcommon; |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
572 case '%': /* % %= */ |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
573 if (p[1] == '=') { |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
574 ++p; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
575 t.type = TOK.ModAssign; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
576 } |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
577 else |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
578 t.type = TOK.Mod; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
579 goto Lcommon; |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
580 // Single character tokens: |
20 | 581 case '(': |
582 t.type = TOK.LParen; | |
583 goto Lcommon; | |
584 case ')': | |
585 t.type = TOK.RParen; | |
586 goto Lcommon; | |
587 case '[': | |
588 t.type = TOK.LBracket; | |
589 goto Lcommon; | |
590 case ']': | |
591 t.type = TOK.RBracket; | |
592 goto Lcommon; | |
593 case '{': | |
594 t.type = TOK.LBrace; | |
595 goto Lcommon; | |
596 case '}': | |
597 t.type = TOK.RBrace; | |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
598 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
599 case ':': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
600 t.type = TOK.Colon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
601 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
602 case ';': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
603 t.type = TOK.Semicolon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
604 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
605 case '?': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
606 t.type = TOK.Question; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
607 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
608 case ',': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
609 t.type = TOK.Comma; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
610 goto Lcommon; |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
611 case '$': |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
612 t.type = TOK.Dollar; |
20 | 613 Lcommon: |
614 ++p; | |
26 | 615 Lcommon2: |
20 | 616 t.end = p; |
617 return; | |
30 | 618 case '#': |
360
b6a3755eba94
- Renamed scanSpecialToken() to scanSpecialTokenSequence().
aziz
parents:
350
diff
changeset
|
619 return scanSpecialTokenSequence(t); |
20 | 620 default: |
621 } | |
622 | |
411 | 623 // Check for EOF |
624 if (c == 0 || c == _Z_) | |
625 { | |
626 assert(*p == 0 || *p == _Z_); | |
627 t.type = TOK.EOF; | |
628 t.end = p; | |
629 tail = &t; | |
630 assert(t.start == t.end); | |
631 return; | |
632 } | |
633 | |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
634 if (c & 128) |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
635 { |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
636 c = decodeUTF8(); |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
637 if (isUniAlpha(c)) |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
638 goto Lidentifier; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
639 } |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
640 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
641 error(t.start, MID.IllegalCharacter, cast(dchar)c); |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
642 |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
643 ++p; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
644 t.type = TOK.Illegal; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
645 t.dchar_ = c; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
646 t.end = p; |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
389
diff
changeset
|
647 return; |
4 | 648 } |
3 | 649 } |
4 | 650 |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
651 template toUint(char[] T) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
652 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
653 static assert(0 < T.length && T.length <= 4); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
654 static if (T.length == 1) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
655 const uint toUint = T[0]; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
656 else |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
657 const uint toUint = (T[0] << ((T.length-1)*8)) | toUint!(T[1..$]); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
658 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
659 static assert(toUint!("\xAA\xBB\xCC\xDD") == 0xAABBCCDD); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
660 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
661 // Can't use this yet due to a bug in DMD (bug id=1534). |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
662 template case_(char[] str, TOK tok, char[] label) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
663 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
664 const char[] case_ = |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
665 `case `~toUint!(str).stringof~`: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
666 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
667 goto `~label~`;`; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
668 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
669 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
670 template case_L4(char[] str, TOK tok) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
671 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
672 const char[] case_L4 = case_!(str, tok, "Lcommon_4"); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
673 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
674 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
675 template case_L3(char[] str, TOK tok) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
676 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
677 const char[] case_L3 = case_!(str, tok, "Lcommon_3"); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
678 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
679 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
680 template case_L2(char[] str, TOK tok) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
681 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
682 const char[] case_L2 = case_!(str, tok, "Lcommon_2"); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
683 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
684 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
685 template case_L1(char[] str, TOK tok) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
686 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
687 const char[] case_L3 = case_!(str, tok, "Lcommon"); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
688 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
689 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
690 public void scan(out Token t) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
691 in |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
692 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
693 assert(text.ptr <= p && p < end); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
694 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
695 out |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
696 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
697 assert(text.ptr <= t.start && t.start < end, Token.toString(t.type)); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
698 assert(text.ptr <= t.end && t.end <= end, Token.toString(t.type)); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
699 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
700 body |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
701 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
702 // Scan whitespace. |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
703 auto pws = p; |
418
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
704 auto old_loc = this.loc; |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
705 while (1) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
706 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
707 switch (*p) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
708 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
709 case '\r': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
710 if (p[1] == '\n') |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
711 ++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
712 case '\n': |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
713 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]); |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
714 ++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
715 ++loc; |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
716 setLineBegin(p); |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
717 continue; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
718 case LS[0]: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
719 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
720 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
721 ++p; ++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
722 goto case '\n'; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
723 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
724 // goto default; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
725 default: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
726 if (!isspace(*p)) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
727 break; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
728 ++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
729 continue; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
730 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
731 break; // Exit loop. |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
732 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
733 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
734 if (p != pws) |
418
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
735 { |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
736 t.ws = pws; |
418
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
737 if (old_loc != this.loc) |
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
738 version(token2LocTable) |
420
ce644d724d87
Made Location a class and extended it with new members.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
419
diff
changeset
|
739 token2LocTable[&t] = new Location(loc, null); |
418
7354f15cd5e9
Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
416
diff
changeset
|
740 } |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
741 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
742 // Scan token. |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
743 t.start = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
744 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
745 uint c = *p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
746 assert(end - p != 0); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
747 switch (end - p) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
748 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
749 case 1: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
750 goto L1character; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
751 case 2: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
752 c <<= 8; c |= p[1]; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
753 goto L2characters; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
754 case 3: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
755 c <<= 8; c |= p[1]; c <<= 8; c |= p[2]; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
756 goto L3characters; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
757 default: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
758 version(BigEndian) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
759 c = *cast(uint*)p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
760 else |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
761 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
762 c <<= 8; c |= p[1]; c <<= 8; c |= p[2]; c <<= 8; c |= p[3]; |
414
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
763 /+ |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
764 c = *cast(uint*)p; |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
765 asm |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
766 { |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
767 mov EDX, c; |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
768 bswap EDX; |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
769 mov c, EDX; |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
770 } |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
771 +/ |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
772 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
773 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
774 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
775 // 4 character tokens. |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
776 switch (c) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
777 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
778 case toUint!(">>>="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
779 t.type = TOK.RShiftAssign; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
780 goto Lcommon_4; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
781 case toUint!("!<>="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
782 t.type = TOK.Unordered; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
783 Lcommon_4: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
784 p += 4; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
785 t.end = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
786 return; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
787 default: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
788 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
789 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
790 c >>>= 8; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
791 L3characters: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
792 assert(p == t.start); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
793 // 3 character tokens. |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
794 switch (c) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
795 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
796 case toUint!(">>="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
797 t.type = TOK.RShiftAssign; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
798 goto Lcommon_3; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
799 case toUint!(">>>"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
800 t.type = TOK.URShift; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
801 goto Lcommon_3; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
802 case toUint!("<>="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
803 t.type = TOK.LorEorG; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
804 goto Lcommon_3; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
805 case toUint!("<<="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
806 t.type = TOK.LShiftAssign; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
807 goto Lcommon_3; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
808 case toUint!("!<="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
809 t.type = TOK.UorG; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
810 goto Lcommon_3; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
811 case toUint!("!>="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
812 t.type = TOK.UorL; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
813 goto Lcommon_3; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
814 case toUint!("!<>"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
815 t.type = TOK.UorE; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
816 goto Lcommon_3; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
817 case toUint!("..."): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
818 t.type = TOK.Ellipses; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
819 Lcommon_3: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
820 p += 3; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
821 t.end = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
822 return; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
823 default: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
824 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
825 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
826 c >>>= 8; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
827 L2characters: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
828 assert(p == t.start); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
829 // 2 character tokens. |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
830 switch (c) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
831 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
832 case toUint!("/+"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
833 ++p; // Skip / |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
834 return scanNestedComment(t); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
835 case toUint!("/*"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
836 ++p; // Skip / |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
837 return scanBlockComment(t); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
838 case toUint!("//"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
839 ++p; // Skip / |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
840 assert(*p == '/'); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
841 while (1) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
842 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
843 c = *++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
844 switch (c) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
845 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
846 case '\r', '\n', 0, _Z_: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
847 break; |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
848 case LS[0]: |
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
849 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
850 break; |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
851 default: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
852 if (c & 128) |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
853 decodeUTF8(); |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
854 continue; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
855 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
856 break; // Exit loop. |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
857 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
858 t.type = TOK.Comment; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
859 t.end = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
860 return; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
861 case toUint!(">="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
862 t.type = TOK.GreaterEqual; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
863 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
864 case toUint!(">>"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
865 t.type = TOK.RShift; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
866 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
867 case toUint!("<<"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
868 t.type = TOK.LShift; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
869 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
870 case toUint!("<="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
871 t.type = TOK.LessEqual; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
872 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
873 case toUint!("<>"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
874 t.type = TOK.LorG; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
875 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
876 case toUint!("!<"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
877 t.type = TOK.UorGorE; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
878 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
879 case toUint!("!>"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
880 t.type = TOK.UorLorE; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
881 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
882 case toUint!("!="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
883 t.type = TOK.NotEqual; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
884 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
885 case toUint!(".."): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
886 t.type = TOK.Slice; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
887 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
888 case toUint!("&&"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
889 t.type = TOK.AndLogical; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
890 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
891 case toUint!("&="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
892 t.type = TOK.AndAssign; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
893 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
894 case toUint!("||"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
895 t.type = TOK.OrLogical; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
896 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
897 case toUint!("|="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
898 t.type = TOK.OrAssign; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
899 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
900 case toUint!("++"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
901 t.type = TOK.PlusPlus; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
902 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
903 case toUint!("+="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
904 t.type = TOK.PlusAssign; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
905 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
906 case toUint!("--"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
907 t.type = TOK.MinusMinus; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
908 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
909 case toUint!("-="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
910 t.type = TOK.MinusAssign; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
911 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
912 case toUint!("=="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
913 t.type = TOK.Equal; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
914 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
915 case toUint!("~="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
916 t.type = TOK.CatAssign; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
917 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
918 case toUint!("*="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
919 t.type = TOK.MulAssign; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
920 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
921 case toUint!("/="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
922 t.type = TOK.DivAssign; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
923 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
924 case toUint!("^="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
925 t.type = TOK.XorAssign; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
926 goto Lcommon_2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
927 case toUint!("%="): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
928 t.type = TOK.ModAssign; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
929 Lcommon_2: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
930 p += 2; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
931 t.end = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
932 return; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
933 default: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
934 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
935 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
936 c >>>= 8; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
937 L1character: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
938 assert(p == t.start); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
939 assert(*p == c, Format("p={0},c={1}", *p, cast(dchar)c)); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
940 // 1 character tokens. |
414
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
941 // TODO: consider storing the token type in ptable. |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
942 switch (c) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
943 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
944 case '\'': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
945 return scanCharacterLiteral(t); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
946 case '`': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
947 return scanRawStringLiteral(t); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
948 case '"': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
949 return scanNormalStringLiteral(t); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
950 case '\\': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
951 char[] buffer; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
952 do |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
953 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
954 c = scanEscapeSequence(); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
955 if (c < 128) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
956 buffer ~= c; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
957 else |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
958 encodeUTF8(buffer, c); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
959 } while (*p == '\\') |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
960 buffer ~= 0; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
961 t.type = TOK.String; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
962 t.str = buffer; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
963 t.end = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
964 return; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
965 case '<': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
966 t.type = TOK.Greater; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
967 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
968 case '>': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
969 t.type = TOK.Less; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
970 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
971 case '^': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
972 t.type = TOK.Xor; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
973 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
974 case '!': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
975 t.type = TOK.Not; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
976 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
977 case '.': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
978 if (isdigit(p[1])) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
979 return scanReal(t); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
980 t.type = TOK.Dot; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
981 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
982 case '&': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
983 t.type = TOK.AndBinary; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
984 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
985 case '|': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
986 t.type = TOK.OrBinary; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
987 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
988 case '+': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
989 t.type = TOK.Plus; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
990 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
991 case '-': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
992 t.type = TOK.Minus; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
993 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
994 case '=': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
995 t.type = TOK.Assign; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
996 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
997 case '~': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
998 t.type = TOK.Tilde; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
999 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1000 case '*': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1001 t.type = TOK.Mul; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1002 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1003 case '/': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1004 t.type = TOK.Div; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1005 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1006 case '%': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1007 t.type = TOK.Mod; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1008 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1009 case '(': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1010 t.type = TOK.LParen; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1011 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1012 case ')': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1013 t.type = TOK.RParen; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1014 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1015 case '[': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1016 t.type = TOK.LBracket; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1017 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1018 case ']': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1019 t.type = TOK.RBracket; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1020 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1021 case '{': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1022 t.type = TOK.LBrace; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1023 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1024 case '}': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1025 t.type = TOK.RBrace; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1026 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1027 case ':': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1028 t.type = TOK.Colon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1029 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1030 case ';': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1031 t.type = TOK.Semicolon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1032 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1033 case '?': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1034 t.type = TOK.Question; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1035 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1036 case ',': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1037 t.type = TOK.Comma; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1038 goto Lcommon; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1039 case '$': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1040 t.type = TOK.Dollar; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1041 Lcommon: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1042 ++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1043 t.end = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1044 return; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1045 case '#': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1046 return scanSpecialTokenSequence(t); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1047 default: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1048 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1049 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1050 assert(p == t.start); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1051 assert(*p == c); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1052 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1053 // TODO: consider moving isidbeg() and isdigit() up. |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1054 if (isidbeg(c)) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1055 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1056 if (c == 'r' && p[1] == '"' && ++p) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1057 return scanRawStringLiteral(t); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1058 if (c == 'x' && p[1] == '"') |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1059 return scanHexStringLiteral(t); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1060 version(D2) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1061 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1062 if (c == 'q' && p[1] == '"') |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1063 return scanDelimitedStringLiteral(t); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1064 if (c == 'q' && p[1] == '{') |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1065 return scanTokenStringLiteral(t); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1066 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1067 Lidentifier: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1068 do |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1069 { c = *++p; } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1070 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8())) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1071 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1072 t.end = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1073 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1074 string str = t.srcText; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1075 Identifier* id = str in idtable; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1076 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1077 if (!id) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1078 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1079 idtable[str] = Identifier(TOK.Identifier, str); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1080 id = str in idtable; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1081 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1082 assert(id); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1083 t.type = id.type; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1084 if (t.type == TOK.Identifier) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1085 return; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1086 if (t.type == TOK.EOF) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1087 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1088 t.type = TOK.EOF; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1089 t.end = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1090 tail = &t; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1091 assert(t.srcText == "__EOF__"); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1092 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1093 else if (t.isSpecialToken) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1094 finalizeSpecialToken(t); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1095 return; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1096 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1097 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1098 if (isdigit(c)) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1099 return scanNumber(t); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1100 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1101 // Check for EOF |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1102 if (c == 0 || c == _Z_) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1103 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1104 assert(*p == 0 || *p == _Z_, *p~""); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1105 t.type = TOK.EOF; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1106 t.end = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1107 tail = &t; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1108 assert(t.start == t.end); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1109 return; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1110 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1111 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1112 if (c & 128) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1113 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1114 c = decodeUTF8(); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1115 if (isUniAlpha(c)) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1116 goto Lidentifier; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1117 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1118 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1119 error(t.start, MID.IllegalCharacter, cast(dchar)c); |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1120 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1121 ++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1122 t.type = TOK.Illegal; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1123 t.dchar_ = c; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1124 t.end = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1125 return; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1126 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1127 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1128 void scanBlockComment(ref Token t) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1129 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1130 assert(p[-1] == '/' && *p == '*'); |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1131 auto tokenLineNum = loc; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1132 auto tokenLineBegin = lineBegin; |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1133 uint c; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1134 while (1) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1135 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1136 c = *++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1137 LswitchBC: // only jumped to from default case of next switch(c) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1138 switch (c) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1139 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1140 case '\r': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1141 if (p[1] == '\n') |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1142 ++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1143 case '\n': |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1144 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]); |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1145 ++loc; |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1146 setLineBegin(p+1); |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1147 continue; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1148 case 0, _Z_: |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1149 error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedBlockComment); |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1150 goto LreturnBC; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1151 default: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1152 if (c & 128) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1153 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1154 c = decodeUTF8(); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1155 if (c == LSd || c == PSd) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1156 goto case '\n'; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1157 continue; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1158 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1159 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1160 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1161 c <<= 8; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1162 c |= *++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1163 switch (c) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1164 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1165 case toUint!("*/"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1166 ++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1167 LreturnBC: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1168 t.type = TOK.Comment; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1169 t.end = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1170 return; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1171 default: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1172 c &= char.max; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1173 goto LswitchBC; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1174 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1175 } |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1176 assert(0); |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1177 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1178 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1179 void scanNestedComment(ref Token t) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1180 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1181 assert(p[-1] == '/' && *p == '+'); |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1182 auto tokenLineNum = loc; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1183 auto tokenLineBegin = lineBegin; |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1184 uint level = 1; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1185 uint c; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1186 while (1) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1187 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1188 c = *++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1189 LswitchNC: // only jumped to from default case of next switch(c) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1190 switch (c) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1191 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1192 case '\r': |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1193 if (p[1] == '\n') |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1194 ++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1195 case '\n': |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1196 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]); |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1197 ++loc; |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1198 setLineBegin(p+1); |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1199 continue; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1200 case 0, _Z_: |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1201 error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedNestedComment); |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1202 goto LreturnNC; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1203 default: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1204 if (c & 128) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1205 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1206 c = decodeUTF8(); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1207 if (c == LSd || c == PSd) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1208 goto case '\n'; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1209 continue; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1210 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1211 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1212 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1213 c <<= 8; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1214 c |= *++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1215 switch (c) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1216 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1217 case toUint!("/+"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1218 ++level; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1219 continue; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1220 case toUint!("+/"): |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1221 if (--level == 0) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1222 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1223 ++p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1224 LreturnNC: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1225 t.type = TOK.Comment; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1226 t.end = p; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1227 return; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1228 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1229 continue; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1230 default: |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1231 c &= char.max; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1232 goto LswitchNC; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1233 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1234 } |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1235 assert(0); |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1236 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
1237 |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1238 void scanNormalStringLiteral(ref Token t) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1239 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1240 assert(*p == '"'); |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1241 auto tokenLineNum = loc; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1242 auto tokenLineBegin = lineBegin; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1243 char[] buffer; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1244 t.type = TOK.String; |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
1245 uint c; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1246 while (1) |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1247 { |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
1248 c = *++p; |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
1249 switch (c) |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1250 { |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1251 case '"': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1252 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1253 Lreturn: |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1254 buffer ~= 0; |
49 | 1255 t.str = buffer; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1256 t.pf = scanPostfix(); |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1257 t.end = p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1258 return; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1259 case '\\': |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
1260 c = scanEscapeSequence(); |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
1261 --p; |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
1262 if (c < 128) |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
1263 break; |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
1264 encodeUTF8(buffer, c); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1265 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1266 case '\r': |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1267 if (p[1] == '\n') |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1268 ++p; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1269 case '\n': |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1270 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1271 ++loc; |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
1272 c = '\n'; // Convert EndOfLine to \n. |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1273 setLineBegin(p+1); |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
1274 break; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1275 case 0, _Z_: |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1276 error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedString); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1277 goto Lreturn; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1278 default: |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
1279 if (c & 128) |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1280 { |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
1281 c = decodeUTF8(); |
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
1282 if (c == LSd || c == PSd) |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1283 goto case '\n'; |
49 | 1284 |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
1285 encodeUTF8(buffer, c); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1286 continue; |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1287 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1288 } |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
1289 assert(isascii(c)); |
384
f7ce725e79c3
Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
1290 buffer ~= c; |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1291 } |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
1292 assert(0); |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1293 } |
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
1294 |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1295 void scanCharacterLiteral(ref Token t) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1296 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1297 assert(*p == '\''); |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1298 MID id = MID.UnterminatedCharacterLiteral; |
46 | 1299 ++p; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1300 TOK type = TOK.CharLiteral; |
46 | 1301 switch (*p) |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1302 { |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1303 case '\\': |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1304 switch (p[1]) |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1305 { |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1306 case 'u': |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1307 type = TOK.WCharLiteral; break; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1308 case 'U': |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1309 type = TOK.DCharLiteral; break; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1310 default: |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1311 } |
46 | 1312 t.dchar_ = scanEscapeSequence(); |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1313 break; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1314 case '\'': |
46 | 1315 ++p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1316 id = MID.EmptyCharacterLiteral; |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1317 // fall through |
46 | 1318 case '\n', '\r', 0, _Z_: |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1319 goto Lerr; |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1320 case LS[0]: |
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1321 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1322 goto Lerr; |
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1323 // fall through |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1324 default: |
46 | 1325 uint c = *p; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1326 if (c & 128) |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1327 { |
49 | 1328 c = decodeUTF8(); |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1329 if (c <= 0xFFFF) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1330 type = TOK.WCharLiteral; |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1331 else |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1332 type = TOK.DCharLiteral; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1333 } |
46 | 1334 t.dchar_ = c; |
1335 ++p; | |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1336 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1337 |
46 | 1338 if (*p == '\'') |
1339 ++p; | |
1340 else | |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1341 Lerr: |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1342 error(t.start, id); |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1343 t.type = type; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1344 t.end = p; |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1345 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
1346 |
33 | 1347 char scanPostfix() |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1348 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1349 switch (*p) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1350 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1351 case 'c': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1352 case 'w': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1353 case 'd': |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1354 return *p++; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1355 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1356 return 0; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1357 } |
112 | 1358 assert(0); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1359 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1360 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1361 void scanRawStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1362 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1363 auto tokenLineNum = loc; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1364 auto tokenLineBegin = lineBegin; |
33 | 1365 uint delim = *p; |
1366 assert(delim == '`' || delim == '"' && p[-1] == 'r'); | |
1367 t.type = TOK.String; | |
1368 char[] buffer; | |
1369 uint c; | |
1370 while (1) | |
1371 { | |
1372 c = *++p; | |
1373 switch (c) | |
1374 { | |
1375 case '\r': | |
1376 if (p[1] == '\n') | |
1377 ++p; | |
1378 case '\n': | |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1379 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1380 c = '\n'; // Convert EndOfLine ('\r','\r\n','\n',LS,PS) to '\n' |
33 | 1381 ++loc; |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1382 setLineBegin(p+1); |
52
f65a83c27638
- Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents:
51
diff
changeset
|
1383 break; |
33 | 1384 case '`': |
1385 case '"': | |
1386 if (c == delim) | |
1387 { | |
1388 ++p; | |
1389 t.pf = scanPostfix(); | |
1390 Lreturn: | |
1391 t.str = buffer ~ '\0'; | |
1392 t.end = p; | |
1393 return; | |
1394 } | |
1395 break; | |
1396 case 0, _Z_: | |
1397 if (delim == 'r') | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1398 error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedRawString); |
33 | 1399 else |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1400 error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedBackQuoteString); |
33 | 1401 goto Lreturn; |
1402 default: | |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1403 if (c & 128) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1404 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1405 c = decodeUTF8(); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1406 if (c == LSd || c == PSd) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1407 goto case '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1408 encodeUTF8(buffer, c); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1409 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1410 } |
33 | 1411 } |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
1412 assert(isascii(c)); |
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
1413 buffer ~= c; |
33 | 1414 } |
1415 assert(0); | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1416 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1417 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1418 void scanHexStringLiteral(ref Token t) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1419 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1420 assert(p[0] == 'x' && p[1] == '"'); |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1421 t.type = TOK.String; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1422 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1423 auto tokenLineNum = loc; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1424 auto tokenLineBegin = lineBegin; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1425 |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1426 uint c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1427 ubyte[] buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1428 ubyte h; // hex number |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1429 uint n; // number of hex digits |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1430 |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1431 ++p; |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1432 assert(*p == '"'); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1433 while (1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1434 { |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1435 c = *++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1436 switch (c) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1437 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1438 case '"': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1439 ++p; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1440 if (n & 1) |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1441 error(tokenLineNum, tokenLineBegin, t.start, MID.OddNumberOfDigitsInHexString); |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1442 t.pf = scanPostfix(); |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1443 Lreturn: |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1444 buffer ~= 0; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1445 t.str = cast(string) buffer; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1446 t.end = p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1447 return; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1448 case '\r': |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1449 if (p[1] == '\n') |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1450 ++p; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1451 case '\n': |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1452 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1453 ++loc; |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1454 setLineBegin(p+1); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1455 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1456 default: |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1457 if (ishexad(c)) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1458 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1459 if (c <= '9') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1460 c -= '0'; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1461 else if (c <= 'F') |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1462 c -= 'A' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1463 else |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1464 c -= 'a' - 10; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1465 |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1466 if (n & 1) |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1467 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1468 h <<= 4; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1469 h |= c; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1470 buffer ~= h; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1471 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1472 else |
112 | 1473 h = cast(ubyte)c; |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1474 ++n; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1475 continue; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1476 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1477 else if (isspace(c)) |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1478 continue; // Skip spaces. |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1479 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1480 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1481 ++p; ++p; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1482 goto case '\n'; |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1483 } |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1484 else if (c == 0 || c == _Z_) |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1485 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1486 error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedHexString); |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1487 t.pf = 0; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1488 goto Lreturn; |
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1489 } |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1490 else |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1491 { |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1492 auto errorAt = p; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1493 if (c & 128) |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1494 c = decodeUTF8(); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1495 error(errorAt, MID.NonHexCharInHexString, cast(dchar)c); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1496 } |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1497 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1498 } |
53
1786c2825491
- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents:
52
diff
changeset
|
1499 assert(0); |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1500 } |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
1501 |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1502 version(D2) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1503 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1504 void scanDelimitedStringLiteral(ref Token t) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1505 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1506 assert(p[0] == 'q' && p[1] == '"'); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1507 t.type = TOK.String; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1508 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1509 auto tokenLineNum = loc; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1510 auto tokenLineBegin = lineBegin; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1511 |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1512 char[] buffer; |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1513 dchar opening_delim = 0, // 0 if no nested delimiter or '[', '(', '<', '{' |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1514 closing_delim; // Will be ']', ')', '>', '}, |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1515 // the first character of an identifier or |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1516 // any other Unicode/ASCII character. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1517 char[] str_delim; // Identifier delimiter. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1518 uint level = 1; // Counter for nestable delimiters. |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1519 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1520 ++p; ++p; // Skip q" |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1521 uint c = *p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1522 switch (c) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1523 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1524 case '(': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1525 opening_delim = c; |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1526 closing_delim = ')'; // c + 1 |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1527 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1528 case '[', '<', '{': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1529 opening_delim = c; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1530 closing_delim = c + 2; // Get to closing counterpart. Feature of ASCII table. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1531 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1532 default: |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1533 dchar scanNewline() |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1534 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1535 switch (*p) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1536 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1537 case '\r': |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1538 if (p[1] == '\n') |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1539 ++p; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1540 case '\n': |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1541 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]); |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1542 ++p; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1543 ++loc; |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1544 setLineBegin(p); |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1545 return '\n'; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1546 case LS[0]: |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1547 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1548 { |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1549 ++p; ++p; |
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1550 goto case '\n'; |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1551 } |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1552 default: |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1553 } |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1554 return 0; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1555 } |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1556 |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1557 // Skip leading newlines: |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1558 while (scanNewline() != 0){} |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1559 assert(*p != '\n' && *p != '\r' && |
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1560 !(*p == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))); |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1561 |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1562 char* begin = p; |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1563 c = *p; |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1564 closing_delim = c; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1565 // TODO: Check for non-printable characters? |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1566 if (c & 128) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1567 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1568 closing_delim = decodeUTF8(); |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1569 if (!isUniAlpha(closing_delim)) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1570 break; // Not an identifier. |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1571 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1572 else if (!isidbeg(c)) |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1573 break; // Not an identifier. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1574 |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1575 // Parse Identifier + EndOfLine |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1576 do |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1577 { c = *++p; } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1578 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8())) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1579 // Store identifier |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1580 str_delim = begin[0..p-begin]; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1581 // Scan newline |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1582 if (scanNewline() == '\n') |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1583 --p; // Go back one because of "c = *++p;" in main loop. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1584 else |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1585 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1586 // TODO: error(p, MID.ExpectedNewlineAfterIdentDelim); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1587 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1588 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1589 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1590 bool checkStringDelim(char* p) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1591 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1592 assert(str_delim.length != 0); |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1593 if (buffer[$-1] == '\n' && // Last character copied to buffer must be '\n'. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1594 end-p >= str_delim.length && // Check remaining length. |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1595 p[0..str_delim.length] == str_delim) // Compare. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1596 return true; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1597 return false; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1598 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1599 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1600 while (1) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1601 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1602 c = *++p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1603 switch (c) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1604 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1605 case '\r': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1606 if (p[1] == '\n') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1607 ++p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1608 case '\n': |
419
89e40d43065d
Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
418
diff
changeset
|
1609 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1610 c = '\n'; // Convert EndOfLine ('\r','\r\n','\n',LS,PS) to '\n' |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1611 ++loc; |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1612 setLineBegin(p+1); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1613 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1614 case 0, _Z_: |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1615 // TODO: error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedDelimitedString); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1616 goto Lreturn3; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1617 default: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1618 if (c & 128) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1619 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1620 auto begin = p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1621 c = decodeUTF8(); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1622 if (c == LSd || c == PSd) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1623 goto case '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1624 if (c == closing_delim) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1625 { |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1626 if (str_delim.length) |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1627 { |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1628 if (checkStringDelim(begin)) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1629 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1630 p = begin + str_delim.length; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1631 goto Lreturn2; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1632 } |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1633 } |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1634 else |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1635 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1636 assert(level == 1); |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1637 --level; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1638 goto Lreturn; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1639 } |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1640 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1641 encodeUTF8(buffer, c); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1642 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1643 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1644 else |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1645 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1646 if (c == opening_delim) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1647 ++level; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1648 else if (c == closing_delim) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1649 { |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1650 if (str_delim.length) |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1651 { |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1652 if (checkStringDelim(p)) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1653 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1654 p += str_delim.length; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1655 goto Lreturn2; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1656 } |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1657 } |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1658 else if (--level == 0) |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1659 goto Lreturn; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1660 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1661 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1662 } |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
1663 assert(isascii(c)); |
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
1664 buffer ~= c; |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1665 } |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1666 Lreturn: // Character delimiter. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1667 assert(c == closing_delim); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1668 assert(level == 0); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1669 ++p; // Skip closing delimiter. |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1670 Lreturn2: // String delimiter. |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1671 if (*p == '"') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1672 ++p; |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1673 else |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1674 { |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1675 // TODO: error(p, MID.ExpectedDblQuoteAfterDelim, str_delim.length ? str_delim : closing_delim~""); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1676 } |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1677 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1678 t.pf = scanPostfix(); |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1679 Lreturn3: // Error. |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1680 t.str = buffer ~ '\0'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1681 t.end = p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1682 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1683 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1684 void scanTokenStringLiteral(ref Token t) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1685 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1686 assert(p[0] == 'q' && p[1] == '{'); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1687 t.type = TOK.String; |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1688 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1689 auto tokenLineNum = loc; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1690 auto tokenLineBegin = lineBegin; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1691 |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1692 // A guard against changes to particular members: |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1693 // this.loc_old, this.loc_hline and this.errorLoc.filePath |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1694 ++inTokenString; |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1695 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1696 uint loc = this.loc; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1697 uint level = 1; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1698 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1699 ++p; ++p; // Skip q{ |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1700 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1701 auto prev_t = &t; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1702 Token* token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1703 while (1) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1704 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1705 token = new Token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1706 scan(*token); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1707 // Save the tokens in a doubly linked list. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1708 // Could be useful for various tools. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1709 token.prev = prev_t; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1710 prev_t.next = token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1711 prev_t = token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1712 switch (token.type) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1713 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1714 case TOK.LBrace: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1715 ++level; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1716 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1717 case TOK.RBrace: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1718 if (--level == 0) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1719 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1720 t.tok_str = t.next; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1721 t.next = null; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1722 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1723 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1724 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1725 case TOK.EOF: |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1726 // TODO: error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedTokenString); |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1727 t.tok_str = t.next; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1728 t.next = token; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1729 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1730 default: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1731 continue; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1732 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1733 break; // Exit loop. |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1734 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1735 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1736 assert(token.type == TOK.RBrace || token.type == TOK.EOF); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1737 assert(token.type == TOK.RBrace && t.next is null || |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1738 token.type == TOK.EOF && t.next !is null); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1739 |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1740 char[] buffer; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1741 // token points to } or EOF |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1742 if (token.type == TOK.EOF) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1743 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1744 t.end = token.start; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1745 buffer = t.srcText[2..$].dup ~ '\0'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1746 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1747 else |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1748 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1749 // Assign to buffer before scanPostfix(). |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1750 t.end = p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1751 buffer = t.srcText[2..$-1].dup ~ '\0'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1752 t.pf = scanPostfix(); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1753 t.end = p; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1754 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1755 // Convert EndOfLines to '\n' |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1756 if (loc != this.loc) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1757 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1758 assert(buffer[$-1] == '\0'); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1759 uint i, j; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1760 for (; i < buffer.length; ++i) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1761 switch (buffer[i]) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1762 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1763 case '\r': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1764 if (buffer[i+1] == '\n') |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1765 ++i; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1766 case '\n': |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1767 buffer[j++] = '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1768 break; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1769 case LS[0]: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1770 auto b = buffer[i..$]; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1771 if (b[1] == LS[1] && (b[2] == LS[2] || b[2] == PS[2])) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1772 { |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1773 ++i; ++i; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1774 goto case '\n'; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1775 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1776 // goto default; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1777 default: |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1778 buffer[j++] = buffer[i]; // Copy character |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1779 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1780 buffer.length = j; // Adjust length |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1781 } |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1782 assert(buffer[$-1] == '\0'); |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1783 t.str = buffer; |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1784 |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1785 --inTokenString; |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1786 } |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1787 } // version(D2) |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
382
diff
changeset
|
1788 |
45 | 1789 dchar scanEscapeSequence() |
433
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1790 out(result) |
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1791 { assert(isEncodable(result)); } |
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1792 body |
45 | 1793 { |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1794 assert(*p == '\\'); |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1795 |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1796 auto sequenceStart = p; // Used for error reporting. |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1797 |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1798 ++p; |
45 | 1799 uint c = char2ev(*p); |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1800 if (c) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1801 { |
46 | 1802 ++p; |
45 | 1803 return c; |
46 | 1804 } |
433
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1805 |
45 | 1806 uint digits = 2; |
1807 | |
1808 switch (*p) | |
1809 { | |
1810 case 'x': | |
433
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1811 assert(c == 0); |
45 | 1812 while (1) |
1813 { | |
1814 ++p; | |
1815 if (ishexad(*p)) | |
1816 { | |
1817 c *= 16; | |
1818 if (*p <= '9') | |
48 | 1819 c += *p - '0'; |
45 | 1820 else if (*p <= 'F') |
48 | 1821 c += *p - 'A' + 10; |
45 | 1822 else |
48 | 1823 c += *p - 'a' + 10; |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1824 |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1825 if (!--digits) |
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
1826 { |
48 | 1827 ++p; |
433
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1828 if (isEncodable(c)) |
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1829 return c; // Return valid escape value. |
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1830 |
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1831 error(sequenceStart, MID.InvalidUnicodeEscapeSequence, sequenceStart[0..p-sequenceStart]); |
45 | 1832 break; |
48 | 1833 } |
433
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1834 continue; |
45 | 1835 } |
432
3ead178e0662
Added message MID.InvalidUnicodeEscapeSequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
430
diff
changeset
|
1836 |
433
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1837 error(sequenceStart, MID.InsufficientHexDigits); |
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1838 break; |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
1839 } |
433
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1840 break; |
45 | 1841 case 'u': |
1842 digits = 4; | |
1843 goto case 'x'; | |
1844 case 'U': | |
1845 digits = 8; | |
1846 goto case 'x'; | |
1847 default: | |
48 | 1848 if (isoctal(*p)) |
45 | 1849 { |
433
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1850 assert(c == 0); |
48 | 1851 c += *p - '0'; |
1852 ++p; | |
1853 if (!isoctal(*p)) | |
1854 return c; | |
1855 c *= 8; | |
1856 c += *p - '0'; | |
1857 ++p; | |
1858 if (!isoctal(*p)) | |
1859 return c; | |
1860 c *= 8; | |
1861 c += *p - '0'; | |
1862 ++p; | |
433
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1863 return c; // Return valid escape value. |
48 | 1864 } |
1865 else if(*p == '&') | |
1866 { | |
1867 if (isalpha(*++p)) | |
45 | 1868 { |
272
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1869 auto begin = p; |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1870 while (isalnum(*++p)) |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1871 {} |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1872 |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1873 if (*p == ';') |
48 | 1874 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1875 // Pass entity excluding '&' and ';'. |
272
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1876 c = entity2Unicode(begin[0..p - begin]); |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1877 ++p; // Skip ; |
433
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1878 if (c != 0xFFFF) |
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1879 return c; // Return valid escape value. |
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1880 else |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1881 error(sequenceStart, MID.UndefinedHTMLEntity, sequenceStart[0 .. p - sequenceStart]); |
272
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1882 } |
0bde32503976
- Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents:
249
diff
changeset
|
1883 else |
432
3ead178e0662
Added message MID.InvalidUnicodeEscapeSequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
430
diff
changeset
|
1884 error(sequenceStart, MID.UnterminatedHTMLEntity, sequenceStart[0 .. p - sequenceStart]); |
45 | 1885 } |
48 | 1886 else |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1887 error(sequenceStart, MID.InvalidBeginHTMLEntity); |
45 | 1888 } |
416
a4783b904fba
Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
414
diff
changeset
|
1889 else if (*p == '\n' || *p == '\r' || |
a4783b904fba
Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
414
diff
changeset
|
1890 *p == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
a4783b904fba
Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
414
diff
changeset
|
1891 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1892 error(sequenceStart, MID.UndefinedEscapeSequence, r"\NewLine"); |
416
a4783b904fba
Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
414
diff
changeset
|
1893 } |
a4783b904fba
Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
414
diff
changeset
|
1894 else if (*p == 0 || *p == _Z_) |
a4783b904fba
Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
414
diff
changeset
|
1895 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1896 error(sequenceStart, MID.UndefinedEscapeSequence, r"\EOF"); |
416
a4783b904fba
Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
414
diff
changeset
|
1897 } |
45 | 1898 else |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1899 { |
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1900 char[] str = `\`; |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
1901 if (*p & 128) |
387
ad0cbd1c8881
Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
386
diff
changeset
|
1902 encodeUTF8(str, decodeUTF8()); |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1903 else |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
1904 str ~= *p; |
387
ad0cbd1c8881
Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
386
diff
changeset
|
1905 ++p; |
ad0cbd1c8881
Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
386
diff
changeset
|
1906 // TODO: check for unprintable character? |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
1907 error(sequenceStart, MID.UndefinedEscapeSequence, str); |
386
392a0068fc61
Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
384
diff
changeset
|
1908 } |
45 | 1909 } |
433
063cd016c913
Fixed some issues in scanEscapeSequence().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
432
diff
changeset
|
1910 return REPLACEMENT_CHAR; // Error: return replacement character. |
45 | 1911 } |
1912 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1913 /* |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1914 IntegerLiteral:= (Dec|Hex|Bin|Oct)Suffix? |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1915 Dec:= (0|[1-9][0-9_]*) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1916 Hex:= 0[xX] HexDigits |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1917 Bin:= 0[bB][01_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1918 Oct:= 0[0-7_]+ |
68
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
1919 Suffix:= (L[uU]?|[uU]L?) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1920 HexDigits:= [0-9a-zA-Z_]+ |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1921 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1922 Invalid: "0b_", "0x_", "._" |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1923 */ |
15 | 1924 void scanNumber(ref Token t) |
1925 { | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1926 ulong ulong_; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1927 bool overflow; |
58 | 1928 bool isDecimal; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
1929 size_t digits; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1930 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1931 if (*p != '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1932 goto LscanInteger; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1933 ++p; // skip zero |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1934 // check for xX bB ... |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1935 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1936 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1937 case 'x','X': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1938 goto LscanHex; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1939 case 'b','B': |
396
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1940 goto LscanBinary; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1941 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1942 if (p[1] == 'i') |
396
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1943 goto LscanReal; // 0Li |
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1944 break; // 0L |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1945 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1946 if (p[1] == '.') |
396
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1947 break; // 0.. |
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1948 // 0. |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1949 case 'i','f','F', // Imaginary and float literal suffixes. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
1950 'e', 'E': // Float exponent. |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1951 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1952 default: |
396
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1953 if (*p == '_') |
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1954 goto LscanOctal; // 0_ |
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1955 else if (isdigit(*p)) |
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1956 { |
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1957 if (*p == '8' || *p == '9') |
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1958 goto Loctal_hasDecimalDigits; // 08 or 09 |
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1959 else |
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1960 goto Loctal_enter_loop; // 0[0-7] |
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1961 } |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1962 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1963 |
323 | 1964 // Number 0 |
1965 assert(p[-1] == '0'); | |
396
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
1966 assert(*p != '_' && !isdigit(*p)); |
323 | 1967 assert(ulong_ == 0); |
58 | 1968 isDecimal = true; |
1969 goto Lfinalize; | |
1970 | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1971 LscanInteger: |
323 | 1972 assert(*p != 0 && isdigit(*p)); |
58 | 1973 isDecimal = true; |
323 | 1974 goto Lenter_loop_int; |
63 | 1975 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1976 { |
63 | 1977 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1978 continue; |
63 | 1979 if (!isdigit(*p)) |
1980 break; | |
323 | 1981 Lenter_loop_int: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1982 if (ulong_ < ulong.max/10 || (ulong_ == ulong.max/10 && *p <= '5')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1983 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1984 ulong_ *= 10; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1985 ulong_ += *p - '0'; |
58 | 1986 continue; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1987 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1988 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1989 overflow = true; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1990 while (isdigit(*++p)) {} |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1991 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1992 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1993 |
61 | 1994 // The number could be a float, so check overflow below. |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1995 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1996 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1997 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1998 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
1999 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2000 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2001 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2002 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2003 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2004 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2005 goto LscanReal; |
58 | 2006 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2007 } |
58 | 2008 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2009 if (overflow) |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2010 error(t.start, MID.OverflowDecimalNumber); |
58 | 2011 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2012 assert((isdigit(p[-1]) || p[-1] == '_') && !isdigit(*p) && *p != '_'); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2013 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2014 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2015 LscanHex: |
61 | 2016 assert(digits == 0); |
410
4d9ee8e60712
Added destructors for two particular tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
403
diff
changeset
|
2017 assert(*p == 'x' || *p == 'X'); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2018 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2019 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2020 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2021 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2022 if (!ishexad(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2023 break; |
61 | 2024 ++digits; |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
2025 ulong_ *= 16; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
2026 if (*p <= '9') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
2027 ulong_ += *p - '0'; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
2028 else if (*p <= 'F') |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
2029 ulong_ += *p - 'A' + 10; |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
2030 else |
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
2031 ulong_ += *p - 'a' + 10; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2032 } |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
2033 |
410
4d9ee8e60712
Added destructors for two particular tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
403
diff
changeset
|
2034 assert(ishexad(p[-1]) || p[-1] == '_' || p[-1] == 'x' || p[-1] == 'X'); |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2035 assert(!ishexad(*p) && *p != '_'); |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2036 |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2037 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2038 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2039 case '.': |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2040 if (p[1] == '.') |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2041 break; |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2042 case 'p', 'P': |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2043 return scanHexReal(t); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2044 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2045 } |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2046 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2047 if (digits == 0 || digits > 16) |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2048 error(t.start, digits == 0 ? MID.NoDigitsInHexNumber : MID.OverflowHexNumber); |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2049 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2050 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2051 |
396
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
2052 LscanBinary: |
57
c0f1c8be3a47
- Added code for converting hex characters to binary numbers.
aziz
parents:
56
diff
changeset
|
2053 assert(digits == 0); |
410
4d9ee8e60712
Added destructors for two particular tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
403
diff
changeset
|
2054 assert(*p == 'b' || *p == 'B'); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2055 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2056 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2057 if (*++p == '0') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2058 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2059 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2060 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2061 } |
367
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
2062 else if (*p == '1') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2063 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2064 ++digits; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2065 ulong_ *= 2; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2066 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2067 } |
367
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
2068 else if (*p == '_') |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
2069 continue; |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
2070 else |
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
2071 break; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2072 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2073 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2074 if (digits == 0 || digits > 64) |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2075 error(t.start, digits == 0 ? MID.NoDigitsInBinNumber : MID.OverflowBinaryNumber); |
59
3e594725899a
- Issuing error when no digits were found in hex and binary numbers.
aziz
parents:
58
diff
changeset
|
2076 |
410
4d9ee8e60712
Added destructors for two particular tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
403
diff
changeset
|
2077 assert(p[-1] == '0' || p[-1] == '1' || p[-1] == '_' || p[-1] == 'b' || p[-1] == 'B', p[-1] ~ ""); |
367
dda55fae37de
- ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents:
365
diff
changeset
|
2078 assert( !(*p == '0' || *p == '1' || *p == '_') ); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2079 goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2080 |
396
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
2081 LscanOctal: |
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
2082 assert(*p == '_'); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2083 while (1) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2084 { |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2085 if (*++p == '_') |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2086 continue; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2087 if (!isoctal(*p)) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2088 break; |
396
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
2089 Loctal_enter_loop: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2090 if (ulong_ < ulong.max/2 || (ulong_ == ulong.max/2 && *p <= '1')) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2091 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2092 ulong_ *= 8; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2093 ulong_ += *p - '0'; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2094 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2095 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2096 // Overflow: skip following digits. |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2097 overflow = true; |
396
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
2098 while (isoctal(*++p)) {} |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2099 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2100 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2101 |
63 | 2102 bool hasDecimalDigits; |
2103 if (isdigit(*p)) | |
2104 { | |
396
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
393
diff
changeset
|
2105 Loctal_hasDecimalDigits: |
63 | 2106 hasDecimalDigits = true; |
2107 while (isdigit(*++p)) {} | |
2108 } | |
2109 | |
2110 // The number could be a float, so check errors below. | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2111 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2112 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2113 case '.': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2114 if (p[1] != '.') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2115 goto LscanReal; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2116 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2117 case 'L': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2118 if (p[1] != 'i') |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2119 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2120 case 'i', 'f', 'F', 'e', 'E': |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2121 goto LscanReal; |
63 | 2122 default: |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2123 } |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2124 |
63 | 2125 if (hasDecimalDigits) |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2126 error(t.start, MID.OctalNumberHasDecimals); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2127 |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2128 if (overflow) |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2129 error(t.start, MID.OverflowOctalNumber); |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2130 // goto Lfinalize; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2131 |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2132 Lfinalize: |
58 | 2133 enum Suffix |
2134 { | |
2135 None = 0, | |
2136 Unsigned = 1, | |
60 | 2137 Long = 2 |
58 | 2138 } |
60 | 2139 |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2140 // Scan optional suffix: L, Lu, LU, u, uL, U or UL. |
58 | 2141 Suffix suffix; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2142 while (1) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2143 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2144 switch (*p) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2145 { |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2146 case 'L': |
58 | 2147 if (suffix & Suffix.Long) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2148 break; |
60 | 2149 suffix |= Suffix.Long; |
2150 ++p; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2151 continue; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2152 case 'u', 'U': |
58 | 2153 if (suffix & Suffix.Unsigned) |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2154 break; |
60 | 2155 suffix |= Suffix.Unsigned; |
2156 ++p; | |
2157 continue; | |
58 | 2158 default: |
2159 break; | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2160 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2161 break; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2162 } |
58 | 2163 |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2164 // Determine type of Integer. |
58 | 2165 switch (suffix) |
2166 { | |
2167 case Suffix.None: | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2168 if (ulong_ & 0x8000_0000_0000_0000) |
58 | 2169 { |
2170 if (isDecimal) | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2171 error(t.start, MID.OverflowDecimalSign); |
58 | 2172 t.type = TOK.Uint64; |
2173 } | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2174 else if (ulong_ & 0xFFFF_FFFF_0000_0000) |
58 | 2175 t.type = TOK.Int64; |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2176 else if (ulong_ & 0x8000_0000) |
58 | 2177 t.type = isDecimal ? TOK.Int64 : TOK.Uint32; |
2178 else | |
2179 t.type = TOK.Int32; | |
2180 break; | |
2181 case Suffix.Unsigned: | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2182 if (ulong_ & 0xFFFF_FFFF_0000_0000) |
58 | 2183 t.type = TOK.Uint64; |
2184 else | |
2185 t.type = TOK.Uint32; | |
2186 break; | |
2187 case Suffix.Long: | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2188 if (ulong_ & 0x8000_0000_0000_0000) |
58 | 2189 { |
2190 if (isDecimal) | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2191 error(t.start, MID.OverflowDecimalSign); |
58 | 2192 t.type = TOK.Uint64; |
2193 } | |
2194 else | |
2195 t.type = TOK.Int64; | |
2196 break; | |
2197 case Suffix.Unsigned | Suffix.Long: | |
2198 t.type = TOK.Uint64; | |
2199 break; | |
2200 default: | |
2201 assert(0); | |
2202 } | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2203 t.ulong_ = ulong_; |
15 | 2204 t.end = p; |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2205 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2206 LscanReal: |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2207 scanReal(t); |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2208 return; |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2209 } |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2210 |
63 | 2211 /* |
2212 FloatLiteral:= Float[fFL]?i? | |
2213 Float:= DecFloat | HexFloat | |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2214 DecFloat:= ([0-9][0-9_]*[.][0-9_]*DecExponent?) | [.][0-9][0-9_]*DecExponent? | [0-9][0-9_]*DecExponent |
63 | 2215 DecExponent:= [eE][+-]?[0-9][0-9_]* |
2216 HexFloat:= 0[xX](HexDigits[.]HexDigits | [.][0-9a-zA-Z]HexDigits? | HexDigits)HexExponent | |
2217 HexExponent:= [pP][+-]?[0-9][0-9_]* | |
2218 */ | |
56
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2219 void scanReal(ref Token t) |
63af7ddf52e1
- Started properly implementing number scanner. Added stub for real numbers.
aziz
parents:
55
diff
changeset
|
2220 { |
63 | 2221 if (*p == '.') |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2222 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2223 assert(p[1] != '.'); |
63 | 2224 // This function was called by scan() or scanNumber(). |
2225 while (isdigit(*++p) || *p == '_') {} | |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2226 } |
63 | 2227 else |
2228 // This function was called by scanNumber(). | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2229 assert(delegate () |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2230 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2231 switch (*p) |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2232 { |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2233 case 'L': |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2234 if (p[1] != 'i') |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2235 return false; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2236 case 'i', 'f', 'F', 'e', 'E': |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2237 return true; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2238 default: |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2239 } |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2240 return false; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2241 }() |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2242 ); |
63 | 2243 |
2244 // Scan exponent. | |
2245 if (*p == 'e' || *p == 'E') | |
2246 { | |
2247 ++p; | |
2248 if (*p == '-' || *p == '+') | |
2249 ++p; | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2250 if (isdigit(*p)) |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2251 while (isdigit(*++p) || *p == '_') {} |
63 | 2252 else |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2253 error(t.start, MID.FloatExpMustStartWithDigit); |
63 | 2254 } |
2255 | |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2256 // Copy whole number and remove underscores from buffer. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2257 char[] buffer = t.start[0..p-t.start].dup; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2258 uint j; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2259 foreach (c; buffer) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2260 if (c != '_') |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2261 buffer[j++] = c; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2262 buffer.length = j; // Adjust length. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2263 buffer ~= 0; // Terminate for C functions. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2264 |
63 | 2265 finalizeFloat(t, buffer); |
30 | 2266 } |
2267 | |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2268 void scanHexReal(ref Token t) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2269 { |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2270 assert(*p == '.' || *p == 'p' || *p == 'P'); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2271 MID mid; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2272 if (*p == '.') |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2273 while (ishexad(*++p) || *p == '_') |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2274 {} |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2275 // Decimal exponent is required. |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2276 if (*p != 'p' && *p != 'P') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2277 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2278 mid = MID.HexFloatExponentRequired; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2279 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2280 } |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2281 // Scan exponent |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2282 assert(*p == 'p' || *p == 'P'); |
393
fce1e6133dac
Applied fix to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
392
diff
changeset
|
2283 ++p; |
fce1e6133dac
Applied fix to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
392
diff
changeset
|
2284 if (*p == '+' || *p == '-') |
fce1e6133dac
Applied fix to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
392
diff
changeset
|
2285 ++p; |
fce1e6133dac
Applied fix to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
392
diff
changeset
|
2286 if (!isdigit(*p)) |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2287 { |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2288 mid = MID.HexFloatExpMustStartWithDigit; |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2289 goto Lerr; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2290 } |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2291 while (isdigit(*++p) || *p == '_') |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2292 {} |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2293 // Copy whole number and remove underscores from buffer. |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2294 char[] buffer = t.start[0..p-t.start].dup; |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2295 uint j; |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2296 foreach (c; buffer) |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2297 if (c != '_') |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2298 buffer[j++] = c; |
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2299 buffer.length = j; // Adjust length. |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2300 buffer ~= 0; // Terminate for C functions. |
63 | 2301 finalizeFloat(t, buffer); |
2302 return; | |
2303 Lerr: | |
2304 t.type = TOK.Float32; | |
2305 t.end = p; | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2306 error(t.start, mid); |
63 | 2307 } |
2308 | |
2309 void finalizeFloat(ref Token t, string buffer) | |
2310 { | |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2311 assert(buffer[$-1] == 0); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2312 // Float number is well-formed. Check suffixes and do conversion. |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2313 switch (*p) |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2314 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2315 case 'f', 'F': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2316 t.type = TOK.Float32; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2317 t.float_ = strtof(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2318 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2319 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2320 case 'L': |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2321 t.type = TOK.Float80; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2322 t.real_ = strtold(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2323 ++p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2324 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2325 default: |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2326 t.type = TOK.Float64; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2327 t.double_ = strtod(buffer.ptr, null); |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2328 break; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2329 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2330 if (*p == 'i') |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2331 { |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2332 ++p; |
63 | 2333 t.type += 3; // Switch to imaginary counterpart. |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2334 } |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
2335 if (errno() == ERANGE) |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2336 error(t.start, MID.OverflowFloatNumber); |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2337 t.end = p; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2338 } |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
61
diff
changeset
|
2339 |
30 | 2340 /// Scan special token: #line Integer [Filespec] EndOfLine |
360
b6a3755eba94
- Renamed scanSpecialToken() to scanSpecialTokenSequence().
aziz
parents:
350
diff
changeset
|
2341 void scanSpecialTokenSequence(ref Token t) |
30 | 2342 { |
47
8aa37a78937b
- Properly implemented scanner for normal string literals.
aziz
parents:
46
diff
changeset
|
2343 assert(*p == '#'); |
323 | 2344 t.type = TOK.HashLine; |
2345 | |
30 | 2346 MID mid; |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2347 auto errorAtColumn = p; |
30 | 2348 |
323 | 2349 ++p; |
2350 if (p[0] != 'l' || p[1] != 'i' || p[2] != 'n' || p[3] != 'e') | |
51 | 2351 { |
370 | 2352 mid = MID.ExpectedIdentifierSTLine; |
51 | 2353 goto Lerr; |
2354 } | |
323 | 2355 p += 3; |
2356 | |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
2357 // TODO: #line58"path/file" is legal. Require spaces? |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
2358 // State.Space could be used for that purpose. |
323 | 2359 enum State |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
2360 { /+Space,+/ Integer, Filespec, End } |
51 | 2361 |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
2362 State state = State.Integer; |
323 | 2363 |
2364 Loop: | |
2365 while (1) | |
51 | 2366 { |
323 | 2367 switch (*++p) |
2368 { | |
403
e7228859d865
Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
402
diff
changeset
|
2369 case LS[0]: |
e7228859d865
Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
402
diff
changeset
|
2370 if (!(p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))) |
e7228859d865
Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
402
diff
changeset
|
2371 goto default; |
e7228859d865
Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
402
diff
changeset
|
2372 case '\r', '\n', 0, _Z_: |
323 | 2373 break Loop; |
2374 default: | |
2375 if (isspace(*p)) | |
2376 continue; | |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
2377 if (state == State.Integer) |
323 | 2378 { |
2379 if (!isdigit(*p)) | |
2380 { | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2381 errorAtColumn = p; |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
2382 mid = MID.ExpectedIntegerAfterSTLine; |
323 | 2383 goto Lerr; |
2384 } | |
2385 t.line_num = new Token; | |
2386 scan(*t.line_num); | |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
2387 if (t.line_num.type != TOK.Int32 && t.line_num.type != TOK.Uint32) |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
2388 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2389 errorAtColumn = t.line_num.start; |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
2390 mid = MID.ExpectedIntegerAfterSTLine; |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
2391 goto Lerr; |
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
2392 } |
389
c4bfceab7246
Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
388
diff
changeset
|
2393 --p; // Go one back because scan() advanced p past the integer. |
323 | 2394 state = State.Filespec; |
2395 } | |
2396 else if (state == State.Filespec) | |
2397 { | |
2398 if (*p != '"') | |
2399 { | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2400 errorAtColumn = p; |
323 | 2401 mid = MID.ExpectedFilespec; |
2402 goto Lerr; | |
2403 } | |
2404 t.line_filespec = new Token; | |
2405 t.line_filespec.start = p; | |
2406 t.line_filespec.type = TOK.Filespec; | |
2407 while (1) | |
2408 { | |
2409 switch (*++p) | |
2410 { | |
2411 case '"': | |
2412 break; | |
2413 case LS[0]: | |
2414 if (!(p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))) | |
2415 goto default; | |
2416 case '\r', '\n', 0, _Z_: | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2417 errorAtColumn = t.line_filespec.start; |
323 | 2418 mid = MID.UnterminatedFilespec; |
2419 t.line_filespec.end = p; | |
2420 goto Lerr; | |
2421 default: | |
2422 if (*p & 128) | |
2423 decodeUTF8(); | |
2424 continue; | |
2425 } | |
2426 break; // Exit loop. | |
2427 } | |
2428 auto start = t.line_filespec.start +1; // +1 skips '"' | |
2429 t.line_filespec.str = start[0 .. p - start]; | |
2430 t.line_filespec.end = p + 1; | |
2431 state = State.End; | |
2432 } | |
2433 else/+ if (state == State.End)+/ | |
2434 { | |
2435 mid = MID.UnterminatedSpecialToken; | |
2436 goto Lerr; | |
2437 } | |
2438 } | |
51 | 2439 } |
403
e7228859d865
Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
402
diff
changeset
|
2440 assert(*p == '\r' || *p == '\n' || *p == 0 || *p == _Z_ || |
e7228859d865
Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
402
diff
changeset
|
2441 *p == LS[0] && (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
e7228859d865
Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
402
diff
changeset
|
2442 ); |
323 | 2443 |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
2444 if (state == State.Integer) |
51 | 2445 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2446 errorAtColumn = p; |
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
387
diff
changeset
|
2447 mid = MID.ExpectedIntegerAfterSTLine; |
30 | 2448 goto Lerr; |
2449 } | |
2450 | |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2451 // Evaluate #line only when not in token string. |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2452 if (!inTokenString) |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2453 { |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2454 this.loc_old = this.loc; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2455 this.loc_hline = t.line_num.uint_ - 1; |
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2456 if (t.line_filespec) |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2457 this.errorLoc.setFilePath(t.line_filespec.str); |
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
2458 } |
323 | 2459 t.end = p; |
30 | 2460 |
2461 return; | |
2462 Lerr: | |
323 | 2463 t.end = p; |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2464 error(errorAtColumn, mid); |
371 | 2465 } |
2466 | |
414
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2467 /+ |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2468 Insert an empty dummy token before t. |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2469 Useful in the parsing phase for representing a node in the AST |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2470 that doesn't consume an actual token from the source text. |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2471 +/ |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2472 Token* insertEmptyTokenBefore(Token* t) |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2473 { |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2474 assert(t !is null && t.prev !is null); |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2475 assert(text.ptr <= t.start && t.start < end, Token.toString(t.type)); |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2476 assert(text.ptr <= t.end && t.end <= end, Token.toString(t.type)); |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2477 |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2478 auto prev_t = t.prev; |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2479 auto new_t = new Token; |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2480 new_t.type = TOK.Empty; |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2481 new_t.start = new_t.end = prev_t.end; |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2482 // Link in new token. |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2483 prev_t.next = new_t; |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2484 new_t.prev = prev_t; |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2485 new_t.next = t; |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2486 t.prev = new_t; |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2487 return new_t; |
9c69615a4876
Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
413
diff
changeset
|
2488 } |
105 | 2489 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2490 void updateErrorLoc(char* columnPos) |
103
511a1aa25896
- Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents:
94
diff
changeset
|
2491 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2492 updateErrorLoc(this.loc, this.lineBegin, columnPos); |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
2493 } |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
2494 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2495 void updateErrorLoc(uint lineNum, char* lineBegin, char* columnPos) |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
2496 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2497 errorLoc.set(this.errorLineNum(lineNum), lineBegin, columnPos); |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
2498 } |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
2499 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2500 uint errorLineNum(uint loc) |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
223
diff
changeset
|
2501 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2502 // ∆loc + line_num_of(#line) |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2503 return loc - this.loc_old + this.loc_hline; |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
2504 } |
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
2505 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2506 void error(char* columnPos, MID mid, ...) |
207 | 2507 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2508 updateErrorLoc(columnPos); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2509 errors ~= new Information(InfoType.Lexer, mid, errorLoc.clone, Format(_arguments, _argptr, GetMsg(mid))); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2510 } |
207 | 2511 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2512 void error(uint lineNum, char* lineBegin, char* columnPos, MID mid, ...) |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2513 { |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2514 updateErrorLoc(lineNum, lineBegin, columnPos); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2515 errors ~= new Information(InfoType.Lexer, mid, errorLoc.clone, Format(_arguments, _argptr, GetMsg(mid))); |
207 | 2516 } |
2517 | |
306 | 2518 Token* getTokens() |
4 | 2519 { |
306 | 2520 while (nextToken() != TOK.EOF) |
2521 {} | |
2522 return head; | |
3 | 2523 } |
48 | 2524 |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2525 static void loadKeywords(ref Identifier[string] table) |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2526 { |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2527 foreach(k; keywords) |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2528 table[k.str] = k; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2529 } |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2530 |
365 | 2531 static bool isNonReservedIdentifier(char[] ident) |
2532 { | |
382
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2533 if (ident.length == 0) |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2534 return false; |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2535 |
365 | 2536 static Identifier[string] reserved_ids_table; |
2537 if (reserved_ids_table is null) | |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2538 loadKeywords(reserved_ids_table); |
365 | 2539 |
382
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2540 size_t idx = 1; // Index to the 2nd character in ident. |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2541 dchar isFirstCharUniAlpha() |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2542 { |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2543 idx = 0; |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2544 // NB: decode() could throw an Exception which would be |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2545 // caught by the next try-catch-block. |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2546 return isUniAlpha(std.utf.decode(ident, idx)); |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2547 } |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2548 |
365 | 2549 try |
382
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2550 { |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2551 if (isidbeg(ident[0]) || |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2552 ident[0] & 128 && isFirstCharUniAlpha()) |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2553 { |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2554 foreach (dchar c; ident[idx..$]) |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2555 if (!isident(c) && !isUniAlpha(c)) |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2556 return false; |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2557 } |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2558 } |
66477017cb95
Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
371
diff
changeset
|
2559 catch (Exception) |
365 | 2560 return false; |
2561 | |
2562 return !(ident in reserved_ids_table); | |
2563 } | |
2564 | |
430
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2565 /++ |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2566 Returns true if d can be encoded as a UTF-8 sequence. |
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2567 +/ |
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2568 bool isEncodable(dchar d) |
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2569 { |
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2570 return d < 0xD800 || |
430
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2571 (d > 0xDFFF && d <= 0x10FFFF); |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2572 } |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2573 |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2574 /++ |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2575 There are a total of 66 noncharacters. |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2576 Returns true if this is one of them. |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2577 See_also: Chapter 16.7 Noncharacters in Unicode 5.0 |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2578 +/ |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2579 bool isNoncharacter(dchar d) |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2580 { |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2581 return 0xFDD0 <= d && d <= 0xFDEF || // 32 |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2582 d <= 0x10FFFF && (d & 0xFFFF) >= 0xFFFE; // 34 |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2583 } |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2584 |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2585 /++ |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2586 Returns true if this character is not a noncharacter, not a surrogate |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2587 code point and not higher than 0x10FFFF. |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2588 +/ |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2589 bool isValidDecodedChar(dchar d) |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2590 { |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2591 return d < 0xD800 || |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2592 (d > 0xDFFF && d < 0xFDD0) || |
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2593 (d > 0xFDEF && d <= 0x10FFFF && (d & 0xFFFF) < 0xFFFE); |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2594 } |
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2595 |
426
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2596 /// Is this a trail byte of a UTF-8 sequence? |
425
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2597 bool isTrailByte(ubyte b) |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2598 { |
426
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2599 return (b & 0xC0) == 0x80; // 10xx_xxxx |
425
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2600 } |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2601 |
426
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2602 /// Is this a lead byte of a UTF-8 sequence? |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2603 bool isLeadByte(ubyte b) |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2604 { |
426
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2605 return (b & 0xC0) == 0xC0; // 11xx_xxxx |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2606 } |
425
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2607 |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2608 dchar decodeUTF8() |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2609 { |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2610 assert(!isascii(*p), "check for ASCII char before calling decodeUTF8()."); |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2611 char* p = this.p; |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2612 dchar d = *p; |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2613 |
426
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2614 ++p; // Move to second byte. |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2615 // Error if second byte is not a trail byte. |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2616 if (!isTrailByte(*p)) |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2617 goto Lerr2; |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2618 |
425
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2619 // Check for overlong sequences. |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2620 switch (d) |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2621 { |
426
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2622 case 0xE0, // 11100000 100xxxxx |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2623 0xF0, // 11110000 1000xxxx |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2624 0xF8, // 11111000 10000xxx |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2625 0xFC: // 11111100 100000xx |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2626 if ((*p & d) == 0x80) |
425
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2627 goto Lerr; |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2628 default: |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2629 if ((d & 0xFE) == 0xC0) // 1100000x |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2630 goto Lerr; |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2631 } |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2632 |
426
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2633 const char[] checkNextByte = "if (!isTrailByte(*++p))" |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2634 " goto Lerr2;"; |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2635 const char[] appendSixBits = "d = (d << 6) | *p & 0b0011_1111;"; |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2636 |
425
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2637 // Decode |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2638 if ((d & 0b1110_0000) == 0b1100_0000) |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2639 { |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2640 // 110xxxxx 10xxxxxx |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2641 d &= 0b0001_1111; |
426
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2642 mixin(appendSixBits); |
425
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2643 } |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2644 else if ((d & 0b1111_0000) == 0b1110_0000) |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2645 { |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2646 // 1110xxxx 10xxxxxx 10xxxxxx |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2647 d &= 0b0000_1111; |
426
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2648 mixin(appendSixBits ~ |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2649 checkNextByte ~ appendSixBits); |
425
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2650 } |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2651 else if ((d & 0b1111_1000) == 0b1111_0000) |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2652 { |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2653 // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2654 d &= 0b0000_0111; |
426
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2655 mixin(appendSixBits ~ |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2656 checkNextByte ~ appendSixBits ~ |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2657 checkNextByte ~ appendSixBits); |
425
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2658 } |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2659 else |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2660 // 5 and 6 byte UTF-8 sequences are not allowed yet. |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2661 // 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2662 // 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2663 goto Lerr; |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2664 |
426
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2665 assert(isTrailByte(*p)); |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2666 |
425
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2667 if (!isEncodable(d)) |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2668 { |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2669 Lerr: |
426
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2670 // Three cases: |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2671 // *) the UTF-8 sequence was successfully decoded but the resulting |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2672 // character is invalid. |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2673 // p points to last trail byte in the sequence. |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2674 // *) the UTF-8 sequence is overlong. |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2675 // p points to second byte in the sequence. |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2676 // *) the UTF-8 sequence has more than 4 bytes or starts with |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2677 // a trail byte. |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2678 // p points to second byte in the sequence. |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2679 assert(isTrailByte(*p)); |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2680 // Move to next ASCII character or lead byte of a UTF-8 sequence. |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2681 while (p < (end-1) && isTrailByte(*p)) |
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2682 ++p; |
425
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2683 --p; |
426
3f7790d3f9d6
Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
425
diff
changeset
|
2684 assert(!isTrailByte(p[1])); |
425
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2685 Lerr2: |
430
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2686 d = REPLACEMENT_CHAR; |
425
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2687 error(this.p, MID.InvalidUTF8Sequence); |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2688 } |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2689 |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2690 this.p = p; |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2691 return d; |
6bf936bf3356
Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
424
diff
changeset
|
2692 } |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2693 |
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2694 private void encodeUTF8(ref char[] str, dchar d) |
48 | 2695 { |
2696 char[6] b; | |
430
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2697 assert(!isascii(d), "check for ASCII char before calling encodeUTF8()."); |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2698 assert(isEncodable(d), "check that 'd' is encodable before calling encodeUTF8()."); |
430
e6c759e151cd
Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
426
diff
changeset
|
2699 |
48 | 2700 if (d < 0x800) |
2701 { | |
2702 b[0] = 0xC0 | (d >> 6); | |
2703 b[1] = 0x80 | (d & 0x3F); | |
2704 str ~= b[0..2]; | |
2705 } | |
2706 else if (d < 0x10000) | |
2707 { | |
2708 b[0] = 0xE0 | (d >> 12); | |
2709 b[1] = 0x80 | ((d >> 6) & 0x3F); | |
2710 b[2] = 0x80 | (d & 0x3F); | |
2711 str ~= b[0..3]; | |
2712 } | |
2713 else if (d < 0x200000) | |
2714 { | |
2715 b[0] = 0xF0 | (d >> 18); | |
2716 b[1] = 0x80 | ((d >> 12) & 0x3F); | |
2717 b[2] = 0x80 | ((d >> 6) & 0x3F); | |
2718 b[3] = 0x80 | (d & 0x3F); | |
2719 str ~= b[0..4]; | |
2720 } | |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2721 /+ // There are no 5 and 6 byte UTF-8 sequences yet. |
48 | 2722 else if (d < 0x4000000) |
2723 { | |
2724 b[0] = 0xF8 | (d >> 24); | |
2725 b[1] = 0x80 | ((d >> 18) & 0x3F); | |
2726 b[2] = 0x80 | ((d >> 12) & 0x3F); | |
2727 b[3] = 0x80 | ((d >> 6) & 0x3F); | |
2728 b[4] = 0x80 | (d & 0x3F); | |
2729 str ~= b[0..5]; | |
2730 } | |
2731 else if (d < 0x80000000) | |
2732 { | |
2733 b[0] = 0xFC | (d >> 30); | |
2734 b[1] = 0x80 | ((d >> 24) & 0x3F); | |
2735 b[2] = 0x80 | ((d >> 18) & 0x3F); | |
2736 b[3] = 0x80 | ((d >> 12) & 0x3F); | |
2737 b[4] = 0x80 | ((d >> 6) & 0x3F); | |
2738 b[5] = 0x80 | (d & 0x3F); | |
2739 str ~= b[0..6]; | |
2740 } | |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2741 +/ |
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2742 else |
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2743 assert(0); |
48 | 2744 } |
5 | 2745 } |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2746 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2747 unittest |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2748 { |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
2749 Stdout("Testing Lexer.\n"); |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2750 struct Pair |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2751 { |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2752 char[] token; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2753 TOK type; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2754 } |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2755 static Pair[] pairs = [ |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2756 {"//çay\n", TOK.Comment}, {"&", TOK.AndBinary}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2757 {"/*çağ*/", TOK.Comment}, {"&&", TOK.AndLogical}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2758 {"/+çak+/", TOK.Comment}, {"&=", TOK.AndAssign}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2759 {">", TOK.Greater}, {"+", TOK.Plus}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2760 {">=", TOK.GreaterEqual}, {"++", TOK.PlusPlus}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2761 {">>", TOK.RShift}, {"+=", TOK.PlusAssign}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2762 {">>=", TOK.RShiftAssign}, {"-", TOK.Minus}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2763 {">>>", TOK.URShift}, {"--", TOK.MinusMinus}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2764 {">>>=", TOK.URShiftAssign}, {"-=", TOK.MinusAssign}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2765 {"<", TOK.Less}, {"=", TOK.Assign}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2766 {"<=", TOK.LessEqual}, {"==", TOK.Equal}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2767 {"<>", TOK.LorG}, {"~", TOK.Tilde}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2768 {"<>=", TOK.LorEorG}, {"~=", TOK.CatAssign}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2769 {"<<", TOK.LShift}, {"*", TOK.Mul}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2770 {"<<=", TOK.LShiftAssign}, {"*=", TOK.MulAssign}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2771 {"!", TOK.Not}, {"/", TOK.Div}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2772 {"!=", TOK.NotEqual}, {"/=", TOK.DivAssign}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2773 {"!<", TOK.UorGorE}, {"^", TOK.Xor}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2774 {"!>", TOK.UorLorE}, {"^=", TOK.XorAssign}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2775 {"!<=", TOK.UorG}, {"%", TOK.Mod}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2776 {"!>=", TOK.UorL}, {"%=", TOK.ModAssign}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2777 {"!<>", TOK.UorE}, {"(", TOK.LParen}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2778 {"!<>=", TOK.Unordered}, {")", TOK.RParen}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2779 {".", TOK.Dot}, {"[", TOK.LBracket}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2780 {"..", TOK.Slice}, {"]", TOK.RBracket}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2781 {"...", TOK.Ellipses}, {"{", TOK.LBrace}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2782 {"|", TOK.OrBinary}, {"}", TOK.RBrace}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2783 {"||", TOK.OrLogical}, {":", TOK.Colon}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2784 {"|=", TOK.OrAssign}, {";", TOK.Semicolon}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2785 {"?", TOK.Question}, {",", TOK.Comma}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2786 {"$", TOK.Dollar}, {"cam", TOK.Identifier}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2787 {"çay", TOK.Identifier}, {".0", TOK.Float64}, |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2788 {"0", TOK.Int32}, |
40 | 2789 ]; |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2790 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2791 char[] src; |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2792 |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2793 foreach (pair; pairs) |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2794 src ~= pair.token ~ " "; |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2795 |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2796 assert(pairs[0].token == "//çay\n"); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2797 // Remove \n after src has been constructed. |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2798 // It won't be part of the scanned token string. |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2799 pairs[0].token = "//çay"; |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2800 |
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2801 auto lx = new Lexer(src, ""); |
350 | 2802 auto token = lx.getTokens(); |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
2803 |
350 | 2804 uint i; |
2805 assert(token == lx.head); | |
2806 token = token.next; | |
2807 do | |
2808 { | |
413
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2809 assert(i < pairs.length); |
0fd78fdcb982
Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
412
diff
changeset
|
2810 assert(token.srcText == pairs[i].token, Format("Scanned '{0}' but expected '{1}'", token.srcText, pairs[i].token)); |
350 | 2811 ++i; |
2812 token = token.next; | |
2813 } while (token.type != TOK.EOF) | |
41 | 2814 } |
55 | 2815 |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2816 unittest |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2817 { |
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2818 Stdout("Testing method Lexer.peek()\n"); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2819 string sourceText = "unittest { }"; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2820 auto lx = new Lexer(sourceText, null); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2821 |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2822 Token* next = lx.head; |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2823 lx.peek(next); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2824 assert(next.type == TOK.Unittest); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2825 lx.peek(next); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2826 assert(next.type == TOK.LBrace); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2827 lx.peek(next); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2828 assert(next.type == TOK.RBrace); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2829 lx.peek(next); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2830 assert(next.type == TOK.EOF); |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2831 } |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2832 |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2833 unittest |
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
420
diff
changeset
|
2834 { |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2835 // Numbers unittest |
68
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2836 // 0L 0ULi 0_L 0_UL 0x0U 0x0p2 0_Fi 0_e2 0_F 0_i |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2837 // 0u 0U 0uL 0UL 0L 0LU 0Lu |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2838 // 0Li 0f 0F 0fi 0Fi 0i |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2839 // 0b_1_LU 0b1000u |
7eb83dd38901
- Simplified suffix rule and added a few more numbers to unittest.
aziz
parents:
67
diff
changeset
|
2840 // 0x232Lu |
67
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2841 } |
996065105910
- Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents:
66
diff
changeset
|
2842 |
55 | 2843 /// ASCII character properties table. |
2844 static const int ptable[256] = [ | |
2845 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0, | |
2846 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2847 32, 0, 0x2200, 0, 0, 0, 0, 0x2700, 0, 0, 0, 0, 0, 0, 0, 0, | |
2848 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0x3f00, | |
2849 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8, | |
2850 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0x5c00, 0, 0,16, | |
2851 0, 0x70c, 0x80c,12,12,12, 0xc0c, 8, 8, 8, 8, 8, 8, 8, 0xa08, 8, | |
2852 8, 8, 0xd08, 8, 0x908, 8, 0xb08, 8, 8, 8, 8, 0, 0, 0, 0, 0, | |
2853 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2854 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2855 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2856 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2857 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2858 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2859 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | |
2860 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 | |
2861 ]; | |
2862 | |
2863 enum CProperty | |
2864 { | |
2865 Octal = 1, | |
2866 Digit = 1<<1, | |
2867 Hex = 1<<2, | |
2868 Alpha = 1<<3, | |
2869 Underscore = 1<<4, | |
2870 Whitespace = 1<<5 | |
2871 } | |
2872 | |
2873 const uint EVMask = 0xFF00; // Bit mask for escape value | |
2874 | |
2875 private alias CProperty CP; | |
2876 int isoctal(char c) { return ptable[c] & CP.Octal; } | |
2877 int isdigit(char c) { return ptable[c] & CP.Digit; } | |
2878 int ishexad(char c) { return ptable[c] & CP.Hex; } | |
2879 int isalpha(char c) { return ptable[c] & CP.Alpha; } | |
2880 int isalnum(char c) { return ptable[c] & (CP.Alpha | CP.Digit); } | |
2881 int isidbeg(char c) { return ptable[c] & (CP.Alpha | CP.Underscore); } | |
2882 int isident(char c) { return ptable[c] & (CP.Alpha | CP.Underscore | CP.Digit); } | |
2883 int isspace(char c) { return ptable[c] & CP.Whitespace; } | |
2884 int char2ev(char c) { return ptable[c] >> 8; /*(ptable[c] & EVMask) >> 8;*/ } | |
424
bb3cb00feeb2
Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
2885 int isascii(uint c) { return c < 128; } |
55 | 2886 |
2887 version(gen_ptable) | |
2888 static this() | |
2889 { | |
2890 alias ptable p; | |
402
22d65b2bef4f
Migrated the code that generates the ptable to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
397
diff
changeset
|
2891 assert(p.length == 256); |
55 | 2892 // Initialize character properties table. |
2893 for (int i; i < p.length; ++i) | |
2894 { | |
402
22d65b2bef4f
Migrated the code that generates the ptable to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
397
diff
changeset
|
2895 p[i] = 0; // Reset |
55 | 2896 if ('0' <= i && i <= '7') |
2897 p[i] |= CP.Octal; | |
2898 if ('0' <= i && i <= '9') | |
2899 p[i] |= CP.Digit; | |
2900 if (isdigit(i) || 'a' <= i && i <= 'f' || 'A' <= i && i <= 'F') | |
2901 p[i] |= CP.Hex; | |
2902 if ('a' <= i && i <= 'z' || 'A' <= i && i <= 'Z') | |
2903 p[i] |= CP.Alpha; | |
2904 if (i == '_') | |
2905 p[i] |= CP.Underscore; | |
2906 if (i == ' ' || i == '\t' || i == '\v' || i == '\f') | |
2907 p[i] |= CP.Whitespace; | |
2908 } | |
2909 // Store escape sequence values in second byte. | |
2910 assert(CProperty.max <= ubyte.max, "character property flags and escape value byte overlap."); | |
2911 p['\''] |= 39 << 8; | |
2912 p['"'] |= 34 << 8; | |
2913 p['?'] |= 63 << 8; | |
2914 p['\\'] |= 92 << 8; | |
2915 p['a'] |= 7 << 8; | |
2916 p['b'] |= 8 << 8; | |
2917 p['f'] |= 12 << 8; | |
2918 p['n'] |= 10 << 8; | |
2919 p['r'] |= 13 << 8; | |
2920 p['t'] |= 9 << 8; | |
2921 p['v'] |= 11 << 8; | |
2922 // Print a formatted array literal. | |
2923 char[] array = "[\n"; | |
402
22d65b2bef4f
Migrated the code that generates the ptable to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
397
diff
changeset
|
2924 foreach (i, c; ptable) |
55 | 2925 { |
402
22d65b2bef4f
Migrated the code that generates the ptable to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
397
diff
changeset
|
2926 array ~= Format((c>255?" 0x{0:x},":"{0,2},"), c) ~ (((i+1) % 16) ? "":"\n"); |
55 | 2927 } |
2928 array[$-2..$] = "\n]"; | |
402
22d65b2bef4f
Migrated the code that generates the ptable to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
397
diff
changeset
|
2929 Stdout(array).newline; |
55 | 2930 } |