Mercurial > projects > dil

annotate trunk/src/dil/Lexer.d @ 384:f7ce725e79c3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Refactored scanNormalStringLiteral().
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Tue, 11 Sep 2007 18:59:45 +0200
parents 6a5fc22cae34
children 392a0068fc61
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
1 /++
8ba2570de175 Initial import.
aziz
parents:
diff changeset
2 Author: Aziz Köksal
249
32d354584b28 - Upgraded license notices to GPL3.
aziz
parents: 239
diff changeset
3 License: GPL3
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
4 +/
326
4a7359b88c11 - Added package 'dil' to module declarations.
aziz
parents: 325
diff changeset
5 module dil.Lexer;
327
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
6 import dil.Token;
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
7 import dil.Information;
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
8 import dil.Keywords;
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
9 import dil.Identifier;
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
10 import dil.Messages;
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
11 import dil.HtmlEntities;
343
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
12 import dil.Settings;
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
13 import std.stdio;
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
14 import std.utf;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
15 import std.uni;
343
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
16 import std.c.stdlib : strtof, strtod, strtold, getErrno, ERANGE;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
17 import std.c.time : time_t, time, ctime;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
18 import std.c.string : strlen;
66
8e84db78ad55 - Added support for variadic arguments in error messages.
aziz
parents: 65
diff changeset
19 import std.string;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
20
11
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
21 const char[3] LS = \u2028;
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
22 const char[3] PS = \u2029;
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
23
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
24 const dchar LSd = 0x2028;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
25 const dchar PSd = 0x2029;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
26
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
27 const uint _Z_ = 26; /// Control+Z
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
28
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
29 class Lexer
8ba2570de175 Initial import.
aziz
parents:
diff changeset
30 {
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
31 Token* head; /// The head of the doubly linked token list.
344
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
32 Token* tail; /// The tail of the linked list. Set in scan().
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
33 Token* token; /// Points to the current token in the token list.
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
34 string text;
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
35 char* p; /// Points to the current character in the source text.
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
36 char* end; /// Points one character past the end of the source text.
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
37
371
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
38 uint loc = 1; /// Actual line of code.
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
39
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
40 uint loc_old; /// Store actual line number when #line token is parsed.
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
41 uint loc_hline; /// Line number set by #line.
17
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
42
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
43 char[] fileName;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
44
69
24db7c5522d5 - Added module Information for compiler messages like warnings, info and errors to the user.
aziz
parents: 68
diff changeset
45 Information[] errors;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
46
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
47 // bool reportErrors;
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
48
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
49 Identifier[string] idtable;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
50
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
51 this(string text, string fileName)
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
52 {
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
53 this.fileName = fileName;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
54
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
55 this.text = text;
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
56 if (text[$-1] != 0)
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
57 {
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
58 this.text.length = this.text.length + 1;
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
59 this.text[$-1] = 0;
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
60 }
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
61
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
62 this.p = this.text.ptr;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
63 this.end = this.p + this.text.length;
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
64 // this.reportErrors = true;
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
65 loadKeywords();
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
66
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
67 this.head = new Token;
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
68 this.head.type = TOK.HEAD;
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
69 this.token = this.head;
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
70 scanShebang();
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
71 }
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
72
344
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
73 ~this()
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
74 {
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
75 auto token = head.next;
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
76 do
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
77 {
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
78 assert(token.type == TOK.EOF ? token == tail && token.next is null : 1);
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
79 delete token.prev;
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
80 token = token.next;
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
81 } while (token !is null)
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
82 delete tail;
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
83 }
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
84
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
85 void scanShebang()
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
86 {
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
87 if (*p == '#' && p[1] == '!')
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
88 {
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
89 Token* t = new Token;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
90 t.start = p;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
91 t.type = TOK.Shebang;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
92 ++p;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
93 while (1)
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
94 {
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
95 t.end = p;
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
96 switch (*++p)
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
97 {
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
98 case '\r':
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
99 if (p[1] == '\n')
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
100 ++p;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
101 case '\n':
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
102 ++p;
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
103 ++loc;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
104 break;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
105 case 0, _Z_:
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
106 break;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
107 default:
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
108 if (*p & 128)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
109 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
110 auto c = decodeUTF8();
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
111 if (c == LSd || c == PSd)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
112 goto case '\n';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
113 }
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
114 continue;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
115 }
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
116 break; // Exit loop.
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
117 }
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
118 this.head.next = t;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
119 t.prev = this.head;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
120 }
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
121 }
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
122
343
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
123 void finalizeSpecialToken(ref Token t)
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
124 {
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
125 assert(t.srcText[0..2] == "__");
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
126 switch (t.type)
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
127 {
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
128 case TOK.FILE:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
129 t.str = this.fileName;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
130 break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
131 case TOK.LINE:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
132 t.uint_ = this.loc;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
133 break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
134 case TOK.DATE,
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
135 TOK.TIME,
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
136 TOK.TIMESTAMP:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
137 time_t time_val;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
138 time(&time_val);
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
139 char* str = ctime(&time_val);
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
140 char[] time_str = str[0 .. strlen(str)];
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
141 switch (t.type)
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
142 {
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
143 case TOK.DATE:
346
ce9e2e77743f - Fix: include one space when slicing time_str for __DATE__.
aziz
parents: 344
diff changeset
144 time_str = time_str[4..11] ~ time_str[20..24] ~ \0; break;
343
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
145 case TOK.TIME:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
146 time_str = time_str[11..19] ~ \0; break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
147 case TOK.TIMESTAMP:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
148 time_str = time_str[0..24] ~ \0; break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
149 default: assert(0);
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
150 }
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
151 t.str = time_str;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
152 break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
153 case TOK.VENDOR:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
154 t.str = VENDOR;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
155 break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
156 case TOK.VERSION:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
157 t.uint_ = VERSION_MAJOR*1000 + VERSION_MINOR;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
158 break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
159 default:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
160 assert(0);
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
161 }
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
162 }
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
163
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
164 public void scan(out Token t)
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
165 in
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
166 {
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
167 assert(text.ptr <= p && p < end);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
168 }
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
169 out
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
170 {
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
171 assert(text.ptr <= t.start && t.start < end);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
172 assert(text.ptr < t.end && t.end <= end, std.string.format(t.type));
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
173 }
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
174 body
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
175 {
10
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
176 uint c = *p;
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
177
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
178 while (1)
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
179 {
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
180 t.start = p;
16
476e8e55c1d4 - Added Whitespace to the character properties table.
aziz
parents: 15
diff changeset
181
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
182 if (c == 0 || c == _Z_)
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
183 {
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
184 assert(*p == 0 || *p == _Z_);
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
185 t.type = TOK.EOF;
17
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
186 t.end = p;
344
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
187 tail = &t;
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
188 assert(t.start == t.end);
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
189 return;
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
190 }
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
191
17
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
192 if (c == '\n')
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
193 {
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
194 c = *++p;
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
195 ++loc;
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
196 continue;
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
197 }
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
198 else if (c == '\r')
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
199 {
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
200 c = *++p;
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
201 if (c != '\n')
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
202 ++loc;
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
203 continue;
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
204 }
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
205 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
206 {
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
207 p += 3;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
208 c = *p;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
209 continue;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
210 }
17
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
211
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
212 if (isidbeg(c))
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
213 {
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
214 if (c == 'r' && p[1] == '"' && ++p)
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
215 return scanRawStringLiteral(t);
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
216 if (c == 'x' && p[1] == '"')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
217 return scanHexStringLiteral(t);
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
218 version(D2)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
219 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
220 if (c == 'q' && p[1] == '"')
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
221 return scanDelimitedStringLiteral(t);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
222 if (c == 'q' && p[1] == '{')
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
223 return scanTokenStringLiteral(t);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
224 }
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
225 Lidentifier:
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
226 do
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
227 { c = *++p; }
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
228 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8()))
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
229
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
230 t.end = p;
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
231
65
6c21ae79fbb3 - Renamed function Token.span to Token.srcText.
aziz
parents: 64
diff changeset
232 string str = t.srcText;
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
233 Identifier* id = str in idtable;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
234
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
235 if (!id)
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
236 {
327
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
237 idtable[str] = Identifier(TOK.Identifier, str);
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
238 id = str in idtable;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
239 }
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
240 assert(id);
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
241 t.type = id.type;
343
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
242 if (t.isSpecialToken)
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
243 finalizeSpecialToken(t);
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
244 return;
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
245 }
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
246
15
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
247 if (isdigit(c))
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
248 return scanNumber(t);
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
249
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
250 if (c == '/')
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
251 {
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
252 c = *++p;
14
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
253 switch(c)
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
254 {
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
255 case '=':
14
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
256 ++p;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
257 t.type = TOK.DivAssign;
14
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
258 t.end = p;
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
259 return;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
260 case '+':
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
261 uint level = 1;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
262 while (1)
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
263 {
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
264 c = *++p;
42
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
265 LswitchNC: // only jumped to from default case of next switch(c)
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
266 switch (c)
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
267 {
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
268 case '\r':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
269 if (p[1] == '\n')
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
270 ++p;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
271 case '\n':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
272 ++loc;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
273 continue;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
274 case 0, _Z_:
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
275 error(MID.UnterminatedNestedComment);
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
276 goto LreturnNC;
42
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
277 default:
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
278 }
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
279
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
280 c <<= 8;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
281 c |= *++p;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
282 switch (c)
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
283 {
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
284 case 0x2F2B: // /+
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
285 ++level;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
286 continue;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
287 case 0x2B2F: // +/
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
288 if (--level == 0)
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
289 {
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
290 ++p;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
291 LreturnNC:
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
292 t.type = TOK.Comment;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
293 t.end = p;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
294 return;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
295 }
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
296 continue;
42
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
297 case 0xE280: // LS[0..1] || PS[0..1]
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
298 if (p[1] == LS[2] || p[1] == PS[2])
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
299 {
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
300 ++loc;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
301 ++p;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
302 }
43
1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents: 42
diff changeset
303 continue;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
304 default:
42
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
305 c &= char.max;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
306 goto LswitchNC;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
307 }
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
308 }
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
309 case '*':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
310 while (1)
7
07e45c06a024 - Parsing nested comments correctly now.
aziz
parents: 5
diff changeset
311 {
43
1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents: 42
diff changeset
312 c = *++p;
1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents: 42
diff changeset
313 LswitchBC: // only jumped to from default case of next switch(c)
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
314 switch (c)
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
315 {
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
316 case '\r':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
317 if (p[1] == '\n')
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
318 ++p;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
319 case '\n':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
320 ++loc;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
321 continue;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
322 case 0, _Z_:
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
323 error(MID.UnterminatedBlockComment);
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
324 goto LreturnBC;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
325 default:
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
326 }
41
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
327
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
328 c <<= 8;
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
329 c |= *++p;
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
330 switch (c)
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
331 {
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
332 case 0x2A2F: // */
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
333 ++p;
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
334 LreturnBC:
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
335 t.type = TOK.Comment;
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
336 t.end = p;
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
337 return;
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
338 case 0xE280: // LS[0..1] || PS[0..1]
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
339 if (p[1] == LS[2] || p[1] == PS[2])
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
340 {
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
341 ++loc;
43
1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents: 42
diff changeset
342 ++p;
41
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
343 }
43
1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents: 42
diff changeset
344 continue;
41
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
345 default:
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
346 c &= char.max;
43
1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents: 42
diff changeset
347 goto LswitchBC;
41
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
348 }
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
349 }
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
350 assert(0);
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
351 case '/':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
352 while (1)
7
07e45c06a024 - Parsing nested comments correctly now.
aziz
parents: 5
diff changeset
353 {
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
354 c = *++p;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
355 switch (c)
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
356 {
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
357 case '\r':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
358 if (p[1] == '\n')
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
359 ++p;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
360 case '\n':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
361 case 0, _Z_:
11
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
362 break;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
363 case LS[0]:
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
364 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
365 break;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
366 continue;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
367 default:
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
368 continue;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
369 }
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
370 t.type = TOK.Comment;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
371 t.end = p;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
372 return;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
373 }
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
374 default:
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
375 t.type = TOK.Div;
10
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
376 t.end = p;
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
377 return;
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
378 }
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
379 }
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
380
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
381 switch (c)
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
382 {
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
383 case '\'':
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
384 return scanCharacterLiteral(t);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
385 case '`':
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
386 return scanRawStringLiteral(t);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
387 case '"':
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
388 return scanNormalStringLiteral(t);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
389 case '\\':
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
390 char[] buffer;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
391 do
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
392 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
393 ++p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
394 c = scanEscapeSequence();
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
395 if (c < 128)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
396 buffer ~= c;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
397 else
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
398 encodeUTF8(buffer, c);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
399 } while (*p == '\\')
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
400 buffer ~= 0;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
401 t.type = TOK.String;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
402 t.str = buffer;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
403 t.end = p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
404 return;
38
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
405 case '>': /* > >= >> >>= >>> >>>= */
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
406 c = *++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
407 switch (c)
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
408 {
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
409 case '=':
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
410 t.type = TOK.GreaterEqual;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
411 goto Lcommon;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
412 case '>':
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
413 if (p[1] == '>')
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
414 {
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
415 ++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
416 if (p[1] == '=')
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
417 { ++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
418 t.type = TOK.URShiftAssign;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
419 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
420 else
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
421 t.type = TOK.URShift;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
422 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
423 else if (p[1] == '=')
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
424 {
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
425 ++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
426 t.type = TOK.RShiftAssign;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
427 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
428 else
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
429 t.type = TOK.RShift;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
430 goto Lcommon;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
431 default:
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
432 t.type = TOK.Greater;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
433 goto Lcommon2;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
434 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
435 assert(0);
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
436 case '<': /* < <= <> <>= << <<= */
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
437 c = *++p;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
438 switch (c)
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
439 {
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
440 case '=':
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
441 t.type = TOK.LessEqual;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
442 goto Lcommon;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
443 case '<':
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
444 if (p[1] == '=') {
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
445 ++p;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
446 t.type = TOK.LShiftAssign;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
447 }
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
448 else
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
449 t.type = TOK.LShift;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
450 goto Lcommon;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
451 case '>':
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
452 if (p[1] == '=') {
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
453 ++p;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
454 t.type = TOK.LorEorG;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
455 }
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
456 else
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
457 t.type = TOK.LorG;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
458 goto Lcommon;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
459 default:
38
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
460 t.type = TOK.Less;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
461 goto Lcommon2;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
462 }
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
463 assert(0);
37
7f3bcb97d017 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 36
diff changeset
464 case '!': /* ! !< !> !<= !>= !<> !<>= */
35
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
465 c = *++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
466 switch (c)
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
467 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
468 case '<':
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
469 c = *++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
470 if (c == '>')
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
471 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
472 if (p[1] == '=') {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
473 ++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
474 t.type = TOK.Unordered;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
475 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
476 else
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
477 t.type = TOK.UorE;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
478 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
479 else if (c == '=')
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
480 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
481 t.type = TOK.UorG;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
482 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
483 else {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
484 t.type = TOK.UorGorE;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
485 goto Lcommon2;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
486 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
487 goto Lcommon;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
488 case '>':
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
489 if (p[1] == '=')
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
490 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
491 ++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
492 t.type = TOK.UorL;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
493 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
494 else
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
495 t.type = TOK.UorLorE;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
496 goto Lcommon;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
497 case '=':
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
498 t.type = TOK.NotEqual;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
499 goto Lcommon;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
500 default:
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
501 t.type = TOK.Not;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
502 goto Lcommon2;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
503 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
504 assert(0);
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
505 case '.': /* . .[0-9] .. ... */
22
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
506 if (p[1] == '.')
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
507 {
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
508 ++p;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
509 if (p[1] == '.') {
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
510 ++p;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
511 t.type = TOK.Ellipses;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
512 }
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
513 else
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
514 t.type = TOK.Slice;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
515 }
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
516 else if (isdigit(p[1]))
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
517 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
518 return scanReal(t);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
519 }
22
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
520 else
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
521 t.type = TOK.Dot;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
522 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
523 case '|': /* | || |= */
23
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
524 c = *++p;
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
525 if (c == '=')
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
526 t.type = TOK.OrAssign;
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
527 else if (c == '|')
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
528 t.type = TOK.OrLogical;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
529 else {
23
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
530 t.type = TOK.OrBinary;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
531 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
532 }
23
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
533 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
534 case '&': /* & && &= */
24
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
535 c = *++p;
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
536 if (c == '=')
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
537 t.type = TOK.AndAssign;
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
538 else if (c == '&')
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
539 t.type = TOK.AndLogical;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
540 else {
24
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
541 t.type = TOK.AndBinary;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
542 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
543 }
24
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
544 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
545 case '+': /* + ++ += */
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
546 c = *++p;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
547 if (c == '=')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
548 t.type = TOK.PlusAssign;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
549 else if (c == '+')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
550 t.type = TOK.PlusPlus;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
551 else {
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
552 t.type = TOK.Plus;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
553 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
554 }
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
555 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
556 case '-': /* - -- -= */
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
557 c = *++p;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
558 if (c == '=')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
559 t.type = TOK.MinusAssign;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
560 else if (c == '-')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
561 t.type = TOK.MinusMinus;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
562 else {
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
563 t.type = TOK.Minus;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
564 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
565 }
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
566 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
567 case '=': /* = == */
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
568 if (p[1] == '=') {
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
569 ++p;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
570 t.type = TOK.Equal;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
571 }
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
572 else
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
573 t.type = TOK.Assign;
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
574 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
575 case '~': /* ~ ~= */
27
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
576 if (p[1] == '=') {
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
577 ++p;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
578 t.type = TOK.CatAssign;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
579 }
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
580 else
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
581 t.type = TOK.Tilde;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
582 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
583 case '*': /* * *= */
29
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
584 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
585 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
586 t.type = TOK.MulAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
587 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
588 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
589 t.type = TOK.Mul;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
590 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
591 case '^': /* ^ ^= */
29
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
592 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
593 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
594 t.type = TOK.XorAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
595 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
596 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
597 t.type = TOK.Xor;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
598 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
599 case '%': /* % %= */
29
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
600 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
601 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
602 t.type = TOK.ModAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
603 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
604 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
605 t.type = TOK.Mod;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
606 goto Lcommon;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
607 // Single character tokens:
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
608 case '(':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
609 t.type = TOK.LParen;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
610 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
611 case ')':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
612 t.type = TOK.RParen;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
613 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
614 case '[':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
615 t.type = TOK.LBracket;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
616 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
617 case ']':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
618 t.type = TOK.RBracket;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
619 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
620 case '{':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
621 t.type = TOK.LBrace;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
622 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
623 case '}':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
624 t.type = TOK.RBrace;
21
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
625 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
626 case ':':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
627 t.type = TOK.Colon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
628 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
629 case ';':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
630 t.type = TOK.Semicolon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
631 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
632 case '?':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
633 t.type = TOK.Question;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
634 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
635 case ',':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
636 t.type = TOK.Comma;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
637 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
638 case '$':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
639 t.type = TOK.Dollar;
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
640 Lcommon:
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
641 ++p;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
642 Lcommon2:
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
643 t.end = p;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
644 return;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
645 case '#':
360
b6a3755eba94 - Renamed scanSpecialToken() to scanSpecialTokenSequence().
aziz
parents: 350
diff changeset
646 return scanSpecialTokenSequence(t);
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
647 default:
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
648 }
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
649
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
650 if (c & 128 && isUniAlpha(decodeUTF8()))
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
651 goto Lidentifier;
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
652 c = *++p;
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
653 }
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
654 }
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
655
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
656 void scanNormalStringLiteral(ref Token t)
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
657 {
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
658 assert(*p == '"');
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
659 char[] buffer;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
660 t.type = TOK.String;
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
661 uint c;
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
662 while (1)
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
663 {
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
664 c = *++p;
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
665 switch (c)
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
666 {
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
667 case '"':
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
668 ++p;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
669 Lreturn:
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
670 buffer ~= 0;
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
671 t.str = buffer;
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
672 t.pf = scanPostfix();
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
673 t.end = p;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
674 return;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
675 case '\\':
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
676 ++p;
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
677 c = scanEscapeSequence();
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
678 --p;
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
679 if (c & 128)
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
680 encodeUTF8(buffer, c);
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
681 else
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
682 break;
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
683 continue;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
684 case '\r':
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
685 if (p[1] == '\n')
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
686 ++p;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
687 case '\n':
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
688 ++loc;
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
689 c = '\n'; // Convert EndOfLine to \n.
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
690 break;
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
691 case 0, _Z_:
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
692 error(MID.UnterminatedString);
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
693 goto Lreturn;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
694 default:
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
695 if (c & 128)
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
696 {
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
697 // char* begin = p;
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
698 c = decodeUTF8();
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
699 if (c == LSd || c == PSd)
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
700 goto case '\n';
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
701
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
702 // We don't copy per pointer because we might include
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
703 // invalid, skipped utf-8 sequences. See decodeUTF8().
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
704 // ++p;
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
705 // buffer ~= begin[0 .. p - begin];
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
706 encodeUTF8(buffer, c);
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
707 continue;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
708 }
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
709 }
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
710 // Copy ASCII character.
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
711 buffer ~= c;
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
712 }
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
713 assert(0);
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
714 }
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
715
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
716 void scanCharacterLiteral(ref Token t)
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
717 {
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
718 assert(*p == '\'');
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
719 MID id = MID.UnterminatedCharacterLiteral;
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
720 ++p;
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
721 TOK type = TOK.CharLiteral;
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
722 switch (*p)
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
723 {
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
724 case '\\':
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
725 ++p;
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
726 switch (*p)
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
727 {
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
728 case 'u':
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
729 type = TOK.WCharLiteral; break;
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
730 case 'U':
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
731 type = TOK.DCharLiteral; break;
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
732 default:
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
733 }
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
734 t.dchar_ = scanEscapeSequence();
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
735 break;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
736 case '\'':
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
737 ++p;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
738 id = MID.EmptyCharacterLiteral;
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
739 case '\n', '\r', 0, _Z_:
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
740 goto Lerr;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
741 default:
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
742 uint c = *p;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
743 if (c & 128)
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
744 {
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
745 c = decodeUTF8();
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
746 if (c == LSd || c == PSd)
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
747 goto Lerr;
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
748 if (c <= 0xFFFF)
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
749 type = TOK.WCharLiteral;
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
750 else
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
751 type = TOK.DCharLiteral;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
752 }
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
753 t.dchar_ = c;
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
754 ++p;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
755 }
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
756
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
757 if (*p == '\'')
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
758 ++p;
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
759 else
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
760 Lerr:
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
761 error(id);
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
762 t.type = type;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
763 t.end = p;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
764 }
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
765
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
766 char scanPostfix()
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
767 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
768 switch (*p)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
769 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
770 case 'c':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
771 case 'w':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
772 case 'd':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
773 return *p++;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
774 default:
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
775 return 0;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
776 }
112
004d98df65af - Implemented parseInterfaceDeclaration().
aziz
parents: 105
diff changeset
777 assert(0);
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
778 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
779
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
780 void scanRawStringLiteral(ref Token t)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
781 {
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
782 uint delim = *p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
783 assert(delim == '`' || delim == '"' && p[-1] == 'r');
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
784 t.type = TOK.String;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
785 char[] buffer;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
786 uint c;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
787 while (1)
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
788 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
789 c = *++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
790 switch (c)
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
791 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
792 case '\r':
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
793 if (p[1] == '\n')
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
794 ++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
795 case '\n':
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
796 c = '\n'; // Convert EndOfLine ('\r','\r\n','\n',LS,PS) to '\n'
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
797 ++loc;
52
f65a83c27638 - Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents: 51
diff changeset
798 break;
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
799 case '`':
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
800 case '"':
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
801 if (c == delim)
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
802 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
803 ++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
804 t.pf = scanPostfix();
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
805 Lreturn:
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
806 t.str = buffer ~ '\0';
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
807 t.end = p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
808 return;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
809 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
810 break;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
811 case 0, _Z_:
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
812 if (delim == 'r')
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
813 error(MID.UnterminatedRawString);
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
814 else
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
815 error(MID.UnterminatedBackQuoteString);
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
816 goto Lreturn;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
817 default:
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
818 if (c & 128)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
819 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
820 c = decodeUTF8();
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
821 if (c == LSd || c == PSd)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
822 goto case '\n';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
823 encodeUTF8(buffer, c);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
824 continue;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
825 }
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
826 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
827 buffer ~= c; // copy character to buffer
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
828 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
829 assert(0);
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
830 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
831
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
832 void scanHexStringLiteral(ref Token t)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
833 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
834 assert(p[0] == 'x' && p[1] == '"');
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
835 t.type = TOK.String;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
836
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
837 uint c;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
838 ubyte[] buffer;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
839 ubyte h; // hex number
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
840 uint n; // number of hex digits
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
841
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
842 ++p;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
843 while (1)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
844 {
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
845 c = *++p;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
846 switch (c)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
847 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
848 case '"':
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
849 ++p;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
850 if (n & 1)
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
851 error(MID.OddNumberOfDigitsInHexString);
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
852 t.pf = scanPostfix();
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
853 Lreturn:
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
854 buffer ~= 0;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
855 t.str = cast(string) buffer;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
856 t.end = p;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
857 return;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
858 case '\r':
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
859 if (p[1] == '\n')
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
860 ++p;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
861 case '\n':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
862 ++loc;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
863 continue;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
864 default:
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
865 if (ishexad(c))
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
866 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
867 if (c <= '9')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
868 c -= '0';
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
869 else if (c <= 'F')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
870 c -= 'A' - 10;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
871 else
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
872 c -= 'a' - 10;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
873
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
874 if (n & 1)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
875 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
876 h <<= 4;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
877 h |= c;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
878 buffer ~= h;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
879 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
880 else
112
004d98df65af - Implemented parseInterfaceDeclaration().
aziz
parents: 105
diff changeset
881 h = cast(ubyte)c;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
882 ++n;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
883 continue;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
884 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
885 else if (isspace(c))
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
886 continue;
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
887
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
888 if (c & 128)
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
889 {
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
890 c = decodeUTF8();
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
891 if (c == LSd || c == PSd)
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
892 {
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
893 ++p; ++p;
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
894 ++loc;
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
895 continue;
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
896 }
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
897 }
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
898 else if (c == 0 || c == _Z_)
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
899 {
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
900 error(MID.UnterminatedHexString);
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
901 t.pf = 0;
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
902 goto Lreturn;
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
903 }
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
904 error(MID.NonHexCharInHexString, cast(dchar)c);
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
905 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
906 }
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
907 assert(0);
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
908 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
909
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
910 version(D2)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
911 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
912 void scanDelimitedStringLiteral(ref Token t)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
913 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
914 assert(p[0] == 'q' && p[1] == '"');
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
915 t.type = TOK.String;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
916
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
917 char[] buffer;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
918 dchar opening_delim, // 0 if no nested delimiter or '[', '(', '<', '{'
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
919 closing_delim; // Will be ']', ')', '>', '}', any other character
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
920 // or the first, decoded character of an identifier.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
921 char[] str_delim; // Identifier delimiter
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
922 uint level = 1;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
923
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
924 ++p; ++p; // Skip q"
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
925 uint c = *p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
926 switch (c)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
927 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
928 case '(':
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
929 opening_delim = c;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
930 closing_delim = ')'; // *p + 1
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
931 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
932 case '[', '<', '{':
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
933 opening_delim = c;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
934 closing_delim = c + 2; // Get to closing counterpart. Feature of ASCII table.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
935 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
936 default:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
937 char* begin = p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
938 closing_delim = c;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
939 // TODO: What to do about newlines? Skip or accept as delimiter?
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
940 // TODO: Check for non-printable characters?
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
941 if (c & 128)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
942 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
943 closing_delim = decodeUTF8();
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
944 if (!isUniAlpha(c))
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
945 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
946 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
947 else if (!isidbeg(c))
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
948 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
949 // Parse identifier + newline
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
950 do
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
951 { c = *++p; }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
952 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8()))
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
953 // Store identifier
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
954 str_delim = begin[0..p-begin];
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
955 // Scan newline
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
956 switch (*p)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
957 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
958 case '\r':
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
959 if (p[1] == '\n')
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
960 ++p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
961 case '\n':
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
962 ++loc;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
963 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
964 case LS[0]:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
965 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
966 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
967 ++p; ++p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
968 ++loc;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
969 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
970 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
971 // goto default;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
972 default:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
973 // TODO: error(MID.ExpectedNewlineAfterIdentDelim);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
974 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
975 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
976
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
977 bool checkStringDelim(char* p)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
978 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
979 assert(str_delim.length != 0);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
980 if (end-p >= str_delim.length && // Check remaining length.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
981 p[0..str_delim.length] == str_delim) // Compare.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
982 return true;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
983 return false;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
984 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
985
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
986 while (1)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
987 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
988 c = *++p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
989 switch (c)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
990 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
991 case '\r':
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
992 if (p[1] == '\n')
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
993 ++p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
994 case '\n':
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
995 c = '\n'; // Convert EndOfLine ('\r','\r\n','\n',LS,PS) to '\n'
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
996 ++loc;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
997 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
998 case 0, _Z_:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
999 // error(MID.UnterminatedDelimitedString);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1000 goto Lreturn3;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1001 default:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1002 if (c & 128)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1003 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1004 auto begin = p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1005 c = decodeUTF8();
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1006 if (c == LSd || c == PSd)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1007 goto case '\n';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1008 if (c == closing_delim)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1009 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1010 if (str_delim.length && checkStringDelim(begin))
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1011 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1012 p = begin + str_delim.length;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1013 goto Lreturn2;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1014 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1015 assert(level == 1);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1016 --level;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1017 goto Lreturn;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1018 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1019 encodeUTF8(buffer, c);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1020 continue;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1021 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1022 else
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1023 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1024 if (c == opening_delim)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1025 ++level;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1026 else if (c == closing_delim)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1027 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1028 if (str_delim.length && checkStringDelim(p))
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1029 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1030 p += str_delim.length;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1031 goto Lreturn2;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1032 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1033 if (--level == 0)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1034 goto Lreturn;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1035 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1036 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1037 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1038 buffer ~= c; // copy character to buffer
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1039 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1040 Lreturn:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1041 assert(*p == closing_delim);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1042 assert(level == 0);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1043 ++p; // Skip closing delimiter.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1044 Lreturn2:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1045 if (*p == '"')
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1046 ++p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1047 // else
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1048 // TODO: error(MID.ExpectedDblQuoteAfterDelim, str_delim.length ? str_delim : p[-1]);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1049
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1050 t.pf = scanPostfix();
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1051 Lreturn3:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1052 t.str = buffer ~ '\0';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1053 t.end = p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1054 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1055
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1056 void scanTokenStringLiteral(ref Token t)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1057 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1058 assert(p[0] == 'q' && p[1] == '{');
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1059 t.type = TOK.String;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1060 // Copy members that might be changed by subsequent tokens. Like #line for example.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1061 auto loc_old = this.loc_old;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1062 auto loc_hline = this.loc_hline;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1063 auto filePath = this.fileName;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1064
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1065 uint loc = this.loc;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1066 uint level = 1;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1067
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1068 ++p; ++p; // Skip q{
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1069
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1070 auto prev_t = &t;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1071 Token* token;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1072 while (1)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1073 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1074 token = new Token;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1075 scan(*token);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1076 // Save the tokens in a doubly linked list.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1077 // Could be useful for various tools.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1078 token.prev = prev_t;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1079 prev_t.next = token;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1080 prev_t = token;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1081 switch (token.type)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1082 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1083 case TOK.LBrace:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1084 ++level;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1085 continue;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1086 case TOK.RBrace:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1087 if (--level == 0)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1088 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1089 t.tok_str = t.next;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1090 t.next = null;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1091 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1092 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1093 continue;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1094 case TOK.EOF:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1095 // TODO: error(MID.UnterminatedTokenString);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1096 t.tok_str = t.next;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1097 t.next = token;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1098 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1099 default:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1100 continue;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1101 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1102 break; // Exit loop.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1103 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1104
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1105 assert(token.type == TOK.RBrace || token.type == TOK.EOF);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1106 assert(token.type == TOK.RBrace && t.next is null ||
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1107 token.type == TOK.EOF && t.next !is null);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1108
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1109 char[] buffer;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1110 // token points to } or EOF
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1111 if (token.type == TOK.EOF)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1112 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1113 t.end = token.start;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1114 buffer = t.srcText[2..$].dup ~ '\0';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1115 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1116 else
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1117 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1118 // Assign to buffer before scanPostfix().
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1119 t.end = p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1120 buffer = t.srcText[2..$-1].dup ~ '\0';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1121 t.pf = scanPostfix();
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1122 t.end = p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1123 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1124 // Convert EndOfLines to '\n'
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1125 if (loc != this.loc)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1126 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1127 assert(buffer[$-1] == '\0');
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1128 uint i, j;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1129 for (; i < buffer.length; ++i)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1130 switch (buffer[i])
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1131 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1132 case '\r':
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1133 if (buffer[i+1] == '\n')
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1134 ++i;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1135 case '\n':
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1136 buffer[j++] = '\n';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1137 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1138 case LS[0]:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1139 auto b = buffer[i..$];
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1140 if (b[1] == LS[1] && (b[2] == LS[2] || b[2] == PS[2]))
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1141 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1142 ++i; ++i;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1143 goto case '\n';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1144 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1145 // goto default;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1146 default:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1147 buffer[j++] = buffer[i]; // Copy character
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1148 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1149 buffer.length = j; // Adjust length
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1150 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1151 assert(buffer[$-1] == '\0');
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1152 t.str = buffer;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1153
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1154 // Restore possibly changed members.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1155 this.loc_old = loc_old;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1156 this.loc_hline = loc_hline;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1157 this.fileName = filePath;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1158 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1159 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1160
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1161 dchar scanEscapeSequence()
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1162 {
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1163 uint c = char2ev(*p);
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1164 if (c)
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1165 {
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1166 ++p;
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1167 return c;
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1168 }
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1169 uint digits = 2;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1170
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1171 switch (*p)
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1172 {
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1173 case 'x':
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1174 c = 0;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1175 while (1)
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1176 {
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1177 ++p;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1178 if (ishexad(*p))
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1179 {
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1180 c *= 16;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1181 if (*p <= '9')
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1182 c += *p - '0';
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1183 else if (*p <= 'F')
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1184 c += *p - 'A' + 10;
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1185 else
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1186 c += *p - 'a' + 10;
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1187
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1188 if (!--digits)
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1189 {
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1190 ++p;
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1191 break;
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1192 }
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1193 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1194 else
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1195 {
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1196 error(MID.InsufficientHexDigits);
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1197 break;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1198 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1199 }
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1200 if (!isValidDchar(c))
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1201 error(MID.InvalidUnicodeCharacter);
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1202 break;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1203 case 'u':
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1204 digits = 4;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1205 goto case 'x';
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1206 case 'U':
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1207 digits = 8;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1208 goto case 'x';
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1209 default:
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1210 if (isoctal(*p))
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1211 {
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1212 c = 0;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1213 c += *p - '0';
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1214 ++p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1215 if (!isoctal(*p))
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1216 return c;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1217 c *= 8;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1218 c += *p - '0';
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1219 ++p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1220 if (!isoctal(*p))
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1221 return c;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1222 c *= 8;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1223 c += *p - '0';
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1224 ++p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1225 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1226 else if(*p == '&')
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1227 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1228 if (isalpha(*++p))
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1229 {
272
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1230 auto begin = p;
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1231 while (isalnum(*++p))
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1232 {}
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1233
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1234 if (*p == ';')
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1235 {
272
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1236 c = entity2Unicode(begin[0..p - begin]);
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1237 ++p;
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1238 if (c == 0xFFFF)
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1239 error(MID.UndefinedHTMLEntity, (begin-1)[0..p-(begin-1)]);
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1240 }
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1241 else
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1242 error(MID.UnterminatedHTMLEntity);
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1243 }
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1244 else
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1245 error(MID.InvalidBeginHTMLEntity);
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1246 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1247 else
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1248 error(MID.UndefinedEscapeSequence);
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1249 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1250
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1251 return c;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1252 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1253
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1254 /*
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1255 IntegerLiteral:= (Dec|Hex|Bin|Oct)Suffix?
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1256 Dec:= (0|[1-9][0-9_]*)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1257 Hex:= 0[xX] HexDigits
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1258 Bin:= 0[bB][01_]+
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1259 Oct:= 0[0-7_]+
68
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
1260 Suffix:= (L[uU]?|[uU]L?)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1261 HexDigits:= [0-9a-zA-Z_]+
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1262
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1263 Invalid: "0b_", "0x_", "._"
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1264 */
15
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
1265 void scanNumber(ref Token t)
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
1266 {
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1267 ulong ulong_;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1268 bool overflow;
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1269 bool isDecimal;
57
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
1270 size_t digits;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1271
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1272 if (*p != '0')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1273 goto LscanInteger;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1274 ++p; // skip zero
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1275 // check for xX bB ...
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1276 switch (*p)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1277 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1278 case 'x','X':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1279 goto LscanHex;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1280 case 'b','B':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1281 goto LscanBin;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1282 case 'L':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1283 if (p[1] == 'i')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1284 goto LscanReal;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1285 case '.':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1286 if (p[1] == '.')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1287 break;
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1288 case 'i','f','F', 'e', 'E': // Imaginary and float literal suffix
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1289 goto LscanReal;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1290 default:
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1291 if (*p == '_' || isoctal(*p))
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1292 goto LscanOct;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1293 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1294
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1295 // Number 0
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1296 assert(p[-1] == '0');
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1297 assert(ulong_ == 0);
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1298 isDecimal = true;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1299 goto Lfinalize;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1300
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1301 LscanInteger:
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1302 assert(*p != 0 && isdigit(*p));
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1303 isDecimal = true;
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1304 goto Lenter_loop_int;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1305 while (1)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1306 {
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1307 if (*++p == '_')
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1308 continue;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1309 if (!isdigit(*p))
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1310 break;
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1311 Lenter_loop_int:
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1312 if (ulong_ < ulong.max/10 || (ulong_ == ulong.max/10 && *p <= '5'))
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1313 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1314 ulong_ *= 10;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1315 ulong_ += *p - '0';
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1316 continue;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1317 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1318 // Overflow: skip following digits.
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1319 overflow = true;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1320 while (isdigit(*++p)) {}
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1321 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1322 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1323
61
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1324 // The number could be a float, so check overflow below.
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1325 switch (*p)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1326 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1327 case '.':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1328 if (p[1] != '.')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1329 goto LscanReal;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1330 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1331 case 'L':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1332 if (p[1] != 'i')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1333 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1334 case 'i', 'f', 'F', 'e', 'E':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1335 goto LscanReal;
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1336 default:
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1337 }
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1338
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1339 if (overflow)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1340 error(MID.OverflowDecimalNumber);
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1341
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1342 assert((isdigit(p[-1]) || p[-1] == '_') && !isdigit(*p) && *p != '_');
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1343 goto Lfinalize;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1344
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1345 LscanHex:
61
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1346 assert(digits == 0);
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1347 assert(*p == 'x');
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1348 while (1)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1349 {
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1350 if (*++p == '_')
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1351 continue;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1352 if (!ishexad(*p))
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1353 break;
61
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1354 ++digits;
57
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
1355 ulong_ *= 16;
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
1356 if (*p <= '9')
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
1357 ulong_ += *p - '0';
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
1358 else if (*p <= 'F')
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
1359 ulong_ += *p - 'A' + 10;
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
1360 else
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
1361 ulong_ += *p - 'a' + 10;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1362 }
57
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
1363
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1364 switch (*p)
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1365 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1366 case '.':
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1367 if (p[1] != '.')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1368 goto LscanHexReal;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1369 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1370 case 'L':
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1371 if (p[1] != 'i')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1372 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1373 case 'i', 'p', 'P':
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1374 goto LscanHexReal;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1375 default:
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1376 }
61
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1377 if (digits == 0)
59
3e594725899a - Issuing error when no digits were found in hex and binary numbers.
aziz
parents: 58
diff changeset
1378 error(MID.NoDigitsInHexNumber);
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1379 else if (digits > 16)
61
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1380 {
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1381 // Overflow: skip following digits.
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1382 error(MID.OverflowHexNumber);
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1383 while (ishexad(*++p)) {}
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1384 }
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1385 goto Lfinalize;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1386 LscanHexReal:
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1387 return scanHexReal(t);
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1388
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1389 LscanBin:
57
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
1390 assert(digits == 0);
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1391 assert(*p == 'b');
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1392 while (1)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1393 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1394 if (*++p == '0')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1395 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1396 ++digits;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1397 ulong_ *= 2;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1398 }
367
dda55fae37de - ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents: 365
diff changeset
1399 else if (*p == '1')
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1400 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1401 ++digits;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1402 ulong_ *= 2;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1403 ulong_ += *p - '0';
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1404 }
367
dda55fae37de - ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents: 365
diff changeset
1405 else if (*p == '_')
dda55fae37de - ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents: 365
diff changeset
1406 continue;
dda55fae37de - ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents: 365
diff changeset
1407 else
dda55fae37de - ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents: 365
diff changeset
1408 break;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1409 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1410
59
3e594725899a - Issuing error when no digits were found in hex and binary numbers.
aziz
parents: 58
diff changeset
1411 if (digits == 0)
3e594725899a - Issuing error when no digits were found in hex and binary numbers.
aziz
parents: 58
diff changeset
1412 error(MID.NoDigitsInBinNumber);
3e594725899a - Issuing error when no digits were found in hex and binary numbers.
aziz
parents: 58
diff changeset
1413
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1414 if (digits > 64)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1415 error(MID.OverflowBinaryNumber);
367
dda55fae37de - ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents: 365
diff changeset
1416 assert(p[-1] == '0' || p[-1] == '1' || p[-1] == '_', p[-1] ~ "");
dda55fae37de - ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents: 365
diff changeset
1417 assert( !(*p == '0' || *p == '1' || *p == '_') );
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1418 goto Lfinalize;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1419
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1420 LscanOct:
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1421 assert(*p == '_' || isoctal(*p));
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1422 if (*p != '_')
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1423 goto Lenter_loop_oct;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1424 while (1)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1425 {
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1426 if (*++p == '_')
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1427 continue;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1428 if (!isoctal(*p))
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1429 break;
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1430 Lenter_loop_oct:
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1431 if (ulong_ < ulong.max/2 || (ulong_ == ulong.max/2 && *p <= '1'))
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1432 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1433 ulong_ *= 8;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1434 ulong_ += *p - '0';
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1435 ++p;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1436 continue;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1437 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1438 // Overflow: skip following digits.
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1439 overflow = true;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1440 while (isdigit(*++p)) {}
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1441 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1442 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1443
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1444 bool hasDecimalDigits;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1445 if (isdigit(*p))
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1446 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1447 hasDecimalDigits = true;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1448 while (isdigit(*++p)) {}
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1449 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1450
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1451 // The number could be a float, so check errors below.
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1452 switch (*p)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1453 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1454 case '.':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1455 if (p[1] != '.')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1456 goto LscanReal;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1457 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1458 case 'L':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1459 if (p[1] != 'i')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1460 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1461 case 'i', 'f', 'F', 'e', 'E':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1462 goto LscanReal;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1463 default:
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1464 }
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1465
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1466 if (hasDecimalDigits)
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1467 error(MID.OctalNumberHasDecimals);
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1468 if (overflow)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1469 error(MID.OverflowOctalNumber);
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1470 // goto Lfinalize;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1471
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1472 Lfinalize:
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1473 enum Suffix
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1474 {
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1475 None = 0,
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1476 Unsigned = 1,
60
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1477 Long = 2
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1478 }
60
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1479
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1480 Suffix suffix;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1481 while (1)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1482 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1483 switch (*p)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1484 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1485 case 'L':
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1486 if (suffix & Suffix.Long)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1487 break;
60
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1488 suffix |= Suffix.Long;
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1489 ++p;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1490 continue;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1491 case 'u', 'U':
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1492 if (suffix & Suffix.Unsigned)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1493 break;
60
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1494 suffix |= Suffix.Unsigned;
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1495 ++p;
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1496 continue;
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1497 default:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1498 break;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1499 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1500 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1501 }
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1502
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1503 switch (suffix)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1504 {
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1505 case Suffix.None:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1506 if (ulong_ & 0x8000000000000000)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1507 {
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1508 if (isDecimal)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1509 error(MID.OverflowDecimalSign);
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1510 t.type = TOK.Uint64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1511 }
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1512 else if (ulong_ & 0xFFFFFFFF00000000)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1513 t.type = TOK.Int64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1514 else if (ulong_ & 0x80000000)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1515 t.type = isDecimal ? TOK.Int64 : TOK.Uint32;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1516 else
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1517 t.type = TOK.Int32;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1518 break;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1519 case Suffix.Unsigned:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1520 if (ulong_ & 0xFFFFFFFF00000000)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1521 t.type = TOK.Uint64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1522 else
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1523 t.type = TOK.Uint32;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1524 break;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1525 case Suffix.Long:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1526 if (ulong_ & 0x8000000000000000)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1527 {
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1528 if (isDecimal)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1529 error(MID.OverflowDecimalSign);
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1530 t.type = TOK.Uint64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1531 }
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1532 else
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1533 t.type = TOK.Int64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1534 break;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1535 case Suffix.Unsigned | Suffix.Long:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1536 t.type = TOK.Uint64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1537 break;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1538 default:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1539 assert(0);
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1540 }
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1541 t.ulong_ = ulong_;
15
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
1542 t.end = p;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1543 return;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1544 LscanReal:
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1545 scanReal(t);
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1546 return;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1547 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1548
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1549 /*
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1550 FloatLiteral:= Float[fFL]?i?
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1551 Float:= DecFloat | HexFloat
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
1552 DecFloat:= ([0-9][0-9_]*[.][0-9_]*DecExponent?) | [.][0-9][0-9_]*DecExponent? | [0-9][0-9_]*DecExponent
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1553 DecExponent:= [eE][+-]?[0-9][0-9_]*
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1554 HexFloat:= 0[xX](HexDigits[.]HexDigits | [.][0-9a-zA-Z]HexDigits? | HexDigits)HexExponent
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1555 HexExponent:= [pP][+-]?[0-9][0-9_]*
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1556 */
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1557 void scanReal(ref Token t)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1558 {
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1559 if (*p == '.')
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1560 // This function was called by scan() or scanNumber().
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1561 while (isdigit(*++p) || *p == '_') {}
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1562 else
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1563 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1564 // This function was called by scanNumber().
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1565 debug switch (*p)
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1566 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1567 case 'L':
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1568 if (p[1] != 'i')
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1569 assert(0);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1570 case 'i', 'f', 'F', 'e', 'E': break;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1571 default: assert(0);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1572 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1573 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1574
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1575 // Scan exponent.
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1576 if (*p == 'e' || *p == 'E')
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1577 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1578 ++p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1579 if (*p == '-' || *p == '+')
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1580 ++p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1581 if (!isdigit(*p))
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1582 error(MID.FloatExponentDigitExpected);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1583 else
64
dd4c5a0e47dd - Improved while loop.
aziz
parents: 63
diff changeset
1584 while (isdigit(*++p) || *p == '_') {}
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1585 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1586
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1587 // Copy string to buffer ignoring underscores.
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1588 char[] buffer;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1589 char* end = p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1590 p = t.start;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1591 do
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1592 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1593 if (*p == '_')
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1594 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1595 ++p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1596 continue;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1597 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1598 buffer ~= *p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1599 ++p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1600 } while (p != end)
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1601 buffer ~= 0;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1602 finalizeFloat(t, buffer);
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1603 }
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1604
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1605 void scanHexReal(ref Token t)
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1606 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1607 assert(*p == '.' || *p == 'i' || *p == 'p' || *p == 'P' || (*p == 'L' && p[1] == 'i'));
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1608 MID mid;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1609 if (*p == '.')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1610 while (ishexad(*++p) || *p == '_') {}
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1611 if (*p != 'p' && *p != 'P')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1612 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1613 mid = MID.HexFloatExponentRequired;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1614 goto Lerr;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1615 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1616 // Copy mantissa to a buffer ignoring underscores.
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1617 char* end = p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1618 p = t.start;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1619 char[] buffer;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1620 do
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1621 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1622 if (*p == '_')
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1623 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1624 ++p;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1625 continue;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1626 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1627 buffer ~= *p;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1628 ++p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1629 } while (p != end)
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1630
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1631 assert(p == end && (*p == 'p' || *p == 'P'));
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1632 // Scan and copy the exponent.
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1633 buffer ~= 'p';
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1634 size_t bufflen = buffer.length;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1635 while (1)
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1636 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1637 if (*++p == '_')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1638 continue;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1639 if (isdigit(*p))
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1640 buffer ~= *p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1641 else
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1642 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1643 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1644 // When the buffer length hasn't changed, no digits were copied.
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1645 if (bufflen == buffer.length) {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1646 mid = MID.HexFloatMissingExpDigits;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1647 goto Lerr;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1648 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1649 buffer ~= 0; // Terminate for C functions.
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1650 finalizeFloat(t, buffer);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1651 return;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1652 Lerr:
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1653 t.type = TOK.Float32;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1654 t.end = p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1655 error(mid);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1656 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1657
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1658 void finalizeFloat(ref Token t, string buffer)
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1659 {
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1660 // Float number is well-formed. Check suffixes and do conversion.
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1661 switch (*p)
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1662 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1663 case 'f', 'F':
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1664 t.type = TOK.Float32;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1665 t.float_ = strtof(buffer.ptr, null);
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1666 ++p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1667 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1668 case 'L':
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1669 t.type = TOK.Float80;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1670 t.real_ = strtold(buffer.ptr, null);
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1671 ++p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1672 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1673 default:
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1674 t.type = TOK.Float64;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1675 t.double_ = strtod(buffer.ptr, null);
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1676 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1677 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1678 if (*p == 'i')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1679 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1680 ++p;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1681 t.type += 3; // Switch to imaginary counterpart.
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1682 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1683 if (getErrno == ERANGE)
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1684 error(MID.OverflowFloatNumber);
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1685 t.end = p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1686 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1687
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1688 /// Scan special token: #line Integer [Filespec] EndOfLine
360
b6a3755eba94 - Renamed scanSpecialToken() to scanSpecialTokenSequence().
aziz
parents: 350
diff changeset
1689 void scanSpecialTokenSequence(ref Token t)
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1690 {
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1691 assert(*p == '#');
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1692
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1693 t.type = TOK.HashLine;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1694
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1695 MID mid;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1696
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1697 ++p;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1698 if (p[0] != 'l' || p[1] != 'i' || p[2] != 'n' || p[3] != 'e')
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1699 {
370
ae4afb66768f - Renamed findModule() to findModulePath().
aziz
parents: 367
diff changeset
1700 mid = MID.ExpectedIdentifierSTLine;
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1701 goto Lerr;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1702 }
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1703 p += 3;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1704
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1705 enum State
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1706 { Number, Filespec, End }
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1707
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1708 State state;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1709
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1710 Loop:
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1711 while (1)
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1712 {
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1713 switch (*++p)
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1714 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1715 case '\r':
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1716 if (p[1] == '\n')
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1717 ++p;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1718 case '\n', 0, _Z_:
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1719 break Loop;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1720 case LS[0]:
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1721 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1722 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1723 ++p; ++p;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1724 break Loop;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1725 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1726 goto default;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1727 default:
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1728 if (isspace(*p))
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1729 continue;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1730 if (state == State.Number)
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1731 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1732 if (!isdigit(*p))
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1733 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1734 mid = MID.ExpectedNumberAfterSTLine;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1735 goto Lerr;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1736 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1737 t.line_num = new Token;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1738 scan(*t.line_num);
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1739 --p;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1740 state = State.Filespec;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1741 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1742 else if (state == State.Filespec)
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1743 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1744 if (*p != '"')
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1745 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1746 mid = MID.ExpectedFilespec;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1747 goto Lerr;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1748 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1749 t.line_filespec = new Token;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1750 t.line_filespec.start = p;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1751 t.line_filespec.type = TOK.Filespec;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1752 while (1)
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1753 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1754 switch (*++p)
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1755 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1756 case '"':
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1757 break;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1758 case LS[0]:
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1759 if (!(p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])))
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1760 goto default;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1761 case '\r', '\n', 0, _Z_:
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1762 mid = MID.UnterminatedFilespec;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1763 t.line_filespec.end = p;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1764 goto Lerr;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1765 default:
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1766 if (*p & 128)
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1767 decodeUTF8();
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1768 continue;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1769 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1770 break; // Exit loop.
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1771 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1772 auto start = t.line_filespec.start +1; // +1 skips '"'
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1773 t.line_filespec.str = start[0 .. p - start];
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1774 t.line_filespec.end = p + 1;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1775 state = State.End;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1776 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1777 else/+ if (state == State.End)+/
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1778 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1779 mid = MID.UnterminatedSpecialToken;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1780 goto Lerr;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1781 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1782 }
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1783 }
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1784
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1785 if (state == State.Number)
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1786 {
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1787 mid = MID.ExpectedNumberAfterSTLine;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1788 goto Lerr;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1789 }
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1790
371
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
1791 this.loc_old = this.loc;
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
1792 this.loc_hline = t.line_num.uint_ - 1;
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1793 if (t.line_filespec)
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1794 this.fileName = t.line_filespec.str;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1795 t.end = p;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1796
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1797 return;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1798 Lerr:
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1799 t.end = p;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1800 error(mid);
15
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
1801 }
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
1802
371
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
1803 uint errorLoc()
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
1804 {
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
1805 // ∆loc + line_num_of(#line)
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
1806 return this.loc - this.loc_old + this.loc_hline;
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
1807 }
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
1808
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
1809 dchar decodeUTF8()
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
1810 {
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1811 assert(*p & 128, "check for ASCII char before calling decodeUTF8().");
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
1812 size_t idx;
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1813 dchar d;
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1814 try
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1815 {
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1816 d = std.utf.decode(p[0 .. end-p], idx);
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1817 p += idx -1;
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1818 }
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1819 catch (UtfException e)
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1820 {
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1821 error(MID.InvalidUTF8Sequence);
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1822 // Skip to next valid utf-8 sequence
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1823 while (p < end && UTF8stride[*++p] != 0xFF) {}
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1824 --p;
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1825 assert(p < end);
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1826 }
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
1827 return d;
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
1828 }
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
1829
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
1830 void loadKeywords()
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
1831 {
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
1832 foreach(k; keywords)
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
1833 idtable[k.str] = k;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
1834 }
382
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1835 /+ // Not needed anymore because tokens are stored in a linked list.
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1836 struct State
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1837 {
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1838 Lexer lexer;
209
921d15b1beeb - Added members token and loc to State.
aziz
parents: 207
diff changeset
1839 Token token;
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1840 char* scanPointer;
209
921d15b1beeb - Added members token and loc to State.
aziz
parents: 207
diff changeset
1841 int loc;
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1842 string fileName;
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1843 size_t errorLen;
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1844 static State opCall(Lexer lx)
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1845 {
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1846 State s;
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1847 s.lexer = lx;
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1848 s.token = lx.token;
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1849 s.scanPointer = lx.p;
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1850 s.loc = lx.loc;
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1851 s.fileName = lx.fileName;
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1852 s.errorLen = lx.errors.length;
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1853 return s;
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1854 }
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1855 void restore()
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1856 {
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1857 lexer.p = scanPointer;
209
921d15b1beeb - Added members token and loc to State.
aziz
parents: 207
diff changeset
1858 lexer.token = token;
921d15b1beeb - Added members token and loc to State.
aziz
parents: 207
diff changeset
1859 lexer.loc = loc;
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1860 lexer.fileName = fileName;
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1861 lexer.errors = lexer.errors[0..errorLen];
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1862 }
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1863 }
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1864
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1865 State getState()
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1866 {
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1867 return State(this);
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1868 }
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1869 +/
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1870
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1871 private void scanNext(ref Token* t)
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1872 {
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1873 assert(t !is null);
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1874 if (t.next)
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1875 t = t.next;
344
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
1876 else if (t != this.tail)
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1877 {
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1878 Token* new_t = new Token;
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1879 scan(*new_t);
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1880 new_t.prev = t;
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1881 t.next = new_t;
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1882 t = new_t;
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1883 }
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1884 }
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1885
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1886 void peek(ref Token* t)
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1887 {
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1888 scanNext(t);
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1889 }
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1890
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1891 TOK nextToken()
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1892 {
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1893 scanNext(this.token);
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1894 return this.token.type;
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1895 }
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1896
66
8e84db78ad55 - Added support for variadic arguments in error messages.
aziz
parents: 65
diff changeset
1897 void error(MID id, ...)
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1898 {
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
1899 // if (reportErrors)
371
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
1900 errors ~= new Information(InfoType.Lexer, id, this.errorLoc, arguments(_arguments, _argptr));
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1901 }
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1902
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1903 unittest
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1904 {
382
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1905 writefln("Testing method Lexer.peek()");
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1906 string sourceText = "unittest { }";
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1907 auto lx = new Lexer(sourceText, null);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1908
350
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
1909 Token* next = lx.head;
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1910 lx.peek(next);
350
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
1911 assert(next.type == TOK.Unittest);
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1912 lx.peek(next);
350
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
1913 assert(next.type == TOK.LBrace);
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1914 lx.peek(next);
350
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
1915 assert(next.type == TOK.RBrace);
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1916 lx.peek(next);
350
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
1917 assert(next.type == TOK.EOF);
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1918 }
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1919
306
9c866aadcb5b - Moved code out of main() to separate functions.
aziz
parents: 272
diff changeset
1920 Token* getTokens()
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
1921 {
306
9c866aadcb5b - Moved code out of main() to separate functions.
aziz
parents: 272
diff changeset
1922 while (nextToken() != TOK.EOF)
9c866aadcb5b - Moved code out of main() to separate functions.
aziz
parents: 272
diff changeset
1923 {}
9c866aadcb5b - Moved code out of main() to separate functions.
aziz
parents: 272
diff changeset
1924 return head;
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
1925 }
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1926
365
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
1927 static bool isNonReservedIdentifier(char[] ident)
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
1928 {
382
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1929 if (ident.length == 0)
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1930 return false;
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1931
365
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
1932 static Identifier[string] reserved_ids_table;
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
1933 if (reserved_ids_table is null)
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
1934 foreach(k; keywords)
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
1935 reserved_ids_table[k.str] = k;
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
1936
382
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1937 size_t idx = 1; // Index to the 2nd character in ident.
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1938 dchar isFirstCharUniAlpha()
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1939 {
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1940 idx = 0;
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1941 // NB: decode() could throw an Exception which would be
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1942 // caught by the next try-catch-block.
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1943 return isUniAlpha(std.utf.decode(ident, idx));
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1944 }
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1945
365
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
1946 try
382
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1947 {
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1948 if (isidbeg(ident[0]) ||
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1949 ident[0] & 128 && isFirstCharUniAlpha())
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1950 {
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1951 foreach (dchar c; ident[idx..$])
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1952 if (!isident(c) && !isUniAlpha(c))
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1953 return false;
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1954 }
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1955 }
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
1956 catch (Exception)
365
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
1957 return false;
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
1958
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
1959 return !(ident in reserved_ids_table);
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
1960 }
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
1961
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1962 private void encodeUTF8(inout char[] str, dchar d)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1963 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1964 char[6] b;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1965 assert(d > 0x7F, "check for ASCII char before calling encodeUTF8().");
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1966 if (d < 0x800)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1967 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1968 b[0] = 0xC0 | (d >> 6);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1969 b[1] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1970 str ~= b[0..2];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1971 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1972 else if (d < 0x10000)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1973 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1974 b[0] = 0xE0 | (d >> 12);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1975 b[1] = 0x80 | ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1976 b[2] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1977 str ~= b[0..3];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1978 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1979 else if (d < 0x200000)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1980 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1981 b[0] = 0xF0 | (d >> 18);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1982 b[1] = 0x80 | ((d >> 12) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1983 b[2] = 0x80 | ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1984 b[3] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1985 str ~= b[0..4];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1986 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1987 else if (d < 0x4000000)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1988 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1989 b[0] = 0xF8 | (d >> 24);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1990 b[1] = 0x80 | ((d >> 18) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1991 b[2] = 0x80 | ((d >> 12) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1992 b[3] = 0x80 | ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1993 b[4] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1994 str ~= b[0..5];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1995 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1996 else if (d < 0x80000000)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1997 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1998 b[0] = 0xFC | (d >> 30);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1999 b[1] = 0x80 | ((d >> 24) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2000 b[2] = 0x80 | ((d >> 18) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2001 b[3] = 0x80 | ((d >> 12) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2002 b[4] = 0x80 | ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2003 b[5] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2004 str ~= b[0..6];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2005 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2006 else
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2007 error(MID.InvalidUnicodeCharacter);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2008 }
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
2009 }
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2010
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2011 unittest
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2012 {
350
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2013 writefln("Testing Lexer.");
40
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
2014 string[] toks = [
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
2015 ">", ">=", ">>", ">>=", ">>>", ">>>=", "<", "<=", "<>",
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
2016 "<>=", "<<", "<<=", "!", "!<", "!>", "!<=", "!>=", "!<>",
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
2017 "!<>=", ".", "..", "...", "&", "&&", "&=", "+", "++",
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
2018 "+=", "-", "--", "-=", "=", "==", "~", "~=", "*",
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
2019 "*=", "/", "/=", "^", "^=", "%", "%=", "(", ")",
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
2020 "[", "]", "{", "}", ":", ";", "?", ",", "$"
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
2021 ];
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2022
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2023 char[] src;
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2024
40
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
2025 foreach (op; toks)
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2026 src ~= op ~ " ";
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2027
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2028 auto lx = new Lexer(src, "");
350
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2029 auto token = lx.getTokens();
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2030
350
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2031 uint i;
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2032 assert(token == lx.head);
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2033 token = token.next;
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2034 do
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2035 {
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2036 assert(i < toks.length);
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2037 assert(token.srcText == toks[i], std.string.format("Scanned '%s' but expected '%s'", token.srcText, toks[i]));
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2038 ++i;
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2039 token = token.next;
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2040 } while (token.type != TOK.EOF)
41
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
2041 }
55
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2042
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
2043 unittest
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
2044 {
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
2045 // Numbers unittest
68
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
2046 // 0L 0ULi 0_L 0_UL 0x0U 0x0p2 0_Fi 0_e2 0_F 0_i
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
2047 // 0u 0U 0uL 0UL 0L 0LU 0Lu
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
2048 // 0Li 0f 0F 0fi 0Fi 0i
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
2049 // 0b_1_LU 0b1000u
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
2050 // 0x232Lu
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
2051 }
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
2052
55
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2053 /// ASCII character properties table.
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2054 static const int ptable[256] = [
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2055 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2056 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2057 32, 0, 0x2200, 0, 0, 0, 0, 0x2700, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2058 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0x3f00,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2059 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2060 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0x5c00, 0, 0,16,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2061 0, 0x70c, 0x80c,12,12,12, 0xc0c, 8, 8, 8, 8, 8, 8, 8, 0xa08, 8,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2062 8, 8, 0xd08, 8, 0x908, 8, 0xb08, 8, 8, 8, 8, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2063 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2064 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2065 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2066 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2067 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2068 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2069 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2070 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2071 ];
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2072
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2073 enum CProperty
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2074 {
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2075 Octal = 1,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2076 Digit = 1<<1,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2077 Hex = 1<<2,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2078 Alpha = 1<<3,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2079 Underscore = 1<<4,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2080 Whitespace = 1<<5
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2081 }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2082
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2083 const uint EVMask = 0xFF00; // Bit mask for escape value
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2084
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2085 private alias CProperty CP;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2086 int isoctal(char c) { return ptable[c] & CP.Octal; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2087 int isdigit(char c) { return ptable[c] & CP.Digit; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2088 int ishexad(char c) { return ptable[c] & CP.Hex; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2089 int isalpha(char c) { return ptable[c] & CP.Alpha; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2090 int isalnum(char c) { return ptable[c] & (CP.Alpha | CP.Digit); }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2091 int isidbeg(char c) { return ptable[c] & (CP.Alpha | CP.Underscore); }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2092 int isident(char c) { return ptable[c] & (CP.Alpha | CP.Underscore | CP.Digit); }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2093 int isspace(char c) { return ptable[c] & CP.Whitespace; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2094 int char2ev(char c) { return ptable[c] >> 8; /*(ptable[c] & EVMask) >> 8;*/ }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2095
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2096 version(gen_ptable)
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2097 static this()
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2098 {
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2099 alias ptable p;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2100 // Initialize character properties table.
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2101 for (int i; i < p.length; ++i)
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2102 {
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2103 p[i] = 0;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2104 if ('0' <= i && i <= '7')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2105 p[i] |= CP.Octal;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2106 if ('0' <= i && i <= '9')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2107 p[i] |= CP.Digit;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2108 if (isdigit(i) || 'a' <= i && i <= 'f' || 'A' <= i && i <= 'F')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2109 p[i] |= CP.Hex;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2110 if ('a' <= i && i <= 'z' || 'A' <= i && i <= 'Z')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2111 p[i] |= CP.Alpha;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2112 if (i == '_')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2113 p[i] |= CP.Underscore;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2114 if (i == ' ' || i == '\t' || i == '\v' || i == '\f')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2115 p[i] |= CP.Whitespace;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2116 }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2117 // Store escape sequence values in second byte.
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2118 assert(CProperty.max <= ubyte.max, "character property flags and escape value byte overlap.");
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2119 p['\''] |= 39 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2120 p['"'] |= 34 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2121 p['?'] |= 63 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2122 p['\\'] |= 92 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2123 p['a'] |= 7 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2124 p['b'] |= 8 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2125 p['f'] |= 12 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2126 p['n'] |= 10 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2127 p['r'] |= 13 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2128 p['t'] |= 9 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2129 p['v'] |= 11 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2130 // Print a formatted array literal.
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2131 char[] array = "[\n";
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2132 for (int i; i < p.length; ++i)
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2133 {
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2134 int c = p[i];
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2135 array ~= std.string.format(c>255?" 0x%x,":"%2d,", c, ((i+1) % 16) ? "":"\n");
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2136 }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2137 array[$-2..$] = "\n]";
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2138 writefln(array);
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2139 }