Mercurial > projects > dil

annotate trunk/src/Lexer.d @ 223:043722eed46e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
- Fix: if-statements should be while-statements.
author aziz
date Thu, 26 Jul 2007 18:54:01 +0000
parents f26a2beb745e
children 7911f6a92e6e
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
1 /++
8ba2570de175 Initial import.
aziz
parents:
diff changeset
2 Author: Aziz Köksal
8ba2570de175 Initial import.
aziz
parents:
diff changeset
3 License: GPL2
8ba2570de175 Initial import.
aziz
parents:
diff changeset
4 +/
8ba2570de175 Initial import.
aziz
parents:
diff changeset
5 module Lexer;
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
6 import Token;
69
24db7c5522d5 - Added module Information for compiler messages like warnings, info and errors to the user.
aziz
parents: 68
diff changeset
7 import Information;
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
8 import Keywords;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
9 import Identifier;
54
e55bd2270f94 - Relocated messages table to a separate module.
aziz
parents: 53
diff changeset
10 import Messages;
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
11 import std.stdio;
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
12 import std.utf;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
13 import std.uni;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
14 import std.c.stdlib;
66
8e84db78ad55 - Added support for variadic arguments in error messages.
aziz
parents: 65
diff changeset
15 import std.string;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
16
11
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
17 const char[3] LS = \u2028;
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
18 const char[3] PS = \u2029;
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
19
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
20 const dchar LSd = 0x2028;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
21 const dchar PSd = 0x2029;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
22
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
23 const uint _Z_ = 26; /// Control+Z
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
24
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
25 class Lexer
8ba2570de175 Initial import.
aziz
parents:
diff changeset
26 {
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
27 Token token;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
28 string text;
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
29 char* p;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
30 char* end;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
31
17
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
32 uint loc = 1; /// line of code
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
33
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
34 char[] fileName;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
35
69
24db7c5522d5 - Added module Information for compiler messages like warnings, info and errors to the user.
aziz
parents: 68
diff changeset
36 Information[] errors;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
37
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
38 bool reportErrors;
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
39
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
40 Identifier[string] idtable;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
41
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
42 this(string text, string fileName)
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
43 {
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
44 this.fileName = fileName;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
45
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
46 this.text = text;
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
47 if (text[$-1] != 0)
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
48 {
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
49 this.text.length = this.text.length + 1;
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
50 this.text[$-1] = 0;
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
51 }
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
52
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
53 this.p = this.text.ptr;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
54 this.end = this.p + this.text.length;
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
55 this.reportErrors = true;
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
56 loadKeywords();
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
57 }
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
58
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
59 public void scan(out Token t)
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
60 in
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
61 {
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
62 assert(text.ptr <= p && p < end);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
63 }
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
64 out
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
65 {
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
66 assert(text.ptr <= t.start && t.start < end);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
67 assert(text.ptr < t.end && t.end <= end, std.string.format(t.type));
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
68 }
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
69 body
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
70 {
10
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
71 uint c = *p;
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
72
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
73 while (1)
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
74 {
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
75 t.start = p;
16
476e8e55c1d4 - Added Whitespace to the character properties table.
aziz
parents: 15
diff changeset
76
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
77 if (c == 0 || c == _Z_)
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
78 {
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
79 assert(*p == 0 || *p == _Z_);
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
80 t.type = TOK.EOF;
17
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
81 t.end = p;
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
82 assert(t.start == t.end);
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
83 return;
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
84 }
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
85
17
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
86 if (c == '\n')
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
87 {
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
88 c = *++p;
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
89 ++loc;
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
90 continue;
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
91 }
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
92 else if (c == '\r')
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
93 {
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
94 c = *++p;
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
95 if (c != '\n')
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
96 ++loc;
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
97 continue;
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
98 }
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
99 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
100 {
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
101 p += 3;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
102 c = *p;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
103 continue;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
104 }
17
9bd0bac79479 - Removed Whitespace from enum list.
aziz
parents: 16
diff changeset
105
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
106 if (isidbeg(c))
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
107 {
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
108 if (c == 'r' && p[1] == '"' && ++p)
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
109 return scanRawStringLiteral(t);
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
110 if (c == 'x' && p[1] == '"')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
111 return scanHexStringLiteral(t);
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
112 Lidentifier:
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
113 do
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
114 { c = *++p; }
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
115 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8()))
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
116
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
117 t.end = p;
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
118
65
6c21ae79fbb3 - Renamed function Token.span to Token.srcText.
aziz
parents: 64
diff changeset
119 string str = t.srcText;
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
120 Identifier* id = str in idtable;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
121
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
122 if (!id)
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
123 {
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
124 idtable[str] = Identifier.Identifier(TOK.Identifier, str);
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
125 id = str in idtable;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
126 }
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
127 assert(id);
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
128 t.type = id.type;
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
129 return;
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
130 }
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
131
15
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
132 if (isdigit(c))
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
133 return scanNumber(t);
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
134
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
135 if (c == '/')
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
136 {
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
137 c = *++p;
14
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
138 switch(c)
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
139 {
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
140 case '=':
14
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
141 ++p;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
142 t.type = TOK.DivAssign;
14
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
143 t.end = p;
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
144 return;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
145 case '+':
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
146 uint level = 1;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
147 while (1)
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
148 {
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
149 c = *++p;
42
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
150 LswitchNC: // only jumped to from default case of next switch(c)
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
151 switch (c)
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
152 {
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
153 case '\r':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
154 if (p[1] == '\n')
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
155 ++p;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
156 case '\n':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
157 ++loc;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
158 continue;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
159 case 0, _Z_:
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
160 error(MID.UnterminatedNestedComment);
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
161 goto LreturnNC;
42
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
162 default:
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
163 }
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
164
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
165 c <<= 8;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
166 c |= *++p;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
167 switch (c)
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
168 {
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
169 case 0x2F2B: // /+
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
170 ++level;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
171 continue;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
172 case 0x2B2F: // +/
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
173 if (--level == 0)
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
174 {
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
175 ++p;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
176 LreturnNC:
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
177 t.type = TOK.Comment;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
178 t.end = p;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
179 return;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
180 }
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
181 continue;
42
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
182 case 0xE280: // LS[0..1] || PS[0..1]
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
183 if (p[1] == LS[2] || p[1] == PS[2])
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
184 {
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
185 ++loc;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
186 ++p;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
187 }
43
1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents: 42
diff changeset
188 continue;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
189 default:
42
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
190 c &= char.max;
c6a9974a6e3c - Optimized scanner of nested comments.
aziz
parents: 41
diff changeset
191 goto LswitchNC;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
192 }
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
193 }
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
194 case '*':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
195 while (1)
7
07e45c06a024 - Parsing nested comments correctly now.
aziz
parents: 5
diff changeset
196 {
43
1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents: 42
diff changeset
197 c = *++p;
1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents: 42
diff changeset
198 LswitchBC: // only jumped to from default case of next switch(c)
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
199 switch (c)
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
200 {
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
201 case '\r':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
202 if (p[1] == '\n')
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
203 ++p;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
204 case '\n':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
205 ++loc;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
206 continue;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
207 case 0, _Z_:
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
208 error(MID.UnterminatedBlockComment);
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
209 goto LreturnBC;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
210 default:
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
211 }
41
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
212
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
213 c <<= 8;
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
214 c |= *++p;
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
215 switch (c)
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
216 {
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
217 case 0x2A2F: // */
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
218 ++p;
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
219 LreturnBC:
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
220 t.type = TOK.Comment;
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
221 t.end = p;
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
222 return;
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
223 case 0xE280: // LS[0..1] || PS[0..1]
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
224 if (p[1] == LS[2] || p[1] == PS[2])
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
225 {
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
226 ++loc;
43
1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents: 42
diff changeset
227 ++p;
41
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
228 }
43
1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents: 42
diff changeset
229 continue;
41
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
230 default:
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
231 c &= char.max;
43
1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments.
aziz
parents: 42
diff changeset
232 goto LswitchBC;
41
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
233 }
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
234 }
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
235 assert(0);
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
236 case '/':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
237 while (1)
7
07e45c06a024 - Parsing nested comments correctly now.
aziz
parents: 5
diff changeset
238 {
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
239 c = *++p;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
240 switch (c)
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
241 {
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
242 case '\r':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
243 if (p[1] == '\n')
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
244 ++p;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
245 case '\n':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
246 case 0, _Z_:
11
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
247 break;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
248 case LS[0]:
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
249 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
250 break;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
251 continue;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
252 default:
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
253 continue;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
254 }
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
255 t.type = TOK.Comment;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
256 t.end = p;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
257 return;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
258 }
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
259 default:
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
260 t.type = TOK.Div;
10
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
261 t.end = p;
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
262 return;
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
263 }
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
264 }
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
265
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
266 switch (c)
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
267 {
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
268 case '\'':
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
269 return scanCharacterLiteral(t);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
270 case '`':
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
271 return scanRawStringLiteral(t);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
272 case '"':
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
273 return scanNormalStringLiteral(t);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
274 case '\\':
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
275 char[] buffer;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
276 do
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
277 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
278 ++p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
279 c = scanEscapeSequence();
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
280 if (c < 128)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
281 buffer ~= c;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
282 else
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
283 encodeUTF8(buffer, c);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
284 } while (*p == '\\')
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
285 buffer ~= 0;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
286 t.type = TOK.String;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
287 t.str = buffer;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
288 t.end = p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
289 return;
38
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
290 case '>': /* > >= >> >>= >>> >>>= */
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
291 c = *++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
292 switch (c)
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
293 {
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
294 case '=':
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
295 t.type = TOK.GreaterEqual;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
296 goto Lcommon;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
297 case '>':
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
298 if (p[1] == '>')
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
299 {
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
300 ++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
301 if (p[1] == '=')
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
302 { ++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
303 t.type = TOK.URShiftAssign;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
304 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
305 else
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
306 t.type = TOK.URShift;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
307 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
308 else if (p[1] == '=')
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
309 {
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
310 ++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
311 t.type = TOK.RShiftAssign;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
312 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
313 else
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
314 t.type = TOK.RShift;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
315 goto Lcommon;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
316 default:
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
317 t.type = TOK.Greater;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
318 goto Lcommon2;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
319 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
320 assert(0);
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
321 case '<': /* < <= <> <>= << <<= */
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
322 c = *++p;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
323 switch (c)
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
324 {
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
325 case '=':
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
326 t.type = TOK.LessEqual;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
327 goto Lcommon;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
328 case '<':
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
329 if (p[1] == '=') {
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
330 ++p;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
331 t.type = TOK.LShiftAssign;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
332 }
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
333 else
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
334 t.type = TOK.LShift;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
335 goto Lcommon;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
336 case '>':
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
337 if (p[1] == '=') {
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
338 ++p;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
339 t.type = TOK.LorEorG;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
340 }
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
341 else
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
342 t.type = TOK.LorG;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
343 goto Lcommon;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
344 default:
38
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
345 t.type = TOK.Less;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
346 goto Lcommon2;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
347 }
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
348 assert(0);
37
7f3bcb97d017 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 36
diff changeset
349 case '!': /* ! !< !> !<= !>= !<> !<>= */
35
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
350 c = *++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
351 switch (c)
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
352 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
353 case '<':
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
354 c = *++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
355 if (c == '>')
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
356 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
357 if (p[1] == '=') {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
358 ++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
359 t.type = TOK.Unordered;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
360 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
361 else
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
362 t.type = TOK.UorE;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
363 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
364 else if (c == '=')
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
365 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
366 t.type = TOK.UorG;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
367 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
368 else {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
369 t.type = TOK.UorGorE;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
370 goto Lcommon2;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
371 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
372 goto Lcommon;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
373 case '>':
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
374 if (p[1] == '=')
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
375 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
376 ++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
377 t.type = TOK.UorL;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
378 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
379 else
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
380 t.type = TOK.UorLorE;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
381 goto Lcommon;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
382 case '=':
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
383 t.type = TOK.NotEqual;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
384 goto Lcommon;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
385 default:
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
386 t.type = TOK.Not;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
387 goto Lcommon2;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
388 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
389 assert(0);
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
390 case '.': /* . .[0-9] .. ... */
22
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
391 if (p[1] == '.')
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
392 {
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
393 ++p;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
394 if (p[1] == '.') {
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
395 ++p;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
396 t.type = TOK.Ellipses;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
397 }
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
398 else
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
399 t.type = TOK.Slice;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
400 }
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
401 else if (isdigit(p[1]))
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
402 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
403 return scanReal(t);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
404 }
22
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
405 else
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
406 t.type = TOK.Dot;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
407 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
408 case '|': /* | || |= */
23
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
409 c = *++p;
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
410 if (c == '=')
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
411 t.type = TOK.OrAssign;
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
412 else if (c == '|')
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
413 t.type = TOK.OrLogical;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
414 else {
23
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
415 t.type = TOK.OrBinary;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
416 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
417 }
23
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
418 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
419 case '&': /* & && &= */
24
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
420 c = *++p;
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
421 if (c == '=')
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
422 t.type = TOK.AndAssign;
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
423 else if (c == '&')
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
424 t.type = TOK.AndLogical;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
425 else {
24
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
426 t.type = TOK.AndBinary;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
427 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
428 }
24
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
429 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
430 case '+': /* + ++ += */
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
431 c = *++p;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
432 if (c == '=')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
433 t.type = TOK.PlusAssign;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
434 else if (c == '+')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
435 t.type = TOK.PlusPlus;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
436 else {
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
437 t.type = TOK.Plus;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
438 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
439 }
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
440 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
441 case '-': /* - -- -= */
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
442 c = *++p;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
443 if (c == '=')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
444 t.type = TOK.MinusAssign;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
445 else if (c == '-')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
446 t.type = TOK.MinusMinus;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
447 else {
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
448 t.type = TOK.Minus;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
449 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
450 }
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
451 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
452 case '=': /* = == */
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
453 if (p[1] == '=') {
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
454 ++p;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
455 t.type = TOK.Equal;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
456 }
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
457 else
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
458 t.type = TOK.Assign;
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
459 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
460 case '~': /* ~ ~= */
27
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
461 if (p[1] == '=') {
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
462 ++p;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
463 t.type = TOK.CatAssign;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
464 }
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
465 else
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
466 t.type = TOK.Tilde;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
467 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
468 case '*': /* * *= */
29
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
469 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
470 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
471 t.type = TOK.MulAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
472 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
473 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
474 t.type = TOK.Mul;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
475 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
476 case '^': /* ^ ^= */
29
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
477 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
478 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
479 t.type = TOK.XorAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
480 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
481 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
482 t.type = TOK.Xor;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
483 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
484 case '%': /* % %= */
29
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
485 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
486 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
487 t.type = TOK.ModAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
488 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
489 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
490 t.type = TOK.Mod;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
491 goto Lcommon;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
492 // Single character tokens:
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
493 case '(':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
494 t.type = TOK.LParen;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
495 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
496 case ')':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
497 t.type = TOK.RParen;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
498 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
499 case '[':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
500 t.type = TOK.LBracket;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
501 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
502 case ']':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
503 t.type = TOK.RBracket;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
504 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
505 case '{':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
506 t.type = TOK.LBrace;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
507 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
508 case '}':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
509 t.type = TOK.RBrace;
21
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
510 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
511 case ':':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
512 t.type = TOK.Colon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
513 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
514 case ';':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
515 t.type = TOK.Semicolon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
516 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
517 case '?':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
518 t.type = TOK.Question;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
519 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
520 case ',':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
521 t.type = TOK.Comma;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
522 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
523 case '$':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
524 t.type = TOK.Dollar;
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
525 Lcommon:
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
526 ++p;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
527 Lcommon2:
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
528 t.end = p;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
529 return;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
530 case '#':
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
531 scanSpecialToken();
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
532 c = *p;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
533 continue;
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
534 default:
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
535 }
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
536
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
537 if (c & 128 && isUniAlpha(decodeUTF8()))
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
538 goto Lidentifier;
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
539 c = *++p;
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
540 }
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
541 }
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
542
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
543 void scanNormalStringLiteral(ref Token t)
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
544 {
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
545 assert(*p == '"');
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
546 ++p;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
547 char[] buffer;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
548 t.type = TOK.String;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
549 while (1)
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
550 {
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
551 switch (*p)
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
552 {
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
553 case '"':
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
554 ++p;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
555 Lreturn:
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
556 buffer ~= 0;
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
557 t.str = buffer;
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
558 t.pf = scanPostfix();
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
559 t.end = p;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
560 return;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
561 case '\\':
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
562 ++p;
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
563 dchar d = scanEscapeSequence();
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
564 if (d < 128)
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
565 buffer ~= d;
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
566 else
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
567 encodeUTF8(buffer, d);
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
568 continue;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
569 case '\r':
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
570 if (p[1] == '\n')
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
571 ++p;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
572 case '\n':
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
573 ++p;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
574 ++loc;
221
f26a2beb745e - Moved ++p below if-statement.
aziz
parents: 215
diff changeset
575 buffer ~= '\n'; // Convert EndOfLine to \n.
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
576 continue;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
577 case 0, _Z_:
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
578 error(MID.UnterminatedString);
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
579 goto Lreturn;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
580 default:
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
581 if (*p & 128)
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
582 {
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
583 // char* begin = p;
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
584 dchar d = decodeUTF8();
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
585 if (d == LSd || d == PSd)
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
586 goto case '\n';
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
587
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
588 // We don't copy per pointer because we might include
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
589 // invalid, skipped utf-8 sequences. See decodeUTF8().
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
590 // ++p;
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
591 // buffer ~= begin[0 .. p - begin];
221
f26a2beb745e - Moved ++p below if-statement.
aziz
parents: 215
diff changeset
592 ++p;
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
593 encodeUTF8(buffer, d);
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
594 continue;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
595 }
223
043722eed46e - Fix: if-statements should be while-statements.
aziz
parents: 221
diff changeset
596 // Copy ASCII character.
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
597 buffer ~= *p++;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
598 }
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
599 }
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
600 assert(0);
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
601 }
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
602
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
603 void scanCharacterLiteral(ref Token t)
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
604 {
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
605 assert(*p == '\'');
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
606 MID id = MID.UnterminatedCharacterLiteral;
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
607 ++p;
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
608 TOK type = TOK.CharLiteral;
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
609 switch (*p)
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
610 {
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
611 case '\\':
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
612 ++p;
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
613 switch (*p)
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
614 {
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
615 case 'u':
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
616 type = TOK.WCharLiteral; break;
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
617 case 'U':
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
618 type = TOK.DCharLiteral; break;
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
619 default:
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
620 }
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
621 t.dchar_ = scanEscapeSequence();
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
622 break;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
623 case '\'':
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
624 ++p;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
625 id = MID.EmptyCharacterLiteral;
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
626 case '\n', '\r', 0, _Z_:
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
627 goto Lerr;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
628 default:
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
629 uint c = *p;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
630 if (c & 128)
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
631 {
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
632 c = decodeUTF8();
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
633 if (c == LSd || c == PSd)
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
634 goto Lerr;
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
635 if (c <= 0xFFFF)
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
636 type = TOK.WCharLiteral;
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
637 else
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
638 type = TOK.DCharLiteral;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
639 }
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
640 t.dchar_ = c;
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
641 ++p;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
642 }
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
643
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
644 if (*p == '\'')
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
645 ++p;
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
646 else
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
647 Lerr:
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
648 error(id);
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
649 t.type = type;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
650 t.end = p;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
651 }
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
652
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
653 char scanPostfix()
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
654 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
655 switch (*p)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
656 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
657 case 'c':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
658 case 'w':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
659 case 'd':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
660 return *p++;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
661 default:
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
662 return 0;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
663 }
112
004d98df65af - Implemented parseInterfaceDeclaration().
aziz
parents: 105
diff changeset
664 assert(0);
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
665 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
666
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
667 void scanRawStringLiteral(ref Token t)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
668 {
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
669 uint delim = *p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
670 assert(delim == '`' || delim == '"' && p[-1] == 'r');
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
671 t.type = TOK.String;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
672 char[] buffer;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
673 uint c;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
674 while (1)
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
675 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
676 c = *++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
677 switch (c)
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
678 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
679 case '\r':
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
680 if (p[1] == '\n')
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
681 ++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
682 c = '\n'; // Convert '\r' and '\r\n' to '\n'
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
683 case '\n':
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
684 ++loc;
52
f65a83c27638 - Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents: 51
diff changeset
685 break;
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
686 case '`':
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
687 case '"':
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
688 if (c == delim)
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
689 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
690 ++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
691 t.pf = scanPostfix();
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
692 Lreturn:
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
693 t.str = buffer ~ '\0';
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
694 t.end = p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
695 return;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
696 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
697 break;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
698 case LS[0]:
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
699 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
700 {
52
f65a83c27638 - Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents: 51
diff changeset
701 c = '\n';
f65a83c27638 - Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents: 51
diff changeset
702 ++p; ++p;
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
703 ++loc;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
704 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
705 break;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
706 case 0, _Z_:
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
707 if (delim == 'r')
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
708 error(MID.UnterminatedRawString);
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
709 else
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
710 error(MID.UnterminatedBackQuoteString);
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
711 goto Lreturn;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
712 default:
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
713 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
714 buffer ~= c; // copy character to buffer
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
715 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
716 assert(0);
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
717 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
718
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
719 void scanHexStringLiteral(ref Token t)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
720 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
721 assert(p[0] == 'x' && p[1] == '"');
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
722 t.type = TOK.String;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
723
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
724 uint c;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
725 ubyte[] buffer;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
726 ubyte h; // hex number
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
727 uint n; // number of hex digits
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
728
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
729 ++p;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
730 while (1)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
731 {
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
732 c = *++p;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
733 switch (c)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
734 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
735 case '"':
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
736 ++p;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
737 if (n & 1)
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
738 error(MID.OddNumberOfDigitsInHexString);
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
739 t.pf = scanPostfix();
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
740 Lreturn:
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
741 buffer ~= 0;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
742 t.str = cast(string) buffer;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
743 t.end = p;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
744 return;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
745 case '\r':
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
746 if (p[1] == '\n')
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
747 ++p;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
748 case '\n':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
749 ++loc;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
750 continue;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
751 default:
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
752 if (ishexad(c))
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
753 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
754 if (c <= '9')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
755 c -= '0';
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
756 else if (c <= 'F')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
757 c -= 'A' - 10;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
758 else
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
759 c -= 'a' - 10;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
760
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
761 if (n & 1)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
762 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
763 h <<= 4;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
764 h |= c;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
765 buffer ~= h;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
766 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
767 else
112
004d98df65af - Implemented parseInterfaceDeclaration().
aziz
parents: 105
diff changeset
768 h = cast(ubyte)c;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
769 ++n;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
770 continue;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
771 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
772 else if (isspace(c))
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
773 continue;
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
774
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
775 if (c >= 128)
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
776 {
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
777 c = decodeUTF8();
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
778 if (c == LSd || c == PSd)
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
779 {
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
780 ++p; ++p;
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
781 ++loc;
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
782 continue;
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
783 }
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
784 }
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
785 else if (c == 0 || c == _Z_)
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
786 {
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
787 error(MID.UnterminatedHexString);
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
788 t.pf = 0;
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
789 goto Lreturn;
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
790 }
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
791 error(MID.NonHexCharInHexString, cast(dchar)c);
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
792 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
793 }
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
794 assert(0);
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
795 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
796
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
797 dchar scanEscapeSequence()
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
798 {
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
799 uint c = char2ev(*p);
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
800 if (c)
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
801 {
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
802 ++p;
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
803 return c;
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
804 }
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
805 uint digits = 2;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
806
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
807 switch (*p)
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
808 {
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
809 case 'x':
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
810 c = 0;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
811 while (1)
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
812 {
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
813 ++p;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
814 if (ishexad(*p))
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
815 {
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
816 c *= 16;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
817 if (*p <= '9')
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
818 c += *p - '0';
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
819 else if (*p <= 'F')
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
820 c += *p - 'A' + 10;
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
821 else
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
822 c += *p - 'a' + 10;
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
823
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
824 if (!--digits)
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
825 {
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
826 ++p;
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
827 break;
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
828 }
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
829 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
830 else
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
831 {
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
832 error(MID.InsufficientHexDigits);
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
833 break;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
834 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
835 }
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
836 if (!isValidDchar(c))
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
837 error(MID.InvalidUnicodeCharacter);
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
838 break;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
839 case 'u':
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
840 digits = 4;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
841 goto case 'x';
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
842 case 'U':
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
843 digits = 8;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
844 goto case 'x';
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
845 default:
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
846 if (isoctal(*p))
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
847 {
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
848 c = 0;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
849 c += *p - '0';
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
850 ++p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
851 if (!isoctal(*p))
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
852 return c;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
853 c *= 8;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
854 c += *p - '0';
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
855 ++p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
856 if (!isoctal(*p))
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
857 return c;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
858 c *= 8;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
859 c += *p - '0';
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
860 ++p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
861 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
862 else if(*p == '&')
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
863 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
864 if (isalpha(*++p))
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
865 {
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
866 while (1)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
867 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
868 if (isalnum(*++p))
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
869 continue;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
870 if (*p == ';') {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
871 // TODO: convert entity to unicode codepoint.
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
872 ++p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
873 break;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
874 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
875 else {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
876 error(MID.UnterminatedHTMLEntity);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
877 break;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
878 }
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
879 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
880 }
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
881 else
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
882 error(MID.InvalidBeginHTMLEntity);
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
883 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
884 else
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
885 error(MID.UndefinedEscapeSequence);
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
886 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
887
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
888 return c;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
889 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
890
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
891 /*
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
892 IntegerLiteral:= (Dec|Hex|Bin|Oct)Suffix?
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
893 Dec:= (0|[1-9][0-9_]*)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
894 Hex:= 0[xX] HexDigits
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
895 Bin:= 0[bB][01_]+
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
896 Oct:= 0[0-7_]+
68
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
897 Suffix:= (L[uU]?|[uU]L?)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
898 HexDigits:= [0-9a-zA-Z_]+
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
899
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
900 Invalid: "0b_", "0x_", "._"
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
901 */
15
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
902 void scanNumber(ref Token t)
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
903 {
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
904 ulong ulong_;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
905 bool overflow;
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
906 bool isDecimal;
57
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
907 size_t digits;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
908
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
909 if (*p != '0')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
910 goto LscanInteger;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
911 ++p; // skip zero
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
912 // check for xX bB ...
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
913 switch (*p)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
914 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
915 case 'x','X':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
916 goto LscanHex;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
917 case 'b','B':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
918 goto LscanBin;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
919 case 'L':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
920 if (p[1] == 'i')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
921 goto LscanReal;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
922 case '.':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
923 if (p[1] == '.')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
924 break;
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
925 case 'i','f','F', 'e', 'E': // Imaginary and float literal suffix
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
926 goto LscanReal;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
927 case '_':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
928 ++p;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
929 goto LscanOct;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
930 default:
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
931 if (isoctal(*p))
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
932 goto LscanOct;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
933 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
934
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
935 ulong_ = p[-1];
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
936 isDecimal = true;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
937 goto Lfinalize;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
938
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
939 LscanInteger:
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
940 isDecimal = true;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
941 while (1)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
942 {
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
943 if (*++p == '_')
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
944 continue;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
945 if (!isdigit(*p))
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
946 break;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
947 if (ulong_ < ulong.max/10 || (ulong_ == ulong.max/10 && *p <= '5'))
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
948 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
949 ulong_ *= 10;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
950 ulong_ += *p - '0';
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
951 continue;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
952 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
953 // Overflow: skip following digits.
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
954 overflow = true;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
955 while (isdigit(*++p)) {}
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
956 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
957 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
958
61
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
959 // The number could be a float, so check overflow below.
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
960 switch (*p)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
961 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
962 case '.':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
963 if (p[1] != '.')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
964 goto LscanReal;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
965 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
966 case 'L':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
967 if (p[1] != 'i')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
968 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
969 case 'i', 'f', 'F', 'e', 'E':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
970 goto LscanReal;
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
971 default:
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
972 }
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
973
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
974 if (overflow)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
975 error(MID.OverflowDecimalNumber);
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
976
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
977 assert((isdigit(p[-1]) || p[-1] == '_') && !isdigit(*p) && *p != '_');
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
978 goto Lfinalize;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
979
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
980 LscanHex:
61
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
981 assert(digits == 0);
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
982 while (1)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
983 {
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
984 if (*++p == '_')
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
985 continue;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
986 if (!ishexad(*p))
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
987 break;
61
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
988 ++digits;
57
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
989 ulong_ *= 16;
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
990 if (*p <= '9')
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
991 ulong_ += *p - '0';
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
992 else if (*p <= 'F')
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
993 ulong_ += *p - 'A' + 10;
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
994 else
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
995 ulong_ += *p - 'a' + 10;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
996 }
57
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
997
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
998 switch (*p)
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
999 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1000 case '.':
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1001 if (p[1] != '.')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1002 goto LscanHexReal;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1003 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1004 case 'L':
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1005 if (p[1] != 'i')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1006 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1007 case 'i', 'p', 'P':
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1008 goto LscanHexReal;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1009 default:
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1010 }
61
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1011 if (digits == 0)
59
3e594725899a - Issuing error when no digits were found in hex and binary numbers.
aziz
parents: 58
diff changeset
1012 error(MID.NoDigitsInHexNumber);
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1013 else if (digits > 16)
61
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1014 {
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1015 // Overflow: skip following digits.
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1016 error(MID.OverflowHexNumber);
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1017 while (ishexad(*++p)) {}
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1018 }
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1019 goto Lfinalize;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1020 LscanHexReal:
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1021 return scanHexReal(t);
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1022
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1023 LscanBin:
57
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
1024 assert(digits == 0);
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1025 while (1)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1026 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1027 if (*++p == '0')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1028 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1029 ++digits;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1030 ulong_ *= 2;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1031 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1032 if (*p == '1')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1033 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1034 ++digits;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1035 ulong_ *= 2;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1036 ulong_ += *p - '0';
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1037 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1038 if (*p == '_')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1039 continue;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1040 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1041 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1042
59
3e594725899a - Issuing error when no digits were found in hex and binary numbers.
aziz
parents: 58
diff changeset
1043 if (digits == 0)
3e594725899a - Issuing error when no digits were found in hex and binary numbers.
aziz
parents: 58
diff changeset
1044 error(MID.NoDigitsInBinNumber);
3e594725899a - Issuing error when no digits were found in hex and binary numbers.
aziz
parents: 58
diff changeset
1045
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1046 if (digits > 64)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1047 error(MID.OverflowBinaryNumber);
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1048 assert((p[-1] == '0' || p[-1] == '1' || p[-1] == '_') && !(*p == '0' || *p == '1' || *p == '_'));
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1049 goto Lfinalize;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1050
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1051 LscanOct:
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1052 while (1)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1053 {
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1054 if (*++p == '_')
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1055 continue;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1056 if (!isoctal(*p))
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1057 break;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1058 if (ulong_ < ulong.max/2 || (ulong_ == ulong.max/2 && *p <= '1'))
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1059 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1060 ulong_ *= 8;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1061 ulong_ += *p - '0';
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1062 ++p;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1063 continue;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1064 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1065 // Overflow: skip following digits.
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1066 overflow = true;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1067 while (isdigit(*++p)) {}
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1068 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1069 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1070
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1071 bool hasDecimalDigits;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1072 if (isdigit(*p))
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1073 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1074 hasDecimalDigits = true;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1075 while (isdigit(*++p)) {}
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1076 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1077
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1078 // The number could be a float, so check errors below.
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1079 switch (*p)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1080 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1081 case '.':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1082 if (p[1] != '.')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1083 goto LscanReal;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1084 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1085 case 'L':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1086 if (p[1] != 'i')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1087 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1088 case 'i', 'f', 'F', 'e', 'E':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1089 goto LscanReal;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1090 default:
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1091 }
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1092
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1093 if (hasDecimalDigits)
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1094 error(MID.OctalNumberHasDecimals);
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1095 if (overflow)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1096 error(MID.OverflowOctalNumber);
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1097 // goto Lfinalize;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1098
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1099 Lfinalize:
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1100 enum Suffix
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1101 {
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1102 None = 0,
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1103 Unsigned = 1,
60
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1104 Long = 2
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1105 }
60
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1106
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1107 Suffix suffix;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1108 while (1)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1109 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1110 switch (*p)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1111 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1112 case 'L':
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1113 if (suffix & Suffix.Long)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1114 break;
60
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1115 suffix |= Suffix.Long;
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1116 ++p;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1117 continue;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1118 case 'u', 'U':
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1119 if (suffix & Suffix.Unsigned)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1120 break;
60
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1121 suffix |= Suffix.Unsigned;
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1122 ++p;
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
1123 continue;
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1124 default:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1125 break;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1126 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1127 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1128 }
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1129
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1130 switch (suffix)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1131 {
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1132 case Suffix.None:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1133 if (ulong_ & 0x8000000000000000)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1134 {
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1135 if (isDecimal)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1136 error(MID.OverflowDecimalSign);
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1137 t.type = TOK.Uint64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1138 }
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1139 else if (ulong_ & 0xFFFFFFFF00000000)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1140 t.type = TOK.Int64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1141 else if (ulong_ & 0x80000000)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1142 t.type = isDecimal ? TOK.Int64 : TOK.Uint32;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1143 else
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1144 t.type = TOK.Int32;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1145 break;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1146 case Suffix.Unsigned:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1147 if (ulong_ & 0xFFFFFFFF00000000)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1148 t.type = TOK.Uint64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1149 else
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1150 t.type = TOK.Uint32;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1151 break;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1152 case Suffix.Long:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1153 if (ulong_ & 0x8000000000000000)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1154 {
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1155 if (isDecimal)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1156 error(MID.OverflowDecimalSign);
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1157 t.type = TOK.Uint64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1158 }
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1159 else
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1160 t.type = TOK.Int64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1161 break;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1162 case Suffix.Unsigned | Suffix.Long:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1163 t.type = TOK.Uint64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1164 break;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1165 default:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1166 assert(0);
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1167 }
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1168 t.ulong_ = ulong_;
15
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
1169 t.end = p;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1170 return;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1171 LscanReal:
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1172 scanReal(t);
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1173 return;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1174 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1175
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1176 /*
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1177 FloatLiteral:= Float[fFL]?i?
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1178 Float:= DecFloat | HexFloat
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
1179 DecFloat:= ([0-9][0-9_]*[.][0-9_]*DecExponent?) | [.][0-9][0-9_]*DecExponent? | [0-9][0-9_]*DecExponent
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1180 DecExponent:= [eE][+-]?[0-9][0-9_]*
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1181 HexFloat:= 0[xX](HexDigits[.]HexDigits | [.][0-9a-zA-Z]HexDigits? | HexDigits)HexExponent
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1182 HexExponent:= [pP][+-]?[0-9][0-9_]*
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1183 */
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1184 void scanReal(ref Token t)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1185 {
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1186 if (*p == '.')
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1187 // This function was called by scan() or scanNumber().
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1188 while (isdigit(*++p) || *p == '_') {}
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1189 else
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1190 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1191 // This function was called by scanNumber().
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1192 debug switch (*p)
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1193 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1194 case 'L':
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1195 if (p[1] != 'i')
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1196 assert(0);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1197 case 'i', 'f', 'F', 'e', 'E': break;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1198 default: assert(0);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1199 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1200 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1201
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1202 // Scan exponent.
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1203 if (*p == 'e' || *p == 'E')
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1204 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1205 ++p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1206 if (*p == '-' || *p == '+')
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1207 ++p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1208 if (!isdigit(*p))
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1209 error(MID.FloatExponentDigitExpected);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1210 else
64
dd4c5a0e47dd - Improved while loop.
aziz
parents: 63
diff changeset
1211 while (isdigit(*++p) || *p == '_') {}
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1212 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1213
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1214 // Copy string to buffer ignoring underscores.
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1215 char[] buffer;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1216 char* end = p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1217 p = t.start;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1218 do
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1219 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1220 if (*p == '_')
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1221 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1222 ++p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1223 continue;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1224 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1225 buffer ~= *p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1226 ++p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1227 } while (p != end)
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1228 buffer ~= 0;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1229 finalizeFloat(t, buffer);
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1230 }
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1231
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1232 void scanHexReal(ref Token t)
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1233 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1234 assert(*p == '.' || *p == 'i' || *p == 'p' || *p == 'P' || (*p == 'L' && p[1] == 'i'));
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1235 MID mid;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1236 if (*p == '.')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1237 while (ishexad(*++p) || *p == '_') {}
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1238 if (*p != 'p' && *p != 'P')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1239 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1240 mid = MID.HexFloatExponentRequired;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1241 goto Lerr;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1242 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1243 // Copy mantissa to a buffer ignoring underscores.
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1244 char* end = p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1245 p = t.start;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1246 char[] buffer;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1247 do
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1248 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1249 if (*p == '_')
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1250 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1251 ++p;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1252 continue;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1253 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1254 buffer ~= *p;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1255 ++p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1256 } while (p != end)
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1257
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1258 assert(p == end && (*p == 'p' || *p == 'P'));
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1259 // Scan and copy the exponent.
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1260 buffer ~= 'p';
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1261 size_t bufflen = buffer.length;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1262 while (1)
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1263 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1264 if (*++p == '_')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1265 continue;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1266 if (isdigit(*p))
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1267 buffer ~= *p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1268 else
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1269 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1270 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1271 // When the buffer length hasn't changed, no digits were copied.
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1272 if (bufflen == buffer.length) {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1273 mid = MID.HexFloatMissingExpDigits;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1274 goto Lerr;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1275 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1276 buffer ~= 0; // Terminate for C functions.
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1277 finalizeFloat(t, buffer);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1278 return;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1279 Lerr:
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1280 t.type = TOK.Float32;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1281 t.end = p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1282 error(mid);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1283 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1284
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1285 void finalizeFloat(ref Token t, string buffer)
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1286 {
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1287 // Float number is well-formed. Check suffixes and do conversion.
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1288 switch (*p)
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1289 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1290 case 'f', 'F':
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1291 t.type = TOK.Float32;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1292 t.float_ = strtof(buffer.ptr, null);
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1293 ++p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1294 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1295 case 'L':
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1296 t.type = TOK.Float80;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1297 t.real_ = strtold(buffer.ptr, null);
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1298 ++p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1299 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1300 default:
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1301 t.type = TOK.Float64;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1302 t.double_ = strtod(buffer.ptr, null);
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1303 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1304 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1305 if (*p == 'i')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1306 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1307 ++p;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1308 t.type += 3; // Switch to imaginary counterpart.
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1309 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1310 if (getErrno == ERANGE)
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1311 error(MID.OverflowFloatNumber);
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1312 t.end = p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1313 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
1314
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1315 /// Scan special token: #line Integer [Filespec] EndOfLine
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1316 // TODO: Handle case like: #line 0 #line 2
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1317 void scanSpecialToken()
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1318 {
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1319 assert(*p == '#');
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1320
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1321 ++p;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1322 MID mid;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1323 Token t;
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1324 uint oldloc = this.loc, newloc;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1325
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1326 peek(t);
65
6c21ae79fbb3 - Renamed function Token.span to Token.srcText.
aziz
parents: 64
diff changeset
1327 if (!(this.loc == oldloc && p == t.start && t.type == TOK.Identifier && t.srcText == "line"))
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1328 {
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1329 this.loc = oldloc; // reset this.loc because we took a peek at the next token
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1330 mid = MID.ExpectedIdentifierSTLine;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1331 goto Lerr;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1332 }
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1333 p = t.end; // consume token
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1334
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1335 peek(t);
131
ce636f3981cc - Removed TOK.Number.
aziz
parents: 112
diff changeset
1336 if (this.loc == oldloc && t.type == TOK.Int32)
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1337 {
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1338 newloc = t.uint_ - 1;
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1339 p = t.end;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1340 }
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1341 else
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1342 {
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1343 this.loc = oldloc;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1344 mid = MID.ExpectedNumberAfterSTLine;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1345 goto Lerr;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1346 }
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1347
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1348 peek(t);
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1349 if (this.loc != oldloc)
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1350 {
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1351 this.loc = oldloc;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1352 mid = MID.NewlineInSpecialToken;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1353 goto Lerr;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1354 }
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1355 if (t.type == TOK.String)
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1356 {
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1357 if (*t.start != '"')
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1358 {
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1359 mid = MID.ExpectedNormalStringLiteral;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1360 goto Lerr;
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1361 }
65
6c21ae79fbb3 - Renamed function Token.span to Token.srcText.
aziz
parents: 64
diff changeset
1362 fileName = t.srcText[1..$-1]; // contents of "..."
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1363 p = t.end;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1364 }
65
6c21ae79fbb3 - Renamed function Token.span to Token.srcText.
aziz
parents: 64
diff changeset
1365 else if (t.type == TOK.Identifier && t.srcText == "__FILE__")
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1366 {
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1367 p = t.end;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1368 }
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1369 /+
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1370 peek(t);
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1371 if (this.loc == oldloc && t.type != TOK.EOF)
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1372 {
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1373 mid = MID.UnterminatedSpecialToken;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1374 goto Lerr;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1375 }
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1376 +/
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1377 while (1)
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1378 {
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1379 switch (*p)
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1380 {
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1381 case '\r':
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1382 if (p[1] == '\n')
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1383 ++p;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1384 case '\n':
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1385 ++p;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1386 break;
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1387 case LS[0]:
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1388 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1389 {
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1390 p += 2;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1391 break;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1392 }
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1393 case 0, _Z_:
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1394 break;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1395 default:
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1396 if (isspace(*p)) {
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1397 ++p;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1398 continue;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1399 }
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1400 mid = MID.UnterminatedSpecialToken;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1401 goto Lerr;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1402 }
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1403 break;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1404 }
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1405
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
1406 this.loc = newloc;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1407 return;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1408 Lerr:
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
1409 error(mid);
15
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
1410 }
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
1411
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
1412 dchar decodeUTF8()
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
1413 {
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1414 assert(*p & 128, "check for ASCII char before calling decodeUTF8().");
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
1415 size_t idx;
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1416 dchar d;
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1417 try
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1418 {
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1419 d = std.utf.decode(p[0 .. end-p], idx);
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1420 p += idx -1;
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1421 }
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1422 catch (UtfException e)
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1423 {
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1424 error(MID.InvalidUTF8Sequence);
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1425 // Skip to next valid utf-8 sequence
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1426 while (UTF8stride[*++p] != 0xFF) {}
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1427 --p;
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1428 }
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
1429 return d;
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
1430 }
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
1431
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
1432 void loadKeywords()
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
1433 {
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
1434 foreach(k; keywords)
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
1435 idtable[k.str] = k;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
1436 }
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
1437
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1438 struct State
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1439 {
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1440 Lexer lexer;
209
921d15b1beeb - Added members token and loc to State.
aziz
parents: 207
diff changeset
1441 Token token;
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1442 char* scanPointer;
209
921d15b1beeb - Added members token and loc to State.
aziz
parents: 207
diff changeset
1443 int loc;
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1444 string fileName;
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1445 size_t errorLen;
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1446 static State opCall(Lexer lx)
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1447 {
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1448 State s;
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1449 s.lexer = lx;
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1450 s.token = lx.token;
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1451 s.scanPointer = lx.p;
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1452 s.loc = lx.loc;
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1453 s.fileName = lx.fileName;
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1454 s.errorLen = lx.errors.length;
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1455 return s;
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1456 }
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1457 void restore()
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1458 {
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1459 lexer.p = scanPointer;
209
921d15b1beeb - Added members token and loc to State.
aziz
parents: 207
diff changeset
1460 lexer.token = token;
921d15b1beeb - Added members token and loc to State.
aziz
parents: 207
diff changeset
1461 lexer.loc = loc;
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1462 lexer.fileName = fileName;
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1463 lexer.errors = lexer.errors[0..errorLen];
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1464 }
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1465 }
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1466
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1467 State getState()
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1468 {
212
c9b9c979a620 - Fix: checking for _Z_.
aziz
parents: 209
diff changeset
1469 return State(this);
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1470 }
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
1471
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1472 void peek(ref Token t)
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1473 {
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1474 // Because peeked tokens are not stored in a linked
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1475 // list we need to switch off error reporting
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1476 // so as to avoid getting the same error more than once.
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1477 reportErrors = false;
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1478 char* save = p;
209
921d15b1beeb - Added members token and loc to State.
aziz
parents: 207
diff changeset
1479 auto saveLoc = loc;
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1480 if (t.end !is null) // For successive peeks.
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1481 {
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1482 p = t.end;
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1483 assert(text.ptr < p && p <= end);
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1484 }
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1485 scan(t);
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1486 p = save;
209
921d15b1beeb - Added members token and loc to State.
aziz
parents: 207
diff changeset
1487 loc = saveLoc;
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1488 reportErrors = true;
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1489 }
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1490
66
8e84db78ad55 - Added support for variadic arguments in error messages.
aziz
parents: 65
diff changeset
1491 void error(MID id, ...)
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1492 {
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1493 if (reportErrors)
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
1494 errors ~= new Information(InfoType.Lexer, id, loc, arguments(_arguments, _argptr));
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1495 }
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1496
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1497 unittest
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1498 {
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1499 string sourceText = "unittest { }";
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1500 auto lx = new Lexer(sourceText, null);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1501
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1502 Token next;
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1503 lx.peek(next);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1504 assert(next == TOK.Unittest);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1505 lx.peek(next);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1506 assert(next == TOK.LBrace);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1507 lx.peek(next);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1508 assert(next == TOK.RBrace);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1509 lx.peek(next);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1510 assert(next == TOK.EOF);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1511 writefln("end of peek() unittest");
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1512 }
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1513
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
1514 public TOK nextToken()
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
1515 {
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
1516 scan(this.token);
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
1517 return this.token.type;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
1518 }
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
1519
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
1520 Token[] getTokens()
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
1521 {
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
1522 Token[] tokens;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
1523 while (nextToken() != TOK.EOF)
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
1524 tokens ~= this.token;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
1525 tokens ~= this.token;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
1526 return tokens;
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
1527 }
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1528
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1529 private void encodeUTF8(inout char[] str, dchar d)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1530 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1531 char[6] b;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1532 assert(d > 0x7F, "check for ASCII char before calling encodeUTF8().");
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1533 if (d < 0x800)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1534 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1535 b[0] = 0xC0 | (d >> 6);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1536 b[1] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1537 str ~= b[0..2];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1538 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1539 else if (d < 0x10000)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1540 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1541 b[0] = 0xE0 | (d >> 12);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1542 b[1] = 0x80 | ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1543 b[2] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1544 str ~= b[0..3];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1545 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1546 else if (d < 0x200000)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1547 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1548 b[0] = 0xF0 | (d >> 18);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1549 b[1] = 0x80 | ((d >> 12) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1550 b[2] = 0x80 | ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1551 b[3] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1552 str ~= b[0..4];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1553 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1554 else if (d < 0x4000000)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1555 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1556 b[0] = 0xF8 | (d >> 24);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1557 b[1] = 0x80 | ((d >> 18) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1558 b[2] = 0x80 | ((d >> 12) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1559 b[3] = 0x80 | ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1560 b[4] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1561 str ~= b[0..5];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1562 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1563 else if (d < 0x80000000)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1564 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1565 b[0] = 0xFC | (d >> 30);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1566 b[1] = 0x80 | ((d >> 24) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1567 b[2] = 0x80 | ((d >> 18) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1568 b[3] = 0x80 | ((d >> 12) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1569 b[4] = 0x80 | ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1570 b[5] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1571 str ~= b[0..6];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1572 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1573 else
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1574 error(MID.InvalidUnicodeCharacter);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1575 }
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
1576 }
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1577
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1578 unittest
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1579 {
40
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
1580 string[] toks = [
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
1581 ">", ">=", ">>", ">>=", ">>>", ">>>=", "<", "<=", "<>",
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
1582 "<>=", "<<", "<<=", "!", "!<", "!>", "!<=", "!>=", "!<>",
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
1583 "!<>=", ".", "..", "...", "&", "&&", "&=", "+", "++",
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
1584 "+=", "-", "--", "-=", "=", "==", "~", "~=", "*",
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
1585 "*=", "/", "/=", "^", "^=", "%", "%=", "(", ")",
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
1586 "[", "]", "{", "}", ":", ";", "?", ",", "$"
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
1587 ];
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1588
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1589 char[] src;
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1590
40
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
1591 foreach (op; toks)
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1592 src ~= op ~ " ";
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1593
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1594 auto lx = new Lexer(src, "");
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1595 auto tokens = lx.getTokens();
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1596
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1597 tokens = tokens[0..$-1]; // exclude TOK.EOF
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1598
40
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
1599 assert(tokens.length == toks.length );
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1600
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
1601 foreach (i, t; tokens)
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
1602 assert(t.srcText == toks[i], std.string.format("Lexed '%s' but expected '%s'", t.srcText, toks[i]));
41
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
1603 }
55
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1604
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
1605 unittest
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
1606 {
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
1607 // Numbers unittest
68
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
1608 // 0L 0ULi 0_L 0_UL 0x0U 0x0p2 0_Fi 0_e2 0_F 0_i
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
1609 // 0u 0U 0uL 0UL 0L 0LU 0Lu
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
1610 // 0Li 0f 0F 0fi 0Fi 0i
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
1611 // 0b_1_LU 0b1000u
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
1612 // 0x232Lu
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
1613 }
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
1614
55
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1615 /// ASCII character properties table.
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1616 static const int ptable[256] = [
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1617 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1618 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1619 32, 0, 0x2200, 0, 0, 0, 0, 0x2700, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1620 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0x3f00,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1621 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1622 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0x5c00, 0, 0,16,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1623 0, 0x70c, 0x80c,12,12,12, 0xc0c, 8, 8, 8, 8, 8, 8, 8, 0xa08, 8,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1624 8, 8, 0xd08, 8, 0x908, 8, 0xb08, 8, 8, 8, 8, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1625 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1626 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1627 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1628 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1629 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1630 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1631 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1632 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1633 ];
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1634
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1635 enum CProperty
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1636 {
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1637 Octal = 1,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1638 Digit = 1<<1,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1639 Hex = 1<<2,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1640 Alpha = 1<<3,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1641 Underscore = 1<<4,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1642 Whitespace = 1<<5
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1643 }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1644
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1645 const uint EVMask = 0xFF00; // Bit mask for escape value
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1646
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1647 private alias CProperty CP;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1648 int isoctal(char c) { return ptable[c] & CP.Octal; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1649 int isdigit(char c) { return ptable[c] & CP.Digit; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1650 int ishexad(char c) { return ptable[c] & CP.Hex; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1651 int isalpha(char c) { return ptable[c] & CP.Alpha; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1652 int isalnum(char c) { return ptable[c] & (CP.Alpha | CP.Digit); }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1653 int isidbeg(char c) { return ptable[c] & (CP.Alpha | CP.Underscore); }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1654 int isident(char c) { return ptable[c] & (CP.Alpha | CP.Underscore | CP.Digit); }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1655 int isspace(char c) { return ptable[c] & CP.Whitespace; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1656 int char2ev(char c) { return ptable[c] >> 8; /*(ptable[c] & EVMask) >> 8;*/ }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1657
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1658 version(gen_ptable)
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1659 static this()
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1660 {
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1661 alias ptable p;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1662 // Initialize character properties table.
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1663 for (int i; i < p.length; ++i)
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1664 {
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1665 p[i] = 0;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1666 if ('0' <= i && i <= '7')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1667 p[i] |= CP.Octal;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1668 if ('0' <= i && i <= '9')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1669 p[i] |= CP.Digit;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1670 if (isdigit(i) || 'a' <= i && i <= 'f' || 'A' <= i && i <= 'F')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1671 p[i] |= CP.Hex;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1672 if ('a' <= i && i <= 'z' || 'A' <= i && i <= 'Z')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1673 p[i] |= CP.Alpha;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1674 if (i == '_')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1675 p[i] |= CP.Underscore;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1676 if (i == ' ' || i == '\t' || i == '\v' || i == '\f')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1677 p[i] |= CP.Whitespace;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1678 }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1679 // Store escape sequence values in second byte.
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1680 assert(CProperty.max <= ubyte.max, "character property flags and escape value byte overlap.");
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1681 p['\''] |= 39 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1682 p['"'] |= 34 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1683 p['?'] |= 63 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1684 p['\\'] |= 92 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1685 p['a'] |= 7 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1686 p['b'] |= 8 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1687 p['f'] |= 12 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1688 p['n'] |= 10 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1689 p['r'] |= 13 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1690 p['t'] |= 9 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1691 p['v'] |= 11 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1692 // Print a formatted array literal.
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1693 char[] array = "[\n";
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1694 for (int i; i < p.length; ++i)
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1695 {
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1696 int c = p[i];
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1697 array ~= std.string.format(c>255?" 0x%x,":"%2d,", c, ((i+1) % 16) ? "":"\n");
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1698 }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1699 array[$-2..$] = "\n]";
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1700 writefln(array);
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
1701 }