Mercurial > projects > dil

annotate trunk/src/Lexer.d @ 14:cdf788d8bdaf

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
- Parsing /= now.
author aziz
date Sat, 23 Jun 2007 13:14:05 +0000
parents e5211758b63c
children c70c028e47dd
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
1 /++
8ba2570de175 Initial import.
aziz
parents:
diff changeset
2 Author: Aziz Köksal
8ba2570de175 Initial import.
aziz
parents:
diff changeset
3 License: GPL2
8ba2570de175 Initial import.
aziz
parents:
diff changeset
4 +/
8ba2570de175 Initial import.
aziz
parents:
diff changeset
5 module Lexer;
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
6 import Token;
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
7 import std.stdio;
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
8 import std.utf;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
9 import std.uni;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
10
8ba2570de175 Initial import.
aziz
parents:
diff changeset
11 /// ASCII character properties table.
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
12 static const int ptable[256] = [
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
13 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
14 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
15 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
16 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
17 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
18 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0, 0, 0,16,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
19 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
20 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
21 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
22 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
23 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
24 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
25 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
26 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
27 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
28 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
29 ];
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
30
8ba2570de175 Initial import.
aziz
parents:
diff changeset
31 enum CProperty
8ba2570de175 Initial import.
aziz
parents:
diff changeset
32 {
1
f3cd3bfde4ba - Corrected some errors to make the file compile.
aziz
parents: 0
diff changeset
33 Octal = 1,
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
34 Digit = 1<<1,
8ba2570de175 Initial import.
aziz
parents:
diff changeset
35 Hex = 1<<2,
8ba2570de175 Initial import.
aziz
parents:
diff changeset
36 Alpha = 1<<3,
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
37 Underscore = 1<<4,
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
38 }
8ba2570de175 Initial import.
aziz
parents:
diff changeset
39
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
40 private alias CProperty CP;
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
41 int isoctal(char c) { return ptable[c] & CP.Octal; }
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
42 int isdigit(char c) { return ptable[c] & CP.Digit; }
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
43 int ishexad(char c) { return ptable[c] & CP.Hex; }
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
44 int isalpha(char c) { return ptable[c] & CP.Alpha; }
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
45 int isalnum(char c) { return ptable[c] & (CP.Alpha | CP.Digit); }
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
46 int isidbeg(char c) { return ptable[c] & (CP.Alpha | CP.Underscore); }
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
47 int isident(char c) { return ptable[c] & (CP.Alpha | CP.Underscore | CP.Digit); }
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
48
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
49 version(gen_ptable)
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
50 static this()
8ba2570de175 Initial import.
aziz
parents:
diff changeset
51 {
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
52 // Initialize character properties table.
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
53 for (int i; i < ptable.length; ++i)
8ba2570de175 Initial import.
aziz
parents:
diff changeset
54 {
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
55 ptable[i] = 0;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
56 if ('0' <= i && i <= '7')
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
57 ptable[i] |= CP.Octal;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
58 if ('0' <= i && i <= '9')
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
59 ptable[i] |= CP.Digit;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
60 if (isdigit(i) || 'a' <= i && i <= 'f' || 'A' <= i && i <= 'F')
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
61 ptable[i] |= CP.Hex;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
62 if ('a' <= i && i <= 'z' || 'A' <= i && i <= 'Z')
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
63 ptable[i] |= CP.Alpha;
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
64 if (i == '_')
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
65 ptable[i] |= CP.Underscore;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
66 }
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
67 // Print a formatted array literal.
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
68 char[] array = "[\n";
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
69 for (int i; i < ptable.length; ++i)
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
70 {
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
71 int c = ptable[i];
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
72 array ~= std.string.format("%2d,", c, ((i+1) % 16) ? "":"\n");
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
73 }
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
74 array[$-2..$] = "\n]";
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
75 writefln(array);
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
76 }
8ba2570de175 Initial import.
aziz
parents:
diff changeset
77
11
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
78 const char[3] LS = \u2028;
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
79 const char[3] PS = \u2029;
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
80
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
81 class Lexer
8ba2570de175 Initial import.
aziz
parents:
diff changeset
82 {
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
83 Token token;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
84 char[] text;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
85 char* p;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
86 char* end;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
87
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
88 this(char[] text)
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
89 {
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
90 this.text = text;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
91 this.text.length = this.text.length + 1;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
92 this.text[$-1] = 0;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
93
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
94 this.p = this.text.ptr;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
95 this.end = this.p + this.text.length;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
96 }
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
97
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
98 public void scan(out Token t)
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
99 {
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
100 assert(p < end);
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
101
10
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
102 uint c = *p;
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
103
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
104 while(1)
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
105 {
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
106 t.start = p;
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
107 if (c == 0)
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
108 {
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
109 t.type = TOK.EOF;
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
110 t.end = p+1;
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
111 return;
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
112 }
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
113
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
114 if (isidbeg(c))
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
115 {
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
116 Lidentifier:
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
117 do
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
118 { c = *++p; }
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
119 while (isident(c) || c & 128 && isUniAlpha(decodeUTF()))
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
120 t.type = TOK.Identifier;
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
121 t.end = p;
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
122 return;
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
123 }
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
124
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
125 if (c == '/')
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
126 {
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
127 c = *++p;
14
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
128 switch(c)
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
129 {
14
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
130 case '=':
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
131 ++p;
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
132 t.type = TOK.DivisionAssign;
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
133 t.end = p;
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
134 return;
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
135 case '+':
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
136 uint level = 1;
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
137 do
7
07e45c06a024 - Parsing nested comments correctly now.
aziz
parents: 5
diff changeset
138 {
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
139 c = *++p;
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
140 if (c == 0)
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
141 throw new Error("unterminated /+ +/ comment.");
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
142 else if (c == '/' && p[1] == '+')
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
143 {
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
144 ++p;
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
145 ++level;
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
146 }
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
147 else if (c == '+' && p[1] == '/')
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
148 {
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
149 ++p;
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
150 if (--level == 0)
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
151 break;
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
152 }
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
153 } while (1)
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
154 p += 2;
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
155 t.type = TOK.Comment;
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
156 t.end = p;
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
157 return;
14
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
158 case '*':
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
159 do
7
07e45c06a024 - Parsing nested comments correctly now.
aziz
parents: 5
diff changeset
160 {
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
161 c = *++p;
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
162 if (c == 0)
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
163 throw new Error("unterminated /* */ comment.");
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
164 } while (c != '*' || p[1] != '/')
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
165 p += 2;
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
166 t.type = TOK.Comment;
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
167 t.end = p;
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
168 return;
14
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
169 case '/':
10
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
170 do
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
171 {
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
172 c = *++p;
11
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
173 if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
174 break;
10
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
175 } while (c != '\n' && c != 0)
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
176 t.type = TOK.Comment;
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
177 t.end = p;
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
178 return;
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
179 }
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
180 }
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
181
9
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
182 if (c == '"')
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
183 {
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
184 do {
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
185 c = *++p;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
186 if (c == 0)
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
187 throw new Error("unterminated string literal.");
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
188 if (c == '\\')
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
189 ++p;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
190 } while (c != '"')
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
191 ++p;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
192 t.type = TOK.String;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
193 t.end = p;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
194 return;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
195 }
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
196
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
197 if (c == '\'')
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
198 {
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
199 do {
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
200 c = *++p;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
201 if (c == 0)
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
202 throw new Error("unterminated character literal.");
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
203 if (c == '\\')
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
204 ++p;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
205 } while (c != '\'')
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
206 ++p;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
207 t.type = TOK.Character;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
208 t.end = p;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
209 return;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 8
diff changeset
210 }
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
211
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
212 if (c & 128 && isUniAlpha(decodeUTF()))
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
213 goto Lidentifier;
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
214 c = *++p;
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
215 }
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
216 }
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
217
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
218 uint decodeUTF()
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
219 {
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
220 assert(*p & 128);
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
221 size_t idx;
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
222 uint d;
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
223 d = std.utf.decode(p[0 .. end-p], idx);
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
224 p += idx -1;
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
225 return d;
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
226 }
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
227
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
228 public TOK nextToken()
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
229 {
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
230 scan(this.token);
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
231 return this.token.type;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
232 }
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
233
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
234 Token[] getTokens()
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
235 {
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
236 Token[] tokens;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
237 while (nextToken() != TOK.EOF)
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
238 tokens ~= this.token;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
239 tokens ~= this.token;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
240 return tokens;
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
241 }
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
242 }