Mercurial > projects > dil

comparison trunk/src/Lexer.d @ 28:3a9daccf7d96

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
- Added table for identifiers to Lexer. - Added keywords table. - Added keywords to TOK.
author aziz
date Sun, 24 Jun 2007 17:19:03 +0000
parents 43b6bf56f0e9
children ef83eea26bbd
comparison
equal deleted inserted replaced
27:43b6bf56f0e9 28:3a9daccf7d96
2 Author: Aziz Köksal 2 Author: Aziz Köksal
3 License: GPL2 3 License: GPL2
4 +/ 4 +/
5 module Lexer; 5 module Lexer;
6 import Token; 6 import Token;
7 import Keywords;
8 import Identifier;
7 import std.stdio; 9 import std.stdio;
8 import std.utf; 10 import std.utf;
9 import std.uni; 11 import std.uni;
10 12
11 /// ASCII character properties table. 13 /// ASCII character properties table.
126 128
127 uint loc = 1; /// line of code 129 uint loc = 1; /// line of code
128 130
129 Problem[] errors; 131 Problem[] errors;
130 132
133 Identifier[string] idtable;
134
131 this(char[] text) 135 this(char[] text)
132 { 136 {
133 this.text = text; 137 this.text = text;
134 this.text.length = this.text.length + 1; 138 this.text.length = this.text.length + 1;
135 this.text[$-1] = 0; 139 this.text[$-1] = 0;
136 140
137 this.p = this.text.ptr; 141 this.p = this.text.ptr;
138 this.end = this.p + this.text.length; 142 this.end = this.p + this.text.length;
143
144 loadKeywords();
139 } 145 }
140 146
141 public void scan(out Token t) 147 public void scan(out Token t)
142 { 148 {
143 assert(p < end); 149 assert(p < end);
174 { 180 {
175 Lidentifier: 181 Lidentifier:
176 do 182 do
177 { c = *++p; } 183 { c = *++p; }
178 while (isident(c) || c & 128 && isUniAlpha(decodeUTF())) 184 while (isident(c) || c & 128 && isUniAlpha(decodeUTF()))
179 t.type = TOK.Identifier; 185
180 t.end = p; 186 t.end = p;
187
188 string str = t.span;
189 Identifier* id = str in idtable;
190
191 if (!id)
192 {
193 idtable[str] = Identifier.Identifier(TOK.Identifier, str);
194 id = str in idtable;
195 }
196 assert(id);
197 t.type = id.type;
181 return; 198 return;
182 } 199 }
183 200
184 if (isdigit(c)) 201 if (isdigit(c))
185 return scanNumber(t); 202 return scanNumber(t);
431 d = std.utf.decode(p[0 .. end-p], idx); 448 d = std.utf.decode(p[0 .. end-p], idx);
432 p += idx -1; 449 p += idx -1;
433 return d; 450 return d;
434 } 451 }
435 452
453 void loadKeywords()
454 {
455 foreach(k; keywords)
456 idtable[k.str] = k;
457 }
458
436 void error(MID id) 459 void error(MID id)
437 { 460 {
438 errors ~= new Problem(Problem.Type.Lexer, id, loc); 461 errors ~= new Problem(Problem.Type.Lexer, id, loc);
439 } 462 }
440 463