Mercurial > projects > dang

comparison lexer/Lexer.d @ 104:7ff4bc2accf2 new_gen

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Lexing more types of strings. Now all's left is to parse the string in the AST.
author Anders Johnsen <skabet@gmail.com>
date Wed, 21 May 2008 21:05:23 +0200
parents 857f0d530789
children c658172ca8a0
comparison
equal deleted inserted replaced
103:09b4d74cb3f5 104:7ff4bc2accf2
35 charTable[c] = CharType.Letter; 35 charTable[c] = CharType.Letter;
36 36
37 foreach (c; "0123456789") 37 foreach (c; "0123456789")
38 charTable[c] = CharType.Number; 38 charTable[c] = CharType.Number;
39 39
40 foreach (c; "(){}[];:.,=!<>+-*/%") 40 foreach (c; "(){}[];:.,=!<>+-*/%\"`")
41 charTable[c] = CharType.Symbol; 41 charTable[c] = CharType.Symbol;
42 42
43 foreach (c; " \n") 43 foreach (c; " \n")
44 charTable[c] = CharType.Whitespace; 44 charTable[c] = CharType.Whitespace;
45
46 foreach (c; "'\\")
47 charTable[c] = CharType.Other;
45 48
46 symbolFunctions.length = 256; 49 symbolFunctions.length = 256;
47 50
48 symbolFunctions['('] = &openParentheses; 51 symbolFunctions['('] = &openParentheses;
49 symbolFunctions[')'] = &closeParentheses; 52 symbolFunctions[')'] = &closeParentheses;
62 symbolFunctions['+'] = &plus; 65 symbolFunctions['+'] = &plus;
63 symbolFunctions['-'] = &minus; 66 symbolFunctions['-'] = &minus;
64 symbolFunctions['*'] = &star; 67 symbolFunctions['*'] = &star;
65 symbolFunctions['/'] = &slash; 68 symbolFunctions['/'] = &slash;
66 symbolFunctions['%'] = &percent; 69 symbolFunctions['%'] = &percent;
70 symbolFunctions['"'] = &string;
71 symbolFunctions['`'] = &string;
67 } 72 }
68 73
69 /** 74 /**
70 Get the next token from the source. This method will move the 75 Get the next token from the source. This method will move the
71 internal position forward to the next Token. 76 internal position forward to the next Token.
91 case CharType.Letter: 96 case CharType.Letter:
92 return lexLetter; 97 return lexLetter;
93 98
94 case CharType.Number: 99 case CharType.Number:
95 return lexNumber; 100 return lexNumber;
101 case CharType.Other:
102 messages.report(UnexpectedTok, Loc(position)).fatal(ExitLevel.Lexer);
96 } 103 }
97 } 104 }
98 105
99 /** 106 /**
100 Get the next token from the source. This method will NOT move the 107 Get the next token from the source. This method will NOT move the
199 } 206 }
200 Token star() 207 Token star()
201 { 208 {
202 return Token(Tok.Star, Loc(position - 1), 1); 209 return Token(Tok.Star, Loc(position - 1), 1);
203 } 210 }
204 Token slash() 211 Token slash()
205 { 212 {
206 switch(source[position]) 213 switch(source[position])
207 { 214 {
208 case '/': 215 case '/':
209 while(getNextChar != CharType.EOF) 216 while(getNextChar != CharType.EOF)
218 while(getNextChar != CharType.EOF) 225 while(getNextChar != CharType.EOF)
219 { 226 {
220 ++position; 227 ++position;
221 if(source[position-2] == '*') 228 if(source[position-2] == '*')
222 if(source[position-1] == '/') 229 if(source[position-1] == '/')
230 {
223 return this.next; 231 return this.next;
232 }
224 } 233 }
225 messages.report(UnexpectedEOFBlock,Loc(position)); 234 messages.report(UnexpectedEOFBlock,Loc(position));
226 235
227 case '+': 236 case '+':
228 position += 2; 237 position += 2;
255 } 264 }
256 265
257 Token percent() 266 Token percent()
258 { 267 {
259 return Token(Tok.Percent, Loc(position - 1), 1); 268 return Token(Tok.Percent, Loc(position - 1), 1);
269 }
270
271 Token string()
272 {
273 --position;
274 int start = position;
275 if(getNextChar() == CharType.Letter)
276 position++;
277 char end = '`';
278 switch(source[position])
279 {
280 case '"':
281 if(position > 0)
282 if(source[position-1] == 'r')
283 {
284 end = '"';
285 goto string_wys;
286 }
287 ++position;
288 while(getNextChar != CharType.EOF)
289 {
290 ++position;
291 if (source[position-1] == '"' )
292 return Token(Tok.String, Loc(start), position - start);
293 else if (source[position-1] == '\\')
294 position++;
295 }
296 break;
297 case '`':
298 string_wys:
299 ++position;
300 while(getNextChar != CharType.EOF)
301 {
302 ++position;
303 if (source[position-1] == end )
304 return Token(Tok.String, Loc(start), position - start);
305 }
306 break;
307 }
308 messages.report(UnexpectedEOFBlock, Loc(position)).fatal(ExitLevel.Lexer);
260 } 309 }
261 310
262 Token lexNumber () 311 Token lexNumber ()
263 { 312 {
264 bool sign = false; 313 bool sign = false;
319 368
320 Token lexLetter () 369 Token lexLetter ()
321 { 370 {
322 int i = 0; 371 int i = 0;
323 bool hasNumber = false; 372 bool hasNumber = false;
373 if (source[position+1] == '"' ||
374 source[position+1] == '`')
375 {
376 ++position;
377 return string;
378 }
324 while (getNextChar(++i) == CharType.Letter || 379 while (getNextChar(++i) == CharType.Letter ||
325 getNextChar(i) == CharType.Number) 380 getNextChar(i) == CharType.Number)
326 { 381 {
327 if (getNextChar(i) == CharType.Number) 382 if (getNextChar(i) == CharType.Number)
328 { 383 {
383 INVALID, 438 INVALID,
384 Letter, 439 Letter,
385 Number, 440 Number,
386 Symbol, 441 Symbol,
387 Whitespace, 442 Whitespace,
443 Other,
388 444
389 EOF 445 EOF
390 } 446 }
391 447