Mercurial > projects > dang

comparison src/lexer/Lexer.d @ 207:e0551773a005

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added the correct version.
author Anders Johnsen <skabet@gmail.com>
date Tue, 12 Aug 2008 18:19:34 +0200
parents d3c148ca429b
children
comparison
equal deleted inserted replaced
206:d3c148ca429b 207:e0551773a005
35 charTable[c] = CharType.Letter; 35 charTable[c] = CharType.Letter;
36 36
37 foreach (c; "0123456789") 37 foreach (c; "0123456789")
38 charTable[c] = CharType.Number; 38 charTable[c] = CharType.Number;
39 39
40 foreach (c; "(){}[];:.,=!<>+-*/%\"`") 40 foreach (c; "(){}[];:.,=!<>+-*/%&\"`")
41 charTable[c] = CharType.Symbol; 41 charTable[c] = CharType.Symbol;
42 42
43 foreach (c; " \n") 43 foreach (c; " \n")
44 charTable[c] = CharType.Whitespace; 44 charTable[c] = CharType.Whitespace;
45 45
65 symbolFunctions['+'] = &plus; 65 symbolFunctions['+'] = &plus;
66 symbolFunctions['-'] = &minus; 66 symbolFunctions['-'] = &minus;
67 symbolFunctions['*'] = &star; 67 symbolFunctions['*'] = &star;
68 symbolFunctions['/'] = &slash; 68 symbolFunctions['/'] = &slash;
69 symbolFunctions['%'] = &percent; 69 symbolFunctions['%'] = &percent;
70 symbolFunctions['&'] = &and;
70 symbolFunctions['"'] = &string; 71 symbolFunctions['"'] = &string;
71 symbolFunctions['`'] = &string; 72 symbolFunctions['`'] = &string;
73
74 last = Token(Tok.EOF, SLoc() + 1, 0);
72 } 75 }
73 76
74 /** 77 /**
75 Get the next token from the source. This method will move the 78 Get the next token from the source. This method will move the
76 internal position forward to the next Token. 79 internal position forward to the next Token.
78 return: A Token - Token.type is TokType.EOF if there is 81 return: A Token - Token.type is TokType.EOF if there is
79 no more tokens in the file. 82 no more tokens in the file.
80 */ 83 */
81 Token next() 84 Token next()
82 { 85 {
86 Token res;
83 switch (getNextChar) 87 switch (getNextChar)
84 { 88 {
85 case CharType.EOF: 89 case CharType.EOF:
86 SLoc loc; 90 return Token(Tok.EOF, last.location, 0);
87 return Token(Tok.EOF, loc, 0);
88 91
89 case CharType.Whitespace: 92 case CharType.Whitespace:
90 position += 1; 93 position += 1;
91 return this.next; 94 res = this.next;
95 break;
92 96
93 case CharType.Symbol: 97 case CharType.Symbol:
94 return lexSymbol; 98 res = lexSymbol;
99 break;
95 100
96 case CharType.Letter: 101 case CharType.Letter:
97 return lexLetter; 102 res = lexLetter;
103 break;
98 104
99 case CharType.Number: 105 case CharType.Number:
100 return lexNumber; 106 res = lexNumber;
107 break;
101 case CharType.Other: 108 case CharType.Other:
102 messages.report(UnexpectedTok, Loc(position)).fatal(ExitLevel.Lexer); 109 messages.report(UnexpectedTok, Loc(position)).fatal(ExitLevel.Lexer);
103 } 110 }
111 if (res.type != Tok.EOF)
112 last = res;
113 return res;
104 } 114 }
105 115
106 /** 116 /**
107 Get the next token from the source. This method will NOT move the 117 Get the next token from the source. This method will NOT move the
108 internal position forward, and thereby having no side-effects. 118 internal position forward, and thereby having no side-effects.
118 Token t = this.next; 128 Token t = this.next;
119 this.position = oldPosition; 129 this.position = oldPosition;
120 return t; 130 return t;
121 } 131 }
122 132
133 Token last;
123 private: 134 private:
135
124 Token eq() 136 Token eq()
125 { 137 {
126 if(source[position] == '=') 138 if(source[position] == '=')
127 return Token(Tok.Eq, Loc(position++ - 1), 2); 139 return Token(Tok.Eq, Loc(position++ - 1), 2);
128 return Token(Tok.Assign, Loc(position - 1), 1); 140 return Token(Tok.Assign, Loc(position - 1), 1);
186 } 198 }
187 Token le() 199 Token le()
188 { 200 {
189 if(source[position] == '=') 201 if(source[position] == '=')
190 return Token(Tok.Le, Loc(position++ - 1), 2); 202 return Token(Tok.Le, Loc(position++ - 1), 2);
203 if(source[position] == '<')
204 return Token(Tok.LeftShift, Loc(position++ - 1), 2);
191 return Token(Tok.Lt, Loc(position - 1), 1); 205 return Token(Tok.Lt, Loc(position - 1), 1);
192 } 206 }
193 Token ge() 207 Token ge()
194 { 208 {
195 if(source[position] == '=') 209 if(source[position] == '=')
196 return Token(Tok.Ge, Loc(position++ - 1), 2); 210 return Token(Tok.Ge, Loc(position++ - 1), 2);
211 if(source[position] == '>')
212 if(source[position+1] == '>')
213 {
214 position += 2;
215 return Token(Tok.UnsignedRightShift, Loc(position - 1), 3);
216 }
217 else
218 return Token(Tok.RightShift, Loc(position++ - 1), 2);
197 return Token(Tok.Gt, Loc(position - 1), 1); 219 return Token(Tok.Gt, Loc(position - 1), 1);
198 } 220 }
199 Token plus() 221 Token plus()
200 { 222 {
223 if(source[position] == '=')
224 return Token(Tok.PlusAssign, Loc(position++ - 1), 2);
201 return Token(Tok.Plus, Loc(position - 1), 1); 225 return Token(Tok.Plus, Loc(position - 1), 1);
202 } 226 }
203 Token minus() 227 Token minus()
204 { 228 {
229 if(source[position] == '=')
230 return Token(Tok.MinusAssign, Loc(position++ - 1), 2);
205 return Token(Tok.Minus, Loc(position - 1), 1); 231 return Token(Tok.Minus, Loc(position - 1), 1);
206 } 232 }
207 Token star() 233 Token star()
208 { 234 {
235 if(source[position] == '=')
236 return Token(Tok.StarAssign, Loc(position++ - 1), 2);
209 return Token(Tok.Star, Loc(position - 1), 1); 237 return Token(Tok.Star, Loc(position - 1), 1);
210 } 238 }
211 Token slash() 239 Token slash()
212 { 240 {
241 int p = position;
213 switch(source[position]) 242 switch(source[position])
214 { 243 {
244 case '=':
245 return Token(Tok.SlashAssign, Loc(position++ - 1), 2);
215 case '/': 246 case '/':
216 while(getNextChar != CharType.EOF) 247 while(getNextChar != CharType.EOF)
217 { 248 {
218 if(source[position++] == '\n') 249 if(source[position++] == '\n')
219 return this.next; 250 return this.next;
229 if(source[position-1] == '/') 260 if(source[position-1] == '/')
230 { 261 {
231 return this.next; 262 return this.next;
232 } 263 }
233 } 264 }
234 messages.report(UnexpectedEOFBlock,Loc(position)); 265 messages.report(UnexpectedEOFBlock,Loc(p)).fatal(ExitLevel.Lexer);
235 266
236 case '+': 267 case '+':
237 position += 2; 268 position += 2;
238 int nesting = 1; 269 int nesting = 1;
239 while(getNextChar != CharType.EOF) 270 while(getNextChar != CharType.EOF)
254 } 285 }
255 286
256 if(nesting == 0) 287 if(nesting == 0)
257 return this.next; 288 return this.next;
258 } 289 }
259 messages.report(UnexpectedEOFBlock,Loc(position)); 290 messages.report(
291 UnexpectedEOFBlock,
292 Loc(p)).fatal(ExitLevel.Lexer);
260 293
261 default: 294 default:
262 return Token(Tok.Slash, Loc(position - 1), 1); 295 return Token(Tok.Slash, Loc(position - 1), 1);
263 } 296 }
264 } 297 }
265 298 Token and()
299 {
300 return Token(Tok.And, Loc(position - 1), 1);
301 }
266 Token percent() 302 Token percent()
267 { 303 {
304 if(source[position] == '=')
305 return Token(Tok.PercentAssign, Loc(position++ - 1), 2);
268 return Token(Tok.Percent, Loc(position - 1), 1); 306 return Token(Tok.Percent, Loc(position - 1), 1);
269 } 307 }
270 308
271 Token string() 309 Token string()
272 { 310 {
287 ++position; 325 ++position;
288 while(getNextChar != CharType.EOF) 326 while(getNextChar != CharType.EOF)
289 { 327 {
290 ++position; 328 ++position;
291 if (source[position-1] == '"' ) 329 if (source[position-1] == '"' )
330 {
331 if(getNextChar != CharType.EOF)
332 if (source[position] == 'c' ||
333 source[position] == 'w' ||
334 source[position] == 'd')
335 position++;
336
292 return Token(Tok.String, Loc(start), position - start); 337 return Token(Tok.String, Loc(start), position - start);
338 }
293 else if (source[position-1] == '\\') 339 else if (source[position-1] == '\\')
294 position++; 340 position++;
295 } 341 }
296 break; 342 break;
297 case '`': 343 case '`':
308 messages.report(UnexpectedEOFBlock, Loc(position)).fatal(ExitLevel.Lexer); 354 messages.report(UnexpectedEOFBlock, Loc(position)).fatal(ExitLevel.Lexer);
309 } 355 }
310 356
311 Token lexNumber () 357 Token lexNumber ()
312 { 358 {
313 bool sign = false; 359 bool sign;
314 bool dot = false;
315 bool e = false;
316 360
317 int i = 0; 361 int i = 0;
362
318 363
319 bool end = false; 364 bool end = false;
320 while(!end) 365 while(!end)
321 { 366 {
322 switch(getNextChar(i)) 367 switch(getNextChar(i))
324 case CharType.Number: 369 case CharType.Number:
325 break; 370 break;
326 case CharType.Symbol: 371 case CharType.Symbol:
327 if(this.source[position+i] == '.') 372 if(this.source[position+i] == '.')
328 { 373 {
329 if(dot)
330 messages.report(OnlyOneDotFloating, Loc(position + i));
331 dot = true;
332 break; 374 break;
375 }
376 if (this.source[position+i] == '+' ||
377 this.source[position+i] == '-')
378 {
379 if (source[position+i-1] == 'e' ||
380 source[position+i-1] == 'E')
381 break;
333 } 382 }
334 end = true; 383 end = true;
335 continue; 384 continue;
336 case CharType.Letter: 385 case CharType.Letter:
337 if(this.source[position+i] == '_') 386 if(this.source[position+i] == '_')
338 break; 387 break;
339 if (this.source[position+i] == 'e' || 388 if (this.source[position+i] == 'e' ||
340 this.source[position+i] == 'E') 389 this.source[position+i] == 'E')
341 { 390 {
342 if (e)
343 messages.report(OnlyOneEFloating, Loc(position + i));
344 e = true;
345 break; 391 break;
346 } 392 }
347 end = true; 393 end = true;
348 continue; 394 continue;
349 395
351 end = true; 397 end = true;
352 continue; 398 continue;
353 } 399 }
354 i++; 400 i++;
355 } 401 }
402
403 while(source[position+i] == 'u' ||
404 source[position+i] == 'U' ||
405 source[position+i] == 'L')
406 i += 1;
407
408
356 409
357 position += i; 410 position += i;
358 411
359 return Token(Tok.Integer, Loc(position - i), i); 412 return Token(Tok.Integer, Loc(position - i), i);
360 } 413 }