Mercurial > projects > dang

comparison lexer/Lexer.d @ 189:75d0544ddc45

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Better error handling on unexpected EOF.
author Anders Johnsen <skabet@gmail.com>
date Fri, 25 Jul 2008 13:50:01 +0200
parents e1e170c2cd44
children 85e492318bb6
comparison
equal deleted inserted replaced
188:b3e0729c8524 189:75d0544ddc45
79 return: A Token - Token.type is TokType.EOF if there is 79 return: A Token - Token.type is TokType.EOF if there is
80 no more tokens in the file. 80 no more tokens in the file.
81 */ 81 */
82 Token next() 82 Token next()
83 { 83 {
84 Token res;
84 switch (getNextChar) 85 switch (getNextChar)
85 { 86 {
86 case CharType.EOF: 87 case CharType.EOF:
87 SLoc loc; 88 return Token(Tok.EOF, last.location, 0);
88 return Token(Tok.EOF, loc+1, 0);
89 89
90 case CharType.Whitespace: 90 case CharType.Whitespace:
91 position += 1; 91 position += 1;
92 return this.next; 92 res = this.next;
93 break;
93 94
94 case CharType.Symbol: 95 case CharType.Symbol:
95 return lexSymbol; 96 res = lexSymbol;
97 break;
96 98
97 case CharType.Letter: 99 case CharType.Letter:
98 return lexLetter; 100 res = lexLetter;
101 break;
99 102
100 case CharType.Number: 103 case CharType.Number:
101 return lexNumber; 104 res = lexNumber;
105 break;
102 case CharType.Other: 106 case CharType.Other:
103 messages.report(UnexpectedTok, Loc(position)).fatal(ExitLevel.Lexer); 107 messages.report(UnexpectedTok, Loc(position)).fatal(ExitLevel.Lexer);
104 } 108 }
109 if (res.type != Tok.EOF)
110 last = res;
111 return res;
105 } 112 }
106 113
107 /** 114 /**
108 Get the next token from the source. This method will NOT move the 115 Get the next token from the source. This method will NOT move the
109 internal position forward, and thereby having no side-effects. 116 internal position forward, and thereby having no side-effects.
119 Token t = this.next; 126 Token t = this.next;
120 this.position = oldPosition; 127 this.position = oldPosition;
121 return t; 128 return t;
122 } 129 }
123 130
131 Token last;
124 private: 132 private:
133
125 Token eq() 134 Token eq()
126 { 135 {
127 if(source[position] == '=') 136 if(source[position] == '=')
128 return Token(Tok.Eq, Loc(position++ - 1), 2); 137 return Token(Tok.Eq, Loc(position++ - 1), 2);
129 return Token(Tok.Assign, Loc(position - 1), 1); 138 return Token(Tok.Assign, Loc(position - 1), 1);