Mercurial > projects > dil

comparison trunk/src/dil/Parser.d @ 485:ea8c7459f1c4

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Changed a lot of things in the Lexer. Newlines are tokenized now, instead of being treated as whitespace. Newline tokens store location info as well, which make quite a few functions unnecessary. Added a static method getLocation() which returns a Location instance for any given token. This will also be very useful for finding the location of AST nodes (through Node.begin,) which is needed for reporting parser and semantic errors and emitting documentation. Removed rescanNewlines(), LocState, getState(), restoreState(), evaluateHashLine() and updateErrorLoc(). Added isUnicodeNewlineChar(), isUnicodeNewline(), isNewline(), isNewlineEnd(), isEndOfLine(), scanNewline(), getLocation() and error_(). Replaced some clunky expressions with isascii(), isNewlineEnd(), isEndOfLine(), isUnicodeNewline(), isUnicodeNewlineChar(). Fix in scanNormalStringLiteral(): scanPostfix() must be before label Lreturn. Fixed Lexer unittest. Fix in parseDeclarationDefinitionsBlock(): 'token' should be 'begin'. Added method isMultiline() to Token and added documentation comments.: File name too long abort: file /home/aziz/dil/trunk/Changed a lot of things in the Lexer. Newlines are tokenized now, instead of being treated as whitespace. Newline tokens store location info as well, which make quite a few functions unnecessary. Added a static method getLocation() which returns a Location instance for any given token. This will also be very useful for finding the location of AST nodes (through Node.begin,) which is needed for reporting parser and semantic errors and emitting documentation. Removed rescanNewlines(), LocState, getState(), restoreState(), evaluateHashLine() and updateErrorLoc(). Added isUnicodeNewlineChar(), isUnicodeNewline(), isNewline(), isNewlineEnd(), isEndOfLine(), scanNewline(), getLocation() and error_(). Replaced some clunky expressions with isascii(), isNewlineEnd(), isEndOfLine(), isUnicodeNewline(), isUnicodeNewlineChar(). Fix in scanNormalStringLiteral(): scanPostfix() must be before label Lreturn. Fixed Lexer unittest. Fix in parseDeclarationDefinitionsBlock(): 'token' should be 'begin'. Added method isMultiline() to Token and added documentation comments.
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Fri, 30 Nov 2007 20:17:29 +0100
parents 265c0b655f18
children bd176bc73e43
comparison
equal deleted inserted replaced
484:265c0b655f18 485:ea8c7459f1c4
107 This method executes the delegate parseMethod and when an error occurred 107 This method executes the delegate parseMethod and when an error occurred
108 the state of the lexer and parser are restored. 108 the state of the lexer and parser are restored.
109 +/ 109 +/
110 ReturnType try_(ReturnType)(ReturnType delegate() parseMethod, out bool success) 110 ReturnType try_(ReturnType)(ReturnType delegate() parseMethod, out bool success)
111 { 111 {
112 ++trying;
113 auto oldToken = this.token; 112 auto oldToken = this.token;
114 auto oldPrevToken = this.prevToken; 113 auto oldPrevToken = this.prevToken;
115 auto oldCount = this.errorCount; 114 auto oldCount = this.errorCount;
116 auto lexerState = this.lx.getState(); 115
117 116 ++trying;
118 auto result = parseMethod(); 117 auto result = parseMethod();
119 118 --trying;
120 // Check if an error occurred. 119 // Check if an error occurred.
121 if (errorCount != oldCount) 120 if (errorCount != oldCount)
122 { 121 {
123 // Restore members. 122 // Restore members.
124 lx.restoreState(lexerState);
125 token = oldToken; 123 token = oldToken;
126 prevToken = oldPrevToken; 124 prevToken = oldPrevToken;
127 lx.token = oldToken; 125 lx.token = oldToken;
128 errorCount = oldCount; 126 errorCount = oldCount;
129 success = false; 127 success = false;
130 } 128 }
131 else 129 else
132 success = true; 130 success = true;
133 --trying;
134 return result; 131 return result;
135 } 132 }
136 133
137 /++ 134 /++
138 Sets the begin and end tokens of an AST node. 135 Sets the begin and end tokens of an AST node.
142 node.setTokens(begin, this.prevToken); 139 node.setTokens(begin, this.prevToken);
143 return node; 140 return node;
144 } 141 }
145 142
146 /++ 143 /++
147 Returns true if set() has been called on a node, or false otherwise. 144 Returns true if set() has been called on a node.
148 +/ 145 +/
149 bool isNodeSet(Node node) 146 bool isNodeSet(Node node)
150 { 147 {
151 return node.begin !is null && node.end !is null; 148 return node.begin !is null && node.end !is null;
152 } 149 }
153 150
154 TOK peekNext() 151 TOK peekNext()
155 { 152 {
156 auto state = lx.getState();
157 Token* next = token; 153 Token* next = token;
158 do 154 do
159 lx.peek(next); 155 lx.peek(next);
160 while (next.isWhitespace) // Skip whitespace 156 while (next.isWhitespace) // Skip whitespace
161 lx.restoreState(state);
162 return next.type; 157 return next.type;
163 } 158 }
164 159
165 TOK peekAfter(ref Token* next) 160 TOK peekAfter(ref Token* next)
166 { 161 {
167 assert(next !is null); 162 assert(next !is null);
168 auto state = lx.getState();
169 do 163 do
170 lx.peek(next); 164 lx.peek(next);
171 while (next.isWhitespace) // Skip whitespace 165 while (next.isWhitespace) // Skip whitespace
172 lx.restoreState(state);
173 return next.type; 166 return next.type;
174 } 167 }
175 168
176 /++++++++++++++++++++++++++++++ 169 /++++++++++++++++++++++++++++++
177 + Declaration parsing methods + 170 + Declaration parsing methods +
188 } while (token.type == T.Dot) 181 } while (token.type == T.Dot)
189 require(T.Semicolon); 182 require(T.Semicolon);
190 return set(new ModuleDeclaration(moduleFQN), begin); 183 return set(new ModuleDeclaration(moduleFQN), begin);
191 } 184 }
192 185
186 /++
187 DeclDefs:
188 DeclDef
189 DeclDefs
190 +/
193 Declarations parseDeclarationDefinitions() 191 Declarations parseDeclarationDefinitions()
194 { 192 {
195 auto decls = new Declarations; 193 auto decls = new Declarations;
196 while (token.type != T.EOF) 194 while (token.type != T.EOF)
197 decls ~= parseDeclarationDefinition(); 195 decls ~= parseDeclarationDefinition();
198 return decls; 196 return decls;
199 } 197 }
200 198
201 /* 199 /++
202 DeclDefsBlock: 200 DeclDefsBlock:
203 { } 201 { }
204 { DeclDefs } 202 { DeclDefs }
205 */ 203 +/
206 Declarations parseDeclarationDefinitionsBlock() 204 Declarations parseDeclarationDefinitionsBlock()
207 { 205 {
208 auto begin = token; 206 auto begin = token;
209 auto decls = new Declarations; 207 auto decls = new Declarations;
210 require(T.LBrace); 208 require(T.LBrace);
211 while (token.type != T.RBrace && token.type != T.EOF) 209 while (token.type != T.RBrace && token.type != T.EOF)
212 decls ~= parseDeclarationDefinition(); 210 decls ~= parseDeclarationDefinition();
213 require(T.RBrace); 211 require(T.RBrace);
214 return set(decls, token); 212 return set(decls, begin);
215 } 213 }
216 214
217 Declaration parseDeclarationDefinition() 215 Declaration parseDeclarationDefinition()
218 { 216 {
219 auto begin = token; 217 auto begin = token;
3856 { 3854 {
3857 // We count nested parentheses tokens because template types may appear inside parameter lists; e.g. (int x, Foo!(int) y). 3855 // We count nested parentheses tokens because template types may appear inside parameter lists; e.g. (int x, Foo!(int) y).
3858 assert(token.type == T.LParen); 3856 assert(token.type == T.LParen);
3859 Token* next = token; 3857 Token* next = token;
3860 uint level = 1; 3858 uint level = 1;
3861 auto state = lx.getState();
3862 Loop: 3859 Loop:
3863 while (1) 3860 while (1)
3864 { 3861 {
3865 lx.peek(next); 3862 lx.peek(next);
3866 switch (next.type) 3863 switch (next.type)
3880 case T.EOF: 3877 case T.EOF:
3881 break Loop; 3878 break Loop;
3882 default: 3879 default:
3883 } 3880 }
3884 } 3881 }
3885 lx.restoreState(state);
3886 return next.type == tok; 3882 return next.type == tok;
3887 } 3883 }
3888 3884
3889 Type parseDeclaratorSuffix(Type t) 3885 Type parseDeclaratorSuffix(Type t)
3890 { 3886 {
4376 if (trying) 4372 if (trying)
4377 { 4373 {
4378 ++errorCount; 4374 ++errorCount;
4379 return; 4375 return;
4380 } 4376 }
4381 lx.updateErrorLoc(token.start); 4377 auto location = Lexer.getLocation(this.token);
4382 errors ~= new Information(InfoType.Parser, mid, lx.errorLoc.clone, Format(_arguments, _argptr, GetMsg(mid))); 4378 auto msg = Format(_arguments, _argptr, GetMsg(mid));
4379 errors ~= new Information(InfoType.Parser, mid, location, msg);
4383 } 4380 }
4384 } 4381 }