Mercurial > projects > dang
diff lexer/Lexer.d @ 42:4e879f82dd64 new_gen
Added some docs for the lexer - now you can understand _some_ of the madness going on here :)
author | Anders Johnsen <skabet@gmail.com> |
---|---|
date | Tue, 22 Apr 2008 22:25:07 +0200 |
parents | f977aa28eb32 |
children | a712c530b7cc |
line wrap: on
line diff
--- a/lexer/Lexer.d Tue Apr 22 19:30:51 2008 +0200 +++ b/lexer/Lexer.d Tue Apr 22 22:25:07 2008 +0200 @@ -8,9 +8,24 @@ import tango.io.Stdout; +/** + The Lexer class will supply you with methods to tokenize a D file. Supply the + Lexer with a DataSource and you can 'peek' and 'next' Tokens from the file. + + For more info about Tokens, look up the lexer.Token module. +*/ class Lexer { public: + + /** + Create a new Lexer. + + params: + source = The source to tokenize. + + */ + this (DataSource source) { this.source = source; @@ -50,6 +65,13 @@ symbolFunctions['/'] = ÷ } + /** + Get the next token from the source. This method will move the + internal position forward to the next Token. + + return: A Token - Token.type is equals TokType.EOF if there is + no more tokens in the file. + */ Token next () { switch (getNextChar) @@ -73,6 +95,13 @@ } } + /** + Get the next token from the source. This method will NOT move the + internal position forward, and thereby having no side-effects. + + return: A Token - Token.type is equals TokType.EOF if there is + no more tokens in the file. + */ Token peek ( int skip = 0) { int oldPosition = this.position; @@ -83,6 +112,11 @@ return t; } + /** + Return all errors that occurred while tokenizing the string. + + TODO: Error system not implemented yet - this is a stub! + */ public Error[] getErrors() { return this.errors;