diff lexer/Lexer.d @ 42:4e879f82dd64 new_gen

Added some docs for the lexer - now you can understand _some_ of the madness going on here :)
author Anders Johnsen <skabet@gmail.com>
date Tue, 22 Apr 2008 22:25:07 +0200
parents f977aa28eb32
children a712c530b7cc
line wrap: on
line diff
--- a/lexer/Lexer.d	Tue Apr 22 19:30:51 2008 +0200
+++ b/lexer/Lexer.d	Tue Apr 22 22:25:07 2008 +0200
@@ -8,9 +8,24 @@
 
 import tango.io.Stdout;
 
+/**
+  The Lexer class will supply you with methods to tokenize a D file. Supply the
+  Lexer with a DataSource and you can 'peek' and 'next' Tokens from the file. 
+
+  For more info about Tokens, look up the lexer.Token module.
+*/  
 class Lexer
 {
 public:
+
+    /**
+      Create a new Lexer.
+
+      params:
+        source = The source to tokenize.
+
+    */
+
     this (DataSource source)
     {
         this.source = source;
@@ -50,6 +65,13 @@
         symbolFunctions['/'] = &div;
     }
 
+    /**
+      Get the next token from the source. This method will move the
+      internal position forward to the next Token.
+
+      return: A Token - Token.type is equals TokType.EOF if there is
+        no more tokens in the file.
+      */
     Token next ()
     {
         switch (getNextChar)
@@ -73,6 +95,13 @@
         }
     }
 
+    /**
+      Get the next token from the source. This method will NOT move the
+      internal position forward, and thereby having no side-effects.
+
+      return: A Token - Token.type is equals TokType.EOF if there is
+        no more tokens in the file.
+      */
     Token peek ( int skip = 0)
     {
         int oldPosition = this.position;
@@ -83,6 +112,11 @@
         return t;
     }
 
+    /**
+      Return all errors that occurred while tokenizing the string.
+
+        TODO: Error system not implemented yet - this is a stub!
+      */
     public Error[] getErrors()
     {
         return this.errors;