--- a/trunk/src/Lexer.d	Fri Jun 22 14:12:02 2007 +0000
+++ b/trunk/src/Lexer.d	Fri Jun 22 19:33:02 2007 +0000
@@ -85,35 +85,45 @@
   public void scan(out Token t)
   {
     assert(p < end);
-    t.start = p;
 
     char c = *p;
 
-    if (c == '\0')
-    {
-      t.type = TOK.EOF;
-      t.end = p+1;
-      return;
-    }
-
-    if (!isident(c) || isdigit(c))
+    while(1)
     {
-      do
-        c = *++p;
-      while ((!isident(c) || isdigit(c)) && c != '\0')
-      t.type = TOK.Whitespace;
-      t.end = p;
-      return;
-    }
+      t.start = p;
+      if (c == 0)
+      {
+        t.type = TOK.EOF;
+        t.end = p+1;
+        return;
+      }
 
-    if (isident(c) && !isdigit(c))
-    {
-      do
-      { c = *++p; }
-      while (isident(c))
-      t.type = TOK.Identifier;
-      t.end = p;
-      return;
+      if (isident(c) && !isdigit(c))
+      {
+        do
+        { c = *++p; }
+        while (isident(c))
+        t.type = TOK.Identifier;
+        t.end = p;
+        return;
+      }
+
+      if (c == '/' && p[1] == '+')
+      {
+        ++p;
+        do
+        {
+          c = *++p;
+          if (c == 0)
+            throw new Error("unterminated /+ +/ comment.");
+        } while (c != '+' || p[1] != '/')
+        p += 2;
+        t.type = TOK.Comment;
+        t.end = p;
+        return;
+      }
+
+      c = *++p;
     }
   }
 
@@ -131,4 +141,4 @@
     tokens ~= this.token;
     return tokens;
   }
-}
\ No newline at end of file
+}

--- a/trunk/src/main.d	Fri Jun 22 14:12:02 2007 +0000
+++ b/trunk/src/main.d	Fri Jun 22 19:33:02 2007 +0000
@@ -8,14 +8,50 @@
 import std.stdio;
 import std.file;
 
+char[] xmlescape(char[] text)
+{
+  char[] result;
+  foreach(c; text)
+    switch(c)
+    {
+      case '<': result ~= "&lt;";  break;
+      case '>': result ~= "&gt;";  break;
+      case '&': result ~= "&amp;"; break;
+      default:  result ~= c;
+    }
+  return result;
+}
+
 void main(char[][] args)
 {
   auto srctext = cast(char[]) std.file.read(args[1]);
   auto lx = new Lexer(srctext);
 
-  foreach(token; lx.getTokens())
+  auto tokens = lx.getTokens();
+  char* end = lx.text.ptr;
+
+  writef(`<?xml version="1.0"?>
+<?xml-stylesheet href="format.css" type="text/css"?>
+<sourcetext>`);
+  foreach(ref token; tokens)
   {
-    if (token.type == TOK.Whitespace)
-      writefln("%s", token.start[0..token.end-token.start]);
+    if (end != token.start)
+      writef("%s", xmlescape(end[0 .. token.start - end]));
+    char[] span = xmlescape(token.start[0 .. token.end-token.start]);
+    switch(token.type)
+    {
+      case TOK.Identifier:
+        writef("<i>%s</i>", span);
+      break;
+      case TOK.Whitespace:
+        writef(span);
+      break;
+      case TOK.Comment:
+        writef("<c>%s</c>", span);
+      break;
+      default:
+    }
+    end = token.end;
   }
+  writef("</sourcetext>");
 }
\ No newline at end of file