# HG changeset patch
# User Aziz K?ksal <aziz.koeksal@gmail.com>
# Date 1202518820 -3600
# Node ID f88b5285b86b1c160727be3fa453e3d4424a71b2
# Parent  2eee29aaa3579f46651bb55a6fd683ef273b5e44
Implemented DDocEmitter.

Fixed quite a few bugs.

diff -r 2eee29aaa357 -r f88b5285b86b trunk/src/cmd/DDoc.d
--- a/trunk/src/cmd/DDoc.d	Mon Feb 04 21:55:44 2008 +0200
+++ b/trunk/src/cmd/DDoc.d	Sat Feb 09 02:00:20 2008 +0100
@@ -7,7 +7,14 @@
 import dil.doc.Parser;
 import dil.doc.Macro;
 import dil.doc.Doc;
+import dil.ast.Node;
+import dil.ast.Declarations,
+       dil.ast.Statements,
+       dil.ast.Expression,
+       dil.ast.Parameters,
+       dil.ast.Types;
 import dil.ast.DefaultVisitor;
+import dil.lexer.Token;
 import dil.semantic.Module;
 import dil.semantic.Pass1;
 import dil.semantic.Symbol;
@@ -18,6 +25,9 @@
 
 import tango.stdc.time : time_t, time, ctime;
 import tango.stdc.string : strlen;
+import tango.text.Ascii : toUpper;
+import tango.io.File;
+import tango.io.FilePath;
 
 void execute(string[] filePaths, string destDir, string[] macroPaths,
              bool incUndoc, InfoManager infoMan)
@@ -35,11 +45,9 @@
 //   foreach (k, v; mtable.table)
 //     Stdout(k)("=")(v.text);
 
-  Module[] modules;
   foreach (filePath; filePaths)
   {
     auto mod = new Module(filePath, infoMan);
-    modules ~= mod;
     // Parse the file.
     mod.parse();
     if (mod.hasErrors)
@@ -48,47 +56,470 @@
     // Start semantic analysis.
     auto pass1 = new SemanticPass1(mod);
     pass1.start();
+
+    // Generate documentation.
+    auto dest = new FilePath(destDir);
+    generateDocumentation(dest, mod, mtable, incUndoc);
   }
-
-  foreach (mod; modules)
-    generateDocumentation(mod, mtable);
 }
 
-void generateDocumentation(Module mod, MacroTable mtable)
+void generateDocumentation(FilePath dest, Module mod, MacroTable mtable, bool incUndoc)
 {
   // Create a macro environment for this module.
   mtable = new MacroTable(mtable);
   // Define runtime macros.
-  mtable.insert(new Macro("TITLE", mod.getFQN()));
-  mtable.insert(new Macro("DOCFILENAME", mod.getFQN()));
+  mtable.insert("TITLE", mod.getFQN());
+  mtable.insert("DOCFILENAME", mod.getFQN());
 
   time_t time_val;
   time(&time_val);
   char* str = ctime(&time_val);
   char[] time_str = str[0 .. strlen(str)];
-  mtable.insert(new Macro("DATETIME", time_str.dup));
-  mtable.insert(new Macro("YEAR", time_str[20..24].dup));
+  mtable.insert("DATETIME", time_str.dup);
+  mtable.insert("YEAR", time_str[20..24].dup);
 
-  if (mod.moduleDecl)
-  {
-    auto ddocComment = getDDocComment(mod.moduleDecl);
-    if (auto copyright = ddocComment.getCopyright())
-      mtable.insert(new Macro("COPYRIGHT", copyright.text));
-  }
-
-  auto docEmitter = new DDocEmitter();
-  docEmitter.emit(mod);
-
-  mtable.insert(new Macro("BODY", docEmitter.text));
-  expandMacros(mtable, "$(DDOC)");
+  auto doc = new DDocEmitter(mtable, incUndoc);
+  doc.emit(mod);
+  // Set BODY macro to the text produced by the DDocEmitter.
+  mtable.insert("BODY", doc.text);
+  // Do the macro expansion pass.
+  auto fileText = expandMacros(mtable, "$(DDOC)");
+  // Finally write the file out to the harddisk.
+  dest.append(mod.getFQN() ~ ".html");
+  auto file = new File(dest);
+  file.write(fileText);
 }
 
+/// Traverses the syntax tree and writes DDoc macros to a string buffer.
 class DDocEmitter : DefaultVisitor
 {
   char[] text;
+  bool includeUndocumented;
+  MacroTable mtable;
 
+  this(MacroTable mtable, bool includeUndocumented)
+  {
+    this.mtable = mtable;
+    this.includeUndocumented = includeUndocumented;
+  }
+
+  /// Entry method.
   char[] emit(Module mod)
   {
+    if (auto d = mod.moduleDecl)
+    {
+      if (ddoc(d))
+      {
+        if (auto copyright = cmnt.takeCopyright())
+          mtable.insert(new Macro("COPYRIGHT", copyright.text));
+        DESC({ writeComment(); });
+      }
+    }
+    MEMBERS("MODULE", { visitD(mod.root); });
+    write(\n);
     return text;
   }
+
+  char[] textSpan(Token* left, Token* right)
+  {
+    //assert(left && right && (left.end <= right.start || left is right));
+    //char[] result;
+    //TODO: filter out whitespace tokens.
+    return Token.textSpan(left, right);
+  }
+
+  bool isTemplatized; /// True if an aggregate declaration is templatized.
+  TemplateParameters tparams; /// The template parameters of the declaration.
+
+  DDocComment cmnt; /// Current comment.
+  DDocComment prevCmnt; /// Previous comment in scope.
+  /// An empty comment. Used for undocumented symbols.
+  static const DDocComment emptyCmnt;
+
+  static this()
+  {
+    this.emptyCmnt = new DDocComment(null, null, null);
+  }
+
+  /// Keeps track of previous comments in each scope.
+  scope class Scope
+  {
+    DDocComment old_prevCmnt;
+    this()
+    { // Save the previous comment of the parent scope.
+      old_prevCmnt = this.outer.prevCmnt;
+      // Entering a new scope. Set to null.
+      this.outer.prevCmnt = null;
+    }
+
+    ~this()
+    { // Restore the previous comment of the parent scope.
+      this.outer.prevCmnt = old_prevCmnt;
+    }
+  }
+
+  DDocComment ddoc(Node node)
+  {
+    auto c = getDDocComment(node);
+    this.cmnt = null;
+    if (c)
+    {
+      if (c.isDitto)
+        this.cmnt = this.prevCmnt;
+      else
+      {
+        this.cmnt = c;
+        this.prevCmnt = c;
+      }
+    }
+    else if (includeUndocumented)
+      this.cmnt = this.emptyCmnt;
+    return this.cmnt;
+  }
+
+  static char[][char[]] specialSections;
+  static this()
+  {
+    foreach (name; ["AUTHORS", "BUGS", "COPYRIGHT", "DATE", "DEPRECATED",
+                    "EXAMPLES", "HISTORY", "LICENSE", "RETURNS", "SEE_ALSO",
+                    "STANDARDS", "THROWS", "VERSION"])
+      specialSections[name] = name;
+  }
+
+  void writeComment()
+  {
+    auto c = this.cmnt;
+    assert(c !is null);
+    if (c.sections.length == 0)
+      return;
+    write("$(DDOC_SECTIONS ");
+      foreach (s; c.sections)
+      {
+        if (s is c.summary)
+          write("\n$(DDOC_SUMMARY ");
+        else if (s is c.description)
+          write("\n$(DDOC_DESCRIPTION ");
+        else if (auto name = toUpper(s.name.dup) in specialSections)
+          write("\n$(DDOC_" ~ *name ~ " ");
+        else if (s.Is("params"))
+        { // Process parameters section.
+          auto ps = new ParamsSection(s.name, s.text);
+          write("\n$(DDOC_PARAMS ");
+          foreach (i, paramName; ps.paramNames)
+            write("$(DDOC_PARAM_ROW ",
+                    "$(DDOC_PARAM_ID ", paramName, ")",
+                    "$(DDOC_PARAM_DESC ", ps.paramDescs[i], ")",
+                  ")");
+          write(")");
+          continue;
+        }
+        else if (s.Is("macros"))
+        { // Declare the macros in this section.
+          auto ms = new MacrosSection(s.name, s.text);
+          mtable.insert(ms.macroNames, ms.macroTexts);
+          continue;
+        }
+        else
+          write("\n$(DDOC_SECTION $(DDOC_SECTION_H " ~ s.name ~ ":)");
+        write(s.text, ")");
+      }
+    write(")");
+  }
+
+  void writeTemplateParams()
+  {
+    if (!isTemplatized)
+      return;
+    text ~= "(" ~ (tparams ? textSpan(tparams.begin, tparams.end) : "") ~ ")";
+    isTemplatized = false;
+    tparams = null;
+  }
+
+  void writeInheritanceList(BaseClassType[] bases)
+  {
+    if (bases.length == 0)
+      return;
+    text ~= " : " ~ textSpan(bases[0].begin, bases[$-1].end);
+  }
+
+  void writeFuncHeader(Declaration d, FuncBodyStatement s)
+  {
+    auto begin = d.begin;
+    auto end = d.end.prev;
+    if (!s.isEmpty)
+      end = s.begin.prev;
+    text ~= textSpan(begin, end);
+  }
+
+  void write(char[][] strings...)
+  {
+    foreach (s; strings)
+      text ~= s;
+  }
+
+  void SYMBOL(char[][] strings...)
+  {
+    write("$(DDOC_PSYMBOL ");
+    write(strings);
+    write(")");
+  }
+
+  void DECL(void delegate() dg)
+  {
+    write("\n$(DDOC_DECL ");
+    dg();
+    write(")");
+  }
+
+  void DESC(void delegate() dg)
+  {
+    write("\n$(DDOC_DECL_DD ");
+    dg();
+    write(")");
+  }
+
+  void MEMBERS(char[] kind, void delegate() dg)
+  {
+    write("\n$(DDOC_"~kind~"_MEMBERS ");
+    dg();
+    write(")");
+  }
+
+  void writeClassOrInterface(T)(T d)
+  {
+    if (!ddoc(d))
+      return d;
+    scope s = new Scope();
+    DECL({
+      write(d.begin.srcText, " ");
+      SYMBOL(d.name.str);
+      writeTemplateParams();
+      writeInheritanceList(d.bases);
+    });
+    DESC({
+      writeComment();
+      MEMBERS(is(T == ClassDeclaration) ? "CLASS" : "INTERFACE", {
+        d.decls && super.visit(d.decls);
+      });
+    });
+  }
+
+  void writeStructOrUnion(T)(T d)
+  {
+    if (!ddoc(d))
+      return d;
+    scope s = new Scope();
+    DECL({
+      write(d.begin.srcText, d.name ? " " : "");
+      if (d.name)
+        SYMBOL(d.name.str);
+      writeTemplateParams();
+    });
+    DESC({
+      writeComment();
+      MEMBERS(is(T == StructDeclaration) ? "STRUCT" : "UNION", {
+        d.decls && super.visit(d.decls);
+      });
+    });
+  }
+
+  alias Declaration D;
+
+override:
+//   D visit(ModuleDeclaration d)
+//   { return d; }
+
+  D visit(AliasDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    DECL({ write(textSpan(d.begin, d.end)); });
+    DESC({ writeComment(); });
+    return d;
+  }
+
+  D visit(TypedefDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    DECL({ write(textSpan(d.begin, d.end)); });
+    DESC({ writeComment(); });
+    return d;
+  }
+
+  D visit(EnumDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    scope s = new Scope();
+    DECL({
+      write("enum", d.name ? " " : "");
+      d.name && SYMBOL(d.name.str);
+    });
+    DESC({
+      writeComment();
+      MEMBERS("ENUM", { super.visit(d); });
+    });
+    return d;
+  }
+
+  D visit(EnumMemberDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    DECL({ SYMBOL(d.name.str); });
+    DESC({ writeComment(); });
+    return d;
+  }
+
+  D visit(TemplateDeclaration d)
+  {
+    if (d.begin.kind != TOK.Template)
+    { // This is a templatized class/interface/struct/union.
+      this.isTemplatized = true;
+      this.tparams = d.tparams;
+      return super.visit(d.decls);
+    }
+    if (!ddoc(d))
+      return d;
+    scope s = new Scope();
+    DECL({
+      write("template ");
+      SYMBOL(d.name.str);
+      write(textSpan(d.begin.next.next, d.decls.begin.prev));
+    });
+    DESC({
+      writeComment();
+      MEMBERS("TEMPLATE", {
+        super.visit(d.decls);
+      });
+    });
+    return d;
+  }
+
+  D visit(ClassDeclaration d)
+  {
+    writeClassOrInterface(d);
+    return d;
+  }
+
+  D visit(InterfaceDeclaration d)
+  {
+    writeClassOrInterface(d);
+    return d;
+  }
+
+  D visit(StructDeclaration d)
+  {
+    writeStructOrUnion(d);
+    return d;
+  }
+
+  D visit(UnionDeclaration d)
+  {
+    writeStructOrUnion(d);
+    return d;
+  }
+
+  D visit(ConstructorDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    DECL({ writeFuncHeader(d, d.funcBody); });
+    DESC({ writeComment(); });
+    return d;
+  }
+
+  D visit(StaticConstructorDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    DECL({ writeFuncHeader(d, d.funcBody); });
+    DESC({ writeComment(); });
+    return d;
+  }
+
+  D visit(DestructorDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    DECL({ writeFuncHeader(d, d.funcBody); });
+    DESC({ writeComment(); });
+    return d;
+  }
+
+  D visit(StaticDestructorDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    DECL({ writeFuncHeader(d, d.funcBody); });
+    DESC({ writeComment(); });
+    return d;
+  }
+
+  D visit(FunctionDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    DECL({ writeFuncHeader(d, d.funcBody); });
+    DESC({ writeComment(); });
+    return d;
+  }
+
+  D visit(NewDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    DECL({ writeFuncHeader(d, d.funcBody); });
+    DESC({ writeComment(); });
+    return d;
+  }
+
+  D visit(DeleteDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    DECL({ writeFuncHeader(d, d.funcBody); });
+    DESC({ writeComment(); });
+    return d;
+  }
+
+  D visit(VariablesDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    char[] type = "auto";
+    if (d.typeNode)
+      type = textSpan(d.typeNode.baseType.begin, d.typeNode.end);
+    foreach (name; d.names)
+    {
+      DECL({ write(type, " "); SYMBOL(name.str); });
+      DESC({ writeComment(); });
+    }
+    return d;
+  }
+
+  D visit(InvariantDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    DECL({ write("invariant"); });
+    DESC({ writeComment(); });
+    return d;
+  }
+
+  D visit(UnittestDeclaration d)
+  {
+    if (!ddoc(d))
+      return d;
+    DECL({ write("unittest"); });
+    DESC({ writeComment(); });
+    return d;
+  }
+
+  D visit(DebugDeclaration d)
+  { return d; }
+
+  D visit(VersionDeclaration d)
+  { return d; }
 }
diff -r 2eee29aaa357 -r f88b5285b86b trunk/src/dil/Unicode.d
--- a/trunk/src/dil/Unicode.d	Mon Feb 04 21:55:44 2008 +0200
+++ b/trunk/src/dil/Unicode.d	Sat Feb 09 02:00:20 2008 +0100
@@ -60,7 +60,7 @@
 /// index is set one past the last trail byte of the valid UTF-8 sequence.
 dchar decode(char[] str, ref size_t index)
 in { assert(str.length && index < str.length); }
-out(c) { assert(isValidChar(c)); }
+out(c) { assert(isValidChar(c) || c == ERROR_CHAR); }
 body
 {
   char* p = str.ptr + index;
@@ -74,7 +74,7 @@
 /// ref_p is set to the last trail byte of the valid UTF-8 sequence.
 dchar decode(ref char* ref_p, char* end)
 in { assert(ref_p && ref_p < end); }
-out(c) { assert(isValidChar(c)); }
+out(c) { assert(isValidChar(c) || c == ERROR_CHAR); }
 body
 {
   char* p = ref_p;
diff -r 2eee29aaa357 -r f88b5285b86b trunk/src/dil/ast/Statements.d
--- a/trunk/src/dil/ast/Statements.d	Mon Feb 04 21:55:44 2008 +0200
+++ b/trunk/src/dil/ast/Statements.d	Sat Feb 09 02:00:20 2008 +0100
@@ -56,6 +56,11 @@
     addOptChild(inBody);
     addOptChild(outBody);
   }
+
+  bool isEmpty()
+  {
+    return funcBody is null;
+  }
 }
 
 class ScopeStatement : Statement
diff -r 2eee29aaa357 -r f88b5285b86b trunk/src/dil/ast/Type.d
--- a/trunk/src/dil/ast/Type.d	Mon Feb 04 21:55:44 2008 +0200
+++ b/trunk/src/dil/ast/Type.d	Sat Feb 09 02:00:20 2008 +0100
@@ -24,4 +24,12 @@
     addOptChild(next);
     this.next = next;
   }
+
+  TypeNode baseType()
+  {
+    auto type = this;
+    while (type.next)
+      type = type.next;
+    return type;
+  }
 }
diff -r 2eee29aaa357 -r f88b5285b86b trunk/src/dil/doc/Doc.d
--- a/trunk/src/dil/doc/Doc.d	Mon Feb 04 21:55:44 2008 +0200
+++ b/trunk/src/dil/doc/Doc.d	Sat Feb 09 02:00:20 2008 +0100
@@ -14,7 +14,7 @@
 
 class DDocComment
 {
-  Section[] sections;
+  Section[] sections; /// The sections of this comment.
   Section summary; /// Optional summary section.
   Section description; /// Optional description section.
 
@@ -25,13 +25,35 @@
     this.description = description;
   }
 
-  Section getCopyright()
+  /// Removes the first copyright section and returns it.
+  Section takeCopyright()
   {
-    foreach (section; sections)
-      if (toLower(section.name) == "copyright")
+    foreach (i, section; sections)
+      if (section.Is("copyright"))
+      {
+        sections = sections[0..i] ~ sections[i+1..$];
         return section;
+      }
     return null;
   }
+
+  /// Returns: true if "ditto" is the only text in this comment.
+  bool isDitto()
+  {
+    if (summary && sections.length == 1 &&
+        toLower(strip(summary.text.dup)) == "ditto")
+      return true;
+    return false;
+  }
+
+//   MacrosSection[] getMacros()
+//   {
+//     MacrosSection[] macros;
+//     foreach (section; sections)
+//       if (section.Is("macros"))
+//         macros ~= new MacrosSection(section.name, section.text);
+//     return macros;
+//   }
 }
 
 /// Returns a node's DDocComment.
@@ -39,14 +61,37 @@
 {
   DDocParser p;
   p.parse(getDDocText(getDocTokens(node)));
-  return new DDocComment(p.sections, p.summary, p.description);
+  if (p.sections.length)
+    return new DDocComment(p.sections, p.summary, p.description);
+  return null;
 }
 
+/// Strips leading and trailing whitespace characters.
+/// Whitespace: ' ', '\t', '\v', '\f' and '\n'
+char[] strip(char[] str)
+{
+  if (str.length == 0)
+    return null;
+  uint i;
+  for (; i < str.length; i++)
+    if (!isspace(str[i]) && str[i] != '\n')
+      break;
+  if (str.length == i)
+    return null;
+  str = str[i..$];
+  assert(str.length);
+  for (i = str.length; i; i--)
+    if (!isspace(str[i-1]) && str[i-1] != '\n')
+      break;
+  return str[0..i];
+}
+
+/// Parses a DDoc comment string.
 struct DDocParser
 {
   char* p;
   char* textEnd;
-  Section[] sections;
+  Section[] sections; /// Parsed sections.
   Section summary; /// Optional summary section.
   Section description; /// Optional description section.
 
@@ -55,8 +100,6 @@
   {
     if (!text.length)
       return null;
-    if (text[$-1] != '\0')
-      text ~= '\0';
     p = text.ptr;
     textEnd = p + text.length;
 
@@ -75,10 +118,11 @@
     else // There are no explicit sections.
     {
       scanSummaryAndDescription(summaryBegin, textEnd);
-      return null;
+      return sections;
     }
 
     assert(idBegin && idEnd);
+    // Continue parsing.
     while (findNextIdColon(nextIdBegin, nextIdEnd))
     {
       sections ~= new Section(makeString(idBegin, idEnd), makeString(idEnd+1, nextIdBegin));
@@ -95,16 +139,21 @@
     assert(p < end);
     char* sectionBegin = p;
     // Search for the end of the first paragraph.
+    end--; // Decrement end, so we can look ahead one character.
     while (p < end && !(*p == '\n' && p[1] == '\n'))
       p++;
+    end++;
+    if (p+1 >= end)
+      p = end;
+    assert(p == end || (*p == '\n' && p[1] == '\n'));
     // The first paragraph is the summary.
     summary = new Section("", makeString(sectionBegin, p));
     sections ~= summary;
     // The rest is the description section.
-    if (p != end)
+    if (p < end)
     {
+      skipWhitespace(p);
       sectionBegin = p;
-      skipWhitespace(p);
       if (p < end)
       {
         description = new Section("", makeString(sectionBegin, end));
@@ -115,43 +164,39 @@
 
   void skipWhitespace(ref char* p)
   {
-    while (isspace(*p) || *p == '\n')
+    while (p < textEnd && (isspace(*p) || *p == '\n'))
       p++;
   }
 
   /// Find next "Identifier:".
   /// Params:
-  ///   p       = current character pointer
   ///   idBegin = set to the first character of the Identifier
   ///   idEnd   = set to the colon following the Identifier
   /// Returns: true if found
   bool findNextIdColon(ref char* ref_idBegin, ref char* ref_idEnd)
   {
-    auto p = this.p;
-    while (*p != '\0')
+    while (p < textEnd)
     {
+      skipWhitespace(p);
+      if (p >= textEnd)
+        break;
+      assert(isascii(*p) || isLeadByte(*p));
       auto idBegin = p;
-      assert(isascii(*p) || isLeadByte(*p));
       if (isidbeg(*p) || isUnicodeAlpha(p, textEnd)) // IdStart
       {
         do // IdChar*
           p++;
-        while (isident(*p) || isUnicodeAlpha(p, textEnd))
-        if (*p == ':') // :
+        while (p < textEnd && (isident(*p) || isUnicodeAlpha(p, textEnd)))
+        if (p < textEnd && *p == ':') // :
         {
           ref_idBegin = idBegin;
           ref_idEnd = p;
-          this.p = p;
           return true;
         }
       }
-      else if (!isascii(*p))
-      { // Skip UTF-8 sequences.
-        while (!isascii(*++p))
-        {}
-        continue;
-      }
-      p++;
+      // Skip this line.
+      while (p < textEnd && *p != '\n')
+        p++;
     }
     return false;
   }
@@ -166,6 +211,11 @@
     this.name = name;
     this.text = text;
   }
+
+  bool Is(char[] name2)
+  {
+    return toLower(name.dup) == name2;
+  }
 }
 
 class ParamsSection : Section
@@ -277,13 +327,21 @@
 /// Extracts the text body of the comment tokens.
 string getDDocText(Token*[] tokens)
 {
+  if (tokens.length == 0)
+    return null;
   string result;
   foreach (token; tokens)
   {
     auto n = isLineComment(token) ? 0 : 2; // 0 for "//", 2 for "+/" and "*/".
     result ~= sanitize(token.srcText[3 .. $-n], token.start[1]);
+    assert(token.next);
+    if (token.next.kind == TOK.Newline)
+      result ~= \n;
+    else
+      result ~= ' ';
   }
-  return result;
+//   Stdout.formatln("→{}←", result);
+  return result[0..$-1]; // Remove \n or ' '
 }
 
 /// Sanitizes a DDoc comment string.
@@ -294,46 +352,50 @@
 ///   commentChar = '/', '+', or '*'
 string sanitize(string comment, char commentChar)
 {
-  string result = comment.dup ~ '\0';
+  alias comment result;
 
-  assert(result[$-1] == '\0');
-  bool newline = true; // Indicates whether a newline has been encountered.
+  bool newline = true; // True when at the beginning of a new line.
   uint i, j;
-  for (; i < result.length; i++)
+  auto len = result.length;
+  for (; i < len; i++, j++)
   {
     if (newline)
     { // Ignore commentChars at the beginning of each new line.
       newline = false;
-      while (isspace(result[i]))
-      { i++; }
-      while (result[i] == commentChar)
-      { i++; }
+      while (i < len && isspace(result[i]))
+        i++;
+      while (i < len && result[i] == commentChar)
+        i++;
+      if (i >= len)
+        break;
     }
     // Check for Newline.
     switch (result[i])
     {
     case '\r':
-      if (result[i+1] == '\n')
+      if (i+1 < len && result[i+1] == '\n')
         i++;
     case '\n':
-      result[j++] = '\n'; // Copy Newline as '\n'.
+      result[j] = '\n'; // Copy Newline as '\n'.
       newline = true;
       continue;
     default:
-      if (isUnicodeNewline(result.ptr + i))
+      if (!isascii(result[i]) && i+2 < len && isUnicodeNewline(result.ptr + i))
       {
         i++; i++;
         goto case '\n';
       }
     }
     // Copy character.
-    result[j++] = result[i];
+    result[j] = result[i];
   }
-  result.length = j - 1; // Adjust length. -1 removes '\0'.
+  result.length = j; // Adjust length.
   // Lastly, strip trailing commentChars.
+  if (!result.length)
+    return null;
   i = result.length;
-  while (--i && result[i] == commentChar)
+  for (; i && result[i-1] == commentChar; i--)
   {}
-  result.length = i + 1;
+  result.length = i;
   return result;
 }
diff -r 2eee29aaa357 -r f88b5285b86b trunk/src/dil/doc/Macro.d
--- a/trunk/src/dil/doc/Macro.d	Mon Feb 04 21:55:44 2008 +0200
+++ b/trunk/src/dil/doc/Macro.d	Sat Feb 09 02:00:20 2008 +0100
@@ -42,6 +42,18 @@
       insert(macro_);
   }
 
+  void insert(string name, string text)
+  {
+    insert(new Macro(name, text));
+  }
+
+  void insert(string[] names, string[] texts)
+  {
+    assert(names.length == texts.length);
+    foreach (i, name; names)
+      insert(name, texts[i]);
+  }
+
   Macro search(string name)
   {
     auto pmacro = name in table;
diff -r 2eee29aaa357 -r f88b5285b86b trunk/src/dil/doc/Parser.d
--- a/trunk/src/dil/doc/Parser.d	Mon Feb 04 21:55:44 2008 +0200
+++ b/trunk/src/dil/doc/Parser.d	Sat Feb 09 02:00:20 2008 +0100
@@ -60,6 +60,8 @@
     while (p < textEnd)
     {
       skipWhitespace();
+      if (p >= textEnd)
+        break;
       auto idBegin = p;
       if (isidbeg(*p) || isUnicodeAlpha(p, textEnd)) // IdStart
       {
@@ -69,7 +71,7 @@
         auto idEnd = p;
 
         skipWhitespace();
-        if (*p == '=')
+        if (p < textEnd && *p == '=')
         {
           p++;
           skipWhitespace();
diff -r 2eee29aaa357 -r f88b5285b86b trunk/src/dil/lexer/Lexer.d
--- a/trunk/src/dil/lexer/Lexer.d	Mon Feb 04 21:55:44 2008 +0200
+++ b/trunk/src/dil/lexer/Lexer.d	Sat Feb 09 02:00:20 2008 +0100
@@ -13,11 +13,12 @@
 import dil.HtmlEntities;
 import dil.CompilerInfo;
 import dil.Unicode;
+import common;
+
 import tango.stdc.stdlib : strtof, strtod, strtold;
 import tango.stdc.errno : errno, ERANGE;
 import tango.stdc.time : time_t, time, ctime;
 import tango.stdc.string : strlen;
-import common;
 
 public import dil.lexer.Funcs;
 
@@ -2372,6 +2373,7 @@
 
   /++
     Insert an empty dummy token before t.
+
     Useful in the parsing phase for representing a node in the AST
     that doesn't consume an actual token from the source text.
   +/
diff -r 2eee29aaa357 -r f88b5285b86b trunk/src/main.d
--- a/trunk/src/main.d	Mon Feb 04 21:55:44 2008 +0200
+++ b/trunk/src/main.d	Sat Feb 09 02:00:20 2008 +0100
@@ -97,7 +97,7 @@
     {
       if (arg == "-i")
         incUndoc = true;
-      else if (arg.length > 5 && toLower(arg[$-4..$]) == "ddoc")
+      else if (arg.length > 5 && toLower(arg[$-4..$].dup) == "ddoc")
         macroPaths ~= arg;
       else
         filePaths ~= arg;