changeset 808:28e1ff1dcfcf

Renamed generate command to highlight and refactored it.
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Sun, 09 Mar 2008 16:10:25 +0100
parents a2880c95eda3
children 7e84472f4e91
files src/cmd/DDoc.d src/cmd/DDocXML.d src/cmd/Generate.d src/cmd/Highlight.d src/main.d
diffstat 5 files changed, 528 insertions(+), 508 deletions(-) [+]
line wrap: on
line diff
--- a/src/cmd/DDoc.d	Sun Mar 09 15:28:24 2008 +0100
+++ b/src/cmd/DDoc.d	Sun Mar 09 16:10:25 2008 +0100
@@ -5,7 +5,7 @@
 module cmd.DDoc;
 
 import cmd.DDocXML;
-import cmd.Generate;
+import cmd.Highlight;
 import dil.doc.Parser;
 import dil.doc.Macro;
 import dil.doc.Doc;
--- a/src/cmd/DDocXML.d	Sun Mar 09 15:28:24 2008 +0100
+++ b/src/cmd/DDocXML.d	Sun Mar 09 16:10:25 2008 +0100
@@ -5,7 +5,7 @@
 module cmd.DDocXML;
 
 import cmd.DDoc;
-import cmd.Generate;
+import cmd.Highlight;
 import dil.doc.Parser;
 import dil.doc.Macro;
 import dil.doc.Doc;
--- a/src/cmd/Generate.d	Sun Mar 09 15:28:24 2008 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,489 +0,0 @@
-/++
-  Author: Aziz Köksal
-  License: GPL3
-+/
-module cmd.Generate;
-
-import dil.ast.DefaultVisitor;
-import dil.ast.Node,
-       dil.ast.Declaration,
-       dil.ast.Statement,
-       dil.ast.Expression,
-       dil.ast.Types;
-import dil.lexer.Lexer;
-import dil.parser.Parser;
-import dil.semantic.Module;
-import dil.SourceText;
-import dil.Information;
-import SettingsLoader;
-import Settings;
-import common;
-
-import tango.io.GrowBuffer;
-import tango.io.Print;
-
-/// Options for the generate command.
-enum GenOption
-{
-  Empty,
-  Tokens = 1,
-  Syntax = 1<<1,
-  HTML   = 1<<2,
-  XML    = 1<<3,
-  PrintLines  = 1<<4
-}
-
-/// Executes the generate command.
-void execute(string filePath, GenOption options, InfoManager infoMan)
-{
-  assert(options != GenOption.Empty);
-  auto mapFilePath = options & GenOption.HTML ? GlobalSettings.htmlMapFile
-                                              : GlobalSettings.xmlMapFile;
-  auto map = TagMapLoader(infoMan).load(mapFilePath);
-  auto tags = new TagMap(map);
-
-  if (infoMan.hasInfo)
-    return;
-
-  if (options & GenOption.Syntax)
-    highlightSyntax(filePath, tags, Stdout, options);
-  else
-    highlightTokens(filePath, tags, Stdout, options);
-}
-
-/// Escapes the characters '<', '>' and '&' with named character entities.
-char[] xml_escape(char[] text)
-{
-  char[] result;
-  foreach(c; text)
-    switch(c)
-    {
-      case '<': result ~= "&lt;";  break;
-      case '>': result ~= "&gt;";  break;
-      case '&': result ~= "&amp;"; break;
-      default:  result ~= c;
-    }
-  if (result.length != text.length)
-    return result;
-  // Nothing escaped. Return original text.
-  delete result;
-  return text;
-}
-
-/// Maps tokens to (format) strings.
-class TagMap
-{
-  string[string] table;
-  string[TOK.MAX] tokenTable;
-
-  this(string[string] table)
-  {
-    this.table = table;
-    Identifier   = this["Identifier", "{0}"];
-    String       = this["String", "{0}"];
-    Char         = this["Char", "{0}"];
-    Number       = this["Number", "{0}"];
-    Keyword      = this["Keyword", "{0}"];
-    LineC        = this["LineC", "{0}"];
-    BlockC       = this["BlockC", "{0}"];
-    NestedC      = this["NestedC", "{0}"];
-    Shebang      = this["Shebang", "{0}"];
-    HLine        = this["HLine", "{0}"];
-    Filespec     = this["Filespec", "{0}"];
-    Illegal      = this["Illegal", "{0}"];
-    Newline      = this["Newline", "{0}"];
-    SpecialToken = this["SpecialToken", "{0}"];
-    Declaration  = this["Declaration", "d"];
-    Statement    = this["Statement", "s"];
-    Expression   = this["Expression", "e"];
-    Type         = this["Type", "t"];
-    Other        = this["Other", "o"];
-    EOF          = this["EOF", ""];
-
-    foreach (i, tokStr; tokToString)
-      if (auto pStr = tokStr in this.table)
-        tokenTable[i] = *pStr;
-  }
-
-  /// Returns the value for str, or 'fallback' if str is not in the table.
-  string opIndex(string str, string fallback = "")
-  {
-    auto p = str in table;
-    if (p)
-      return *p;
-    return fallback;
-  }
-
-  /// Returns the value for tok in O(1) time.
-  string opIndex(TOK tok)
-  {
-    return tokenTable[tok];
-  }
-
-  /// Shortcuts for quick access.
-  string Identifier, String, Char, Number, Keyword, LineC, BlockC,
-         NestedC, Shebang, HLine, Filespec, Illegal, Newline, SpecialToken,
-         Declaration, Statement, Expression, Type, Other, EOF;
-
-  /// Returns the tag for the category 'nc'.
-  string getTag(NodeCategory nc)
-  {
-    string tag;
-    switch (nc)
-    { alias NodeCategory NC;
-    case NC.Declaration: tag = Declaration; break;
-    case NC.Statement:   tag = Statement; break;
-    case NC.Expression:  tag = Expression; break;
-    case NC.Type:        tag = Type; break;
-    case NC.Other:       tag = Other; break;
-    default: assert(0);
-    }
-    return tag;
-  }
-}
-
-/// Find the last occurrence of object in subject.
-/// Returns: the index if found, or -1 if not.
-int rfind(char[] subject, char object)
-{
-  foreach_reverse(i, c; subject)
-    if (c == object)
-      return i;
-  return -1;
-}
-
-/// Returns the short class name of a class descending from Node.$(BR)
-/// E.g.: dil.ast.Declarations.ClassDeclaration -> Class
-char[] getShortClassName(Node node)
-{
-  static char[][] name_table;
-  if (name_table is null)
-    name_table = new char[][NodeKind.max+1]; // Create a new table.
-  // Look up in table.
-  char[] name = name_table[node.kind];
-  if (name !is null)
-    return name; // Return cached name.
-
-  name = node.classinfo.name; // Get the fully qualified name of the class.
-  name = name[rfind(name, '.')+1 .. $]; // Remove package and module name.
-
-  uint suffixLength;
-  switch (node.category)
-  {
-  alias NodeCategory NC;
-  case NC.Declaration:
-    suffixLength = "Declaration".length;
-    break;
-  case NC.Statement:
-    suffixLength = "Statement".length;
-    break;
-  case NC.Expression:
-    suffixLength = "Expression".length;
-    break;
-  case NC.Type:
-    suffixLength = "Type".length;
-    break;
-  case NC.Other:
-    break;
-  default:
-    assert(0);
-  }
-  // Remove common suffix.
-  name = name[0 .. $ - suffixLength];
-  // Store the name in the table.
-  name_table[node.kind] = name;
-  return name;
-}
-
-/// Extended token structure.
-struct TokenEx
-{
-  Token* token; /// The lexer token.
-  Node[] beginNodes; /// beginNodes[n].begin == token
-  Node[] endNodes; /// endNodes[n].end == token
-}
-
-/// Builds an array of TokenEx items.
-class TokenExBuilder : DefaultVisitor
-{
-  private TokenEx*[Token*] tokenTable;
-
-  TokenEx[] build(Node root, Token* first)
-  {
-    auto token = first;
-
-    uint count; // Count tokens.
-    for (; token; token = token.next)
-      count++;
-    // Creat the exact number of TokenEx instances.
-    auto toks = new TokenEx[count];
-    token = first;
-    foreach (ref tokEx; toks)
-    {
-      tokEx.token = token;
-      if (!token.isWhitespace)
-        tokenTable[token] = &tokEx;
-      token = token.next;
-    }
-
-    super.visitN(root);
-    tokenTable = null;
-    return toks;
-  }
-
-  TokenEx* getTokenEx()(Token* t)
-  {
-    auto p = t in tokenTable;
-    assert(p, t.srcText~" is not in tokenTable");
-    return *p;
-  }
-
-  // Override dispatch function.
-  override Node dispatch(Node n)
-  {
-    auto begin = n.begin;
-    if (begin)
-    { assert(n.end);
-      auto txbegin = getTokenEx(begin);
-      auto txend = getTokenEx(n.end);
-      txbegin.beginNodes ~= n;
-      txend.endNodes ~= n;
-    }
-    return super.dispatch(n);
-  }
-}
-
-void printErrors(Lexer lx, TagMap tags, Print!(char) print)
-{
-  foreach (e; lx.errors)
-    print.format(tags["LexerError"], e.filePath, e.loc, e.col, xml_escape(e.getMsg));
-}
-
-void printErrors(Parser parser, TagMap tags, Print!(char) print)
-{
-  foreach (e; parser.errors)
-    print.format(tags["ParserError"], e.filePath, e.loc, e.col, xml_escape(e.getMsg));
-}
-
-void printLines(uint lines, TagMap tags, Print!(char) print)
-{
-  auto lineNumberFormat = tags["LineNumber"];
-  for (auto lineNum = 1; lineNum <= lines; lineNum++)
-    print.format(lineNumberFormat, lineNum);
-}
-
-// void printMultiline(Token* token, TagMap tags, Print!(char) print)
-// {
-// }
-
-/// Highlights the syntax in a source file.
-void highlightSyntax(string filePath, TagMap tags, Print!(char) print, GenOption options)
-{
-  auto parser = new Parser(new SourceText(filePath, true));
-  auto root = parser.start();
-  auto lx = parser.lexer;
-
-  auto builder = new TokenExBuilder();
-  auto tokenExList = builder.build(root, lx.firstToken());
-
-  print(tags["DocHead"]);
-  if (lx.errors.length || parser.errors.length)
-  { // Output error messages.
-    print(tags["CompBegin"]);
-    printErrors(lx, tags, print);
-    printErrors(parser, tags, print);
-    print(tags["CompEnd"]);
-  }
-
-  if (options & GenOption.PrintLines)
-  {
-    print(tags["LineNumberBegin"]);
-    printLines(lx.lineNum, tags, print);
-    print(tags["LineNumberEnd"]);
-  }
-
-  print(tags["SourceBegin"]);
-
-  auto tagNodeBegin = tags["NodeBegin"];
-  auto tagNodeEnd = tags["NodeEnd"];
-
-  // Iterate over list of tokens.
-  foreach (ref tokenEx; tokenExList)
-  {
-    auto token = tokenEx.token;
-
-    token.ws && print(token.wsChars); // Print preceding whitespace.
-    if (token.isWhitespace) {
-      printToken(token, tags, print);
-      continue;
-    }
-    // <node>
-    foreach (node; tokenEx.beginNodes)
-      print.format(tagNodeBegin, tags.getTag(node.category), getShortClassName(node));
-    // Token text.
-    printToken(token, tags, print);
-    // </node>
-    if (options & GenOption.HTML)
-      foreach_reverse (node; tokenEx.endNodes)
-        print(tagNodeEnd);
-    else
-      foreach_reverse (node; tokenEx.endNodes)
-        print.format(tagNodeEnd, tags.getTag(node.category));
-  }
-  print(tags["SourceEnd"]);
-  print(tags["DocEnd"]);
-}
-
-/// Highlights all tokens of a source file.
-void highlightTokens(string filePath, TagMap tags, Print!(char) print, GenOption options)
-{
-  auto lx = new Lexer(new SourceText(filePath, true));
-  lx.scanAll();
-
-  print(tags["DocHead"]);
-  if (lx.errors.length)
-  {
-    print(tags["CompBegin"]);
-    printErrors(lx, tags, print);
-    print(tags["CompEnd"]);
-  }
-
-  if (options & GenOption.PrintLines)
-  {
-    print(tags["LineNumberBegin"]);
-    printLines(lx.lineNum, tags, print);
-    print(tags["LineNumberEnd"]);
-  }
-
-  print(tags["SourceBegin"]);
-  // Traverse linked list and print tokens.
-  for (auto token = lx.firstToken(); token; token = token.next) {
-    token.ws && print(token.wsChars); // Print preceding whitespace.
-    printToken(token, tags, print);
-  }
-  print(tags["SourceEnd"]);
-  print(tags["DocEnd"]);
-}
-
-/// A token highlighter designed for DDoc.
-class TokenHighlighter
-{
-  TagMap tags;
-  this(InfoManager infoMan, bool useHTML = true)
-  {
-    string filePath = GlobalSettings.htmlMapFile;
-    if (!useHTML)
-      filePath = GlobalSettings.xmlMapFile;
-    auto map = TagMapLoader(infoMan).load(filePath);
-    tags = new TagMap(map);
-  }
-
-  /// Highlights tokens in a DDoc code section.
-  /// Returns: a string with the highlighted tokens (in HTML tags.)
-  string highlight(string text, string filePath)
-  {
-    auto buffer = new GrowBuffer(text.length);
-    auto print = new Print!(char)(Format, buffer);
-
-    auto lx = new Lexer(new SourceText(filePath, text));
-    lx.scanAll();
-
-    // Traverse linked list and print tokens.
-    print("$(D_CODE\n");
-    if (lx.errors.length)
-    { // Output error messages.
-      print(tags["CompBegin"]);
-      printErrors(lx, tags, print);
-      print(tags["CompEnd"]);
-    }
-    // Traverse linked list and print tokens.
-    for (auto token = lx.firstToken(); token; token = token.next) {
-      token.ws && print(token.wsChars); // Print preceding whitespace.
-      printToken(token, tags, print);
-    }
-    print("\n)");
-    return cast(char[])buffer.slice();
-  }
-}
-
-/// Prints a token to the stream print.
-void printToken(Token* token, TagMap tags, Print!(char) print)
-{
-  switch(token.kind)
-  {
-  case TOK.Identifier:
-    print.format(tags.Identifier, token.srcText);
-    break;
-  case TOK.Comment:
-    string formatStr;
-    switch (token.start[1])
-    {
-    case '/': formatStr = tags.LineC; break;
-    case '*': formatStr = tags.BlockC; break;
-    case '+': formatStr = tags.NestedC; break;
-    default: assert(0);
-    }
-    print.format(formatStr, xml_escape(token.srcText));
-    break;
-  case TOK.String:
-    print.format(tags.String, xml_escape(token.srcText));
-    break;
-  case TOK.CharLiteral:
-    print.format(tags.Char, xml_escape(token.srcText));
-    break;
-  case TOK.Int32, TOK.Int64, TOK.Uint32, TOK.Uint64,
-       TOK.Float32, TOK.Float64, TOK.Float80,
-       TOK.Imaginary32, TOK.Imaginary64, TOK.Imaginary80:
-    print.format(tags.Number, token.srcText);
-    break;
-  case TOK.Shebang:
-    print.format(tags.Shebang, xml_escape(token.srcText));
-    break;
-  case TOK.HashLine:
-    auto formatStr = tags.HLine;
-    // The text to be inserted into formatStr.
-    auto buffer = new GrowBuffer;
-    auto print2 = new Print!(char)(Format, buffer);
-
-    void printWS(char* start, char* end)
-    {
-      start != end && print2(start[0 .. end - start]);
-    }
-
-    auto num = token.tokLineNum;
-    if (num is null)
-    { // Malformed #line
-      print.format(formatStr, token.srcText);
-      break;
-    }
-
-    // Print whitespace between #line and number.
-    printWS(token.start, num.start); // Prints "#line" as well.
-    printToken(num, tags, print2); // Print the number.
-
-    if (auto filespec = token.tokLineFilespec)
-    { // Print whitespace between number and filespec.
-      printWS(num.end, filespec.start);
-      print2.format(tags.Filespec, xml_escape(filespec.srcText));
-    }
-    // Finally print the whole token.
-    print.format(formatStr, cast(char[])buffer.slice());
-    break;
-  case TOK.Illegal:
-    print.format(tags.Illegal, token.srcText());
-    break;
-  case TOK.Newline:
-    print.format(tags.Newline, token.srcText());
-    break;
-  case TOK.EOF:
-    print(tags.EOF);
-    break;
-  default:
-    if (token.isKeyword())
-      print.format(tags.Keyword, token.srcText);
-    else if (token.isSpecialToken)
-      print.format(tags.SpecialToken, token.srcText);
-    else
-      print(tags[token.kind]);
-  }
-}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/cmd/Highlight.d	Sun Mar 09 16:10:25 2008 +0100
@@ -0,0 +1,507 @@
+/++
+  Author: Aziz Köksal
+  License: GPL3
++/
+module cmd.Highlight;
+
+import dil.ast.DefaultVisitor;
+import dil.ast.Node,
+       dil.ast.Declaration,
+       dil.ast.Statement,
+       dil.ast.Expression,
+       dil.ast.Types;
+import dil.lexer.Lexer;
+import dil.parser.Parser;
+import dil.semantic.Module;
+import dil.SourceText;
+import dil.Information;
+import SettingsLoader;
+import Settings;
+import common;
+
+import tango.io.GrowBuffer;
+import tango.io.Print;
+
+/// The highlight command.
+struct HighlightCommand
+{
+  /// Options for the command.
+  enum Option
+  {
+    None        = 0,
+    Tokens      = 1,
+    Syntax      = 1<<1,
+    HTML        = 1<<2,
+    XML         = 1<<3,
+    PrintLines  = 1<<4
+  }
+  alias Option Options;
+
+  Options options; /// Command options.
+  string filePath; /// File path to the module to be highlighted.
+  InfoManager infoMan;
+
+  /// Adds o to the options.
+  void add(Option o)
+  {
+    options |= o;
+  }
+
+  /// Executes the command.
+  void run()
+  {
+    add(HighlightCommand.Option.Tokens);
+    if (!(options & (Option.XML | Option.HTML)))
+      add(Option.XML); // Default to XML.
+
+    auto mapFilePath = options & Option.HTML ? GlobalSettings.htmlMapFile
+                                             : GlobalSettings.xmlMapFile;
+    auto map = TagMapLoader(infoMan).load(mapFilePath);
+    auto tags = new TagMap(map);
+
+    if (infoMan.hasInfo)
+      return;
+
+    if (options & Option.Syntax)
+      highlightSyntax(filePath, tags, Stdout, options);
+    else
+      highlightTokens(filePath, tags, Stdout, options);
+  }
+}
+
+/// Escapes the characters '<', '>' and '&' with named character entities.
+char[] xml_escape(char[] text)
+{
+  char[] result;
+  foreach(c; text)
+    switch(c)
+    {
+      case '<': result ~= "&lt;";  break;
+      case '>': result ~= "&gt;";  break;
+      case '&': result ~= "&amp;"; break;
+      default:  result ~= c;
+    }
+  if (result.length != text.length)
+    return result;
+  // Nothing escaped. Return original text.
+  delete result;
+  return text;
+}
+
+/// Maps tokens to (format) strings.
+class TagMap
+{
+  string[string] table;
+  string[TOK.MAX] tokenTable;
+
+  this(string[string] table)
+  {
+    this.table = table;
+    Identifier   = this["Identifier", "{0}"];
+    String       = this["String", "{0}"];
+    Char         = this["Char", "{0}"];
+    Number       = this["Number", "{0}"];
+    Keyword      = this["Keyword", "{0}"];
+    LineC        = this["LineC", "{0}"];
+    BlockC       = this["BlockC", "{0}"];
+    NestedC      = this["NestedC", "{0}"];
+    Shebang      = this["Shebang", "{0}"];
+    HLine        = this["HLine", "{0}"];
+    Filespec     = this["Filespec", "{0}"];
+    Illegal      = this["Illegal", "{0}"];
+    Newline      = this["Newline", "{0}"];
+    SpecialToken = this["SpecialToken", "{0}"];
+    Declaration  = this["Declaration", "d"];
+    Statement    = this["Statement", "s"];
+    Expression   = this["Expression", "e"];
+    Type         = this["Type", "t"];
+    Other        = this["Other", "o"];
+    EOF          = this["EOF", ""];
+
+    foreach (i, tokStr; tokToString)
+      if (auto pStr = tokStr in this.table)
+        tokenTable[i] = *pStr;
+  }
+
+  /// Returns the value for str, or 'fallback' if str is not in the table.
+  string opIndex(string str, string fallback = "")
+  {
+    auto p = str in table;
+    if (p)
+      return *p;
+    return fallback;
+  }
+
+  /// Returns the value for tok in O(1) time.
+  string opIndex(TOK tok)
+  {
+    return tokenTable[tok];
+  }
+
+  /// Shortcuts for quick access.
+  string Identifier, String, Char, Number, Keyword, LineC, BlockC,
+         NestedC, Shebang, HLine, Filespec, Illegal, Newline, SpecialToken,
+         Declaration, Statement, Expression, Type, Other, EOF;
+
+  /// Returns the tag for the category 'nc'.
+  string getTag(NodeCategory nc)
+  {
+    string tag;
+    switch (nc)
+    { alias NodeCategory NC;
+    case NC.Declaration: tag = Declaration; break;
+    case NC.Statement:   tag = Statement; break;
+    case NC.Expression:  tag = Expression; break;
+    case NC.Type:        tag = Type; break;
+    case NC.Other:       tag = Other; break;
+    default: assert(0);
+    }
+    return tag;
+  }
+}
+
+/// Find the last occurrence of object in subject.
+/// Returns: the index if found, or -1 if not.
+int rfind(char[] subject, char object)
+{
+  foreach_reverse(i, c; subject)
+    if (c == object)
+      return i;
+  return -1;
+}
+
+/// Returns the short class name of a class descending from Node.$(BR)
+/// E.g.: dil.ast.Declarations.ClassDeclaration -> Class
+char[] getShortClassName(Node node)
+{
+  static char[][] name_table;
+  if (name_table is null)
+    name_table = new char[][NodeKind.max+1]; // Create a new table.
+  // Look up in table.
+  char[] name = name_table[node.kind];
+  if (name !is null)
+    return name; // Return cached name.
+
+  name = node.classinfo.name; // Get the fully qualified name of the class.
+  name = name[rfind(name, '.')+1 .. $]; // Remove package and module name.
+
+  uint suffixLength;
+  switch (node.category)
+  {
+  alias NodeCategory NC;
+  case NC.Declaration:
+    suffixLength = "Declaration".length;
+    break;
+  case NC.Statement:
+    suffixLength = "Statement".length;
+    break;
+  case NC.Expression:
+    suffixLength = "Expression".length;
+    break;
+  case NC.Type:
+    suffixLength = "Type".length;
+    break;
+  case NC.Other:
+    break;
+  default:
+    assert(0);
+  }
+  // Remove common suffix.
+  name = name[0 .. $ - suffixLength];
+  // Store the name in the table.
+  name_table[node.kind] = name;
+  return name;
+}
+
+/// Extended token structure.
+struct TokenEx
+{
+  Token* token; /// The lexer token.
+  Node[] beginNodes; /// beginNodes[n].begin == token
+  Node[] endNodes; /// endNodes[n].end == token
+}
+
+/// Builds an array of TokenEx items.
+class TokenExBuilder : DefaultVisitor
+{
+  private TokenEx*[Token*] tokenTable;
+
+  TokenEx[] build(Node root, Token* first)
+  {
+    auto token = first;
+
+    uint count; // Count tokens.
+    for (; token; token = token.next)
+      count++;
+    // Creat the exact number of TokenEx instances.
+    auto toks = new TokenEx[count];
+    token = first;
+    foreach (ref tokEx; toks)
+    {
+      tokEx.token = token;
+      if (!token.isWhitespace)
+        tokenTable[token] = &tokEx;
+      token = token.next;
+    }
+
+    super.visitN(root);
+    tokenTable = null;
+    return toks;
+  }
+
+  TokenEx* getTokenEx()(Token* t)
+  {
+    auto p = t in tokenTable;
+    assert(p, t.srcText~" is not in tokenTable");
+    return *p;
+  }
+
+  // Override dispatch function.
+  override Node dispatch(Node n)
+  {
+    auto begin = n.begin;
+    if (begin)
+    { assert(n.end);
+      auto txbegin = getTokenEx(begin);
+      auto txend = getTokenEx(n.end);
+      txbegin.beginNodes ~= n;
+      txend.endNodes ~= n;
+    }
+    return super.dispatch(n);
+  }
+}
+
+void printErrors(Lexer lx, TagMap tags, Print!(char) print)
+{
+  foreach (e; lx.errors)
+    print.format(tags["LexerError"], e.filePath, e.loc, e.col, xml_escape(e.getMsg));
+}
+
+void printErrors(Parser parser, TagMap tags, Print!(char) print)
+{
+  foreach (e; parser.errors)
+    print.format(tags["ParserError"], e.filePath, e.loc, e.col, xml_escape(e.getMsg));
+}
+
+void printLines(uint lines, TagMap tags, Print!(char) print)
+{
+  auto lineNumberFormat = tags["LineNumber"];
+  for (auto lineNum = 1; lineNum <= lines; lineNum++)
+    print.format(lineNumberFormat, lineNum);
+}
+
+/// Highlights the syntax in a source file.
+void highlightSyntax(string filePath, TagMap tags,
+                     Print!(char) print,
+                     HighlightCommand.Options options)
+{
+  auto parser = new Parser(new SourceText(filePath, true));
+  auto root = parser.start();
+  auto lx = parser.lexer;
+
+  auto builder = new TokenExBuilder();
+  auto tokenExList = builder.build(root, lx.firstToken());
+
+  print(tags["DocHead"]);
+  if (lx.errors.length || parser.errors.length)
+  { // Output error messages.
+    print(tags["CompBegin"]);
+    printErrors(lx, tags, print);
+    printErrors(parser, tags, print);
+    print(tags["CompEnd"]);
+  }
+
+  if (options & HighlightCommand.Option.PrintLines)
+  {
+    print(tags["LineNumberBegin"]);
+    printLines(lx.lineNum, tags, print);
+    print(tags["LineNumberEnd"]);
+  }
+
+  print(tags["SourceBegin"]);
+
+  auto tagNodeBegin = tags["NodeBegin"];
+  auto tagNodeEnd = tags["NodeEnd"];
+
+  // Iterate over list of tokens.
+  foreach (ref tokenEx; tokenExList)
+  {
+    auto token = tokenEx.token;
+
+    token.ws && print(token.wsChars); // Print preceding whitespace.
+    if (token.isWhitespace) {
+      printToken(token, tags, print);
+      continue;
+    }
+    // <node>
+    foreach (node; tokenEx.beginNodes)
+      print.format(tagNodeBegin, tags.getTag(node.category), getShortClassName(node));
+    // Token text.
+    printToken(token, tags, print);
+    // </node>
+    if (options & HighlightCommand.Option.HTML)
+      foreach_reverse (node; tokenEx.endNodes)
+        print(tagNodeEnd);
+    else
+      foreach_reverse (node; tokenEx.endNodes)
+        print.format(tagNodeEnd, tags.getTag(node.category));
+  }
+  print(tags["SourceEnd"]);
+  print(tags["DocEnd"]);
+}
+
+/// Highlights all tokens of a source file.
+void highlightTokens(string filePath, TagMap tags,
+                     Print!(char) print,
+                     HighlightCommand.Options options)
+{
+  auto lx = new Lexer(new SourceText(filePath, true));
+  lx.scanAll();
+
+  print(tags["DocHead"]);
+  if (lx.errors.length)
+  {
+    print(tags["CompBegin"]);
+    printErrors(lx, tags, print);
+    print(tags["CompEnd"]);
+  }
+
+  if (options & HighlightCommand.Option.PrintLines)
+  {
+    print(tags["LineNumberBegin"]);
+    printLines(lx.lineNum, tags, print);
+    print(tags["LineNumberEnd"]);
+  }
+
+  print(tags["SourceBegin"]);
+  // Traverse linked list and print tokens.
+  for (auto token = lx.firstToken(); token; token = token.next) {
+    token.ws && print(token.wsChars); // Print preceding whitespace.
+    printToken(token, tags, print);
+  }
+  print(tags["SourceEnd"]);
+  print(tags["DocEnd"]);
+}
+
+/// A token highlighter designed for DDoc.
+class TokenHighlighter
+{
+  TagMap tags;
+  this(InfoManager infoMan, bool useHTML = true)
+  {
+    string filePath = GlobalSettings.htmlMapFile;
+    if (!useHTML)
+      filePath = GlobalSettings.xmlMapFile;
+    auto map = TagMapLoader(infoMan).load(filePath);
+    tags = new TagMap(map);
+  }
+
+  /// Highlights tokens in a DDoc code section.
+  /// Returns: a string with the highlighted tokens (in HTML tags.)
+  string highlight(string text, string filePath)
+  {
+    auto buffer = new GrowBuffer(text.length);
+    auto print = new Print!(char)(Format, buffer);
+
+    auto lx = new Lexer(new SourceText(filePath, text));
+    lx.scanAll();
+
+    // Traverse linked list and print tokens.
+    print("$(D_CODE\n");
+    if (lx.errors.length)
+    { // Output error messages.
+      print(tags["CompBegin"]);
+      printErrors(lx, tags, print);
+      print(tags["CompEnd"]);
+    }
+    // Traverse linked list and print tokens.
+    for (auto token = lx.firstToken(); token; token = token.next) {
+      token.ws && print(token.wsChars); // Print preceding whitespace.
+      printToken(token, tags, print);
+    }
+    print("\n)");
+    return cast(char[])buffer.slice();
+  }
+}
+
+/// Prints a token to the stream print.
+void printToken(Token* token, TagMap tags, Print!(char) print)
+{
+  switch(token.kind)
+  {
+  case TOK.Identifier:
+    print.format(tags.Identifier, token.srcText);
+    break;
+  case TOK.Comment:
+    string formatStr;
+    switch (token.start[1])
+    {
+    case '/': formatStr = tags.LineC; break;
+    case '*': formatStr = tags.BlockC; break;
+    case '+': formatStr = tags.NestedC; break;
+    default: assert(0);
+    }
+    print.format(formatStr, xml_escape(token.srcText));
+    break;
+  case TOK.String:
+    print.format(tags.String, xml_escape(token.srcText));
+    break;
+  case TOK.CharLiteral:
+    print.format(tags.Char, xml_escape(token.srcText));
+    break;
+  case TOK.Int32, TOK.Int64, TOK.Uint32, TOK.Uint64,
+       TOK.Float32, TOK.Float64, TOK.Float80,
+       TOK.Imaginary32, TOK.Imaginary64, TOK.Imaginary80:
+    print.format(tags.Number, token.srcText);
+    break;
+  case TOK.Shebang:
+    print.format(tags.Shebang, xml_escape(token.srcText));
+    break;
+  case TOK.HashLine:
+    auto formatStr = tags.HLine;
+    // The text to be inserted into formatStr.
+    auto buffer = new GrowBuffer;
+    auto print2 = new Print!(char)(Format, buffer);
+
+    void printWS(char* start, char* end)
+    {
+      start != end && print2(start[0 .. end - start]);
+    }
+
+    auto num = token.tokLineNum;
+    if (num is null)
+    { // Malformed #line
+      print.format(formatStr, token.srcText);
+      break;
+    }
+
+    // Print whitespace between #line and number.
+    printWS(token.start, num.start); // Prints "#line" as well.
+    printToken(num, tags, print2); // Print the number.
+
+    if (auto filespec = token.tokLineFilespec)
+    { // Print whitespace between number and filespec.
+      printWS(num.end, filespec.start);
+      print2.format(tags.Filespec, xml_escape(filespec.srcText));
+    }
+    // Finally print the whole token.
+    print.format(formatStr, cast(char[])buffer.slice());
+    break;
+  case TOK.Illegal:
+    print.format(tags.Illegal, token.srcText());
+    break;
+  case TOK.Newline:
+    print.format(tags.Newline, token.srcText());
+    break;
+  case TOK.EOF:
+    print(tags.EOF);
+    break;
+  default:
+    if (token.isKeyword())
+      print.format(tags.Keyword, token.srcText);
+    else if (token.isSpecialToken)
+      print.format(tags.SpecialToken, token.srcText);
+    else
+      print(tags[token.kind]);
+  }
+}
--- a/src/main.d	Sun Mar 09 15:28:24 2008 +0100
+++ b/src/main.d	Sun Mar 09 16:10:25 2008 +0100
@@ -24,7 +24,7 @@
 import dil.SourceText;
 import dil.Compilation;
 
-import cmd.Generate;
+import cmd.Highlight;
 import cmd.Statistics;
 import cmd.ImportGraph;
 import cmd.DDoc;
@@ -129,28 +129,30 @@
     cmd.run();
     infoMan.hasInfo && printErrors(infoMan);
     break;
-  case "gen", "generate":
-    char[] fileName;
-    GenOption options = GenOption.Tokens;
+  case "hl", "highlight":
+    if (args.length < 3)
+      return printHelp("hl");
+
+    HighlightCommand cmd;
+    cmd.infoMan = infoMan;
+
     foreach (arg; args[2..$])
     {
       switch (arg)
       {
       case "--syntax":
-        options |= GenOption.Syntax; break;
+        cmd.add(HighlightCommand.Option.Syntax); break;
       case "--xml":
-        options |= GenOption.XML; break;
+        cmd.add(HighlightCommand.Option.XML); break;
       case "--html":
-        options |= GenOption.HTML; break;
+        cmd.add(HighlightCommand.Option.HTML); break;
       case "--lines":
-        options |= GenOption.PrintLines; break;
+        cmd.add(HighlightCommand.Option.PrintLines); break;
       default:
-        fileName = arg;
+        cmd.filePath = arg;
       }
     }
-    if (!(options & (GenOption.XML | GenOption.HTML)))
-      options |= GenOption.XML; // Default to XML.
-    cmd.Generate.execute(fileName, options, infoMan);
+    cmd.run();
     infoMan.hasInfo && printErrors(infoMan);
     break;
   case "importgraph", "igraph":
@@ -326,8 +328,8 @@
 const char[] COMMANDS =
   "  compile (c)\n"
   "  ddoc (d)\n"
-  "  generate (gen)\n"
   "  help (?)\n"
+  "  highlight (hl)\n"
   "  importgraph (igraph)\n"
   "  statistics (stats)\n"
   "  tokenize (tok)\n"
@@ -457,11 +459,11 @@
 Example:
   dil d doc/ src/main.d src/macros_dil.ddoc -i`;
     break;
-  case "gen", "generate":
+  case "hl", "highlight":
 //     msg = GetMsg(MID.HelpGenerate);
-    msg = `Generate an XML or HTML document from a D source file.
+    msg = `Highlight a D source file with XML or HTML tags.
 Usage:
-  dil gen file.d [Options]
+  dil hl file.d [Options]
 
 Options:
   --syntax         : generate tags for the syntax tree
@@ -470,7 +472,7 @@
   --lines          : print line numbers
 
 Example:
-  dil gen Parser.d --html --syntax > Parser.html`;
+  dil hl Parser.d --html --syntax > Parser.html`;
     break;
   case "importgraph", "igraph":
 //     msg = GetMsg(MID.HelpImportGraph);