Mercurial > projects > dil
view trunk/src/cmd/Generate.d @ 389:c4bfceab7246
Applied fixes and improvements to hex float scanner.
'0x1234i' and '0x1234Li' were wrongly matched as hex floats.
Refactored scanHexReal(). It's faster now and easier to read.
Renamed MID.HexFloatMissingExpDigits to MID.HexFloatExpMustStartWithDigit.
Renamed MID.FloatExponentDigitExpected to MID.FloatExpMustStartWidhtDigit.
Fix in scanSpecialTokenSequence(): --p must come after if statement.
author | Aziz K?ksal <aziz.koeksal@gmail.com> |
---|---|
date | Wed, 12 Sep 2007 18:18:29 +0200 |
parents | ae154eceba65 |
children | 4d36eea1bbc9 |
line wrap: on
line source
/++ Author: Aziz Köksal License: GPL3 +/ module cmd.Generate; import dil.SyntaxTree; import dil.Token; import dil.Parser, dil.Lexer; import dil.File; import std.stdio; enum DocOption { Tokens, Syntax = 1<<1, HTML = 1<<2, XML = 1<<3 } void execute(string fileName, DocOption options) { if (options & DocOption.Syntax) syntaxToDoc(fileName, options); else tokensToDoc(fileName, options); } char[] xml_escape(char[] text) { char[] result; foreach(c; text) switch(c) { case '<': result ~= "<"; break; case '>': result ~= ">"; break; case '&': result ~= "&"; break; default: result ~= c; } return result; } char[] getShortClassName(Node n) { static char[][] name_table; if (name_table is null) name_table = new char[][NodeKind.max+1]; char[] name = name_table[n.kind]; if (name !is null) return name; alias std.string.find find; name = n.classinfo.name; name = name[find(name, ".")+1 .. $]; // Remove package name name = name[find(name, ".")+1 .. $]; // Remove module name char[] remove; switch (n.category) { alias NodeCategory NC; case NC.Declaration: remove = "Declaration"; break; case NC.Statement: if (n.kind == NodeKind.Statements) return name; remove = "Statement"; break; case NC.Expression: remove = "Expression"; break; case NC.Type: remove = "Type"; break; case NC.Other: return name; default: } // Remove common suffix. auto idx = find(name, remove); if (idx != -1) name = name[0 .. idx]; // Store the name. name_table[n.kind] = name; return name; } enum DocPart { Head, CompBegin, CompEnd, Error, SyntaxBegin, SyntaxEnd, SrcBegin, SrcEnd, Tail, // Tokens: Identifier, Comment, StringLiteral, CharLiteral, Operator, LorG, LessEqual, GreaterEqual, AndLogical, OrLogical, NotEqual, Not, Number, Bracket, SpecialToken, Shebang, Keyword, HLineBegin, HLineEnd, Filespec, } auto html_tags = [ // Head `<html>`\n `<head>`\n `<meta http-equiv="Content-Type" content="text/html; charset=utf-8">`\n `<link href="dil_html.css" rel="stylesheet" type="text/css">`\n `</head>`\n `<body>`[], // CompBegin `<div class="compilerinfo">`, // CompEnd `</div>`, // Error `<p class="error %s">%s(%d)%s: %s</p>`, // SyntaxBegin `<span class="%s %s">`, // SyntaxEnd `</span>`, // SrcBegin `<pre class="sourcecode">`, // SrcEnd `</pre>`, // Tail `</html>`, // Identifier `<span class="i">%s</span>`, // Comment `<span class="c%s">%s</span>`, // StringLiteral `<span class="sl">%s</span>`, // CharLiteral `<span class="cl">%s</span>`, // Operator `<span class="op">%s</span>`, // LorG `<span class="oplg"><></span>`, // LessEqual `<span class="ople"><=</span>`, // GreaterEqual `<span class="opge">>=</span>`, // AndLogical `<span class="opaa">&&</span>`, // OrLogical `<span class="opoo">||</span>`, // NotEqual `<span class="opne">!=</span>`, // Not `<span class="opn">!</span>`, // Number `<span class="n">%s</span>`, // Bracket `<span class="br">%s</span>`, // SpecialToken `<span class="st">%s</span>`, // Shebang `<span class="shebang">%s</span>`, // Keyword `<span class="k">%s</span>`, // HLineBegin `<span class="hl">`, // HLineEnd "</span>", // Filespec `<span class="fs">%s</span>`, ]; auto xml_tags = [ // Head `<?xml version="1.0"?>`\n `<?xml-stylesheet href="dil_xml.css" type="text/css"?>`\n `<root>`[], // CompBegin `<compilerinfo>`, // CompEnd `</compilerinfo>`, // Error `<error t="%s">%s(%d)%s: %s</error>`, // SyntaxBegin `<%s t="%s">`, // SyntaxEnd `</%s>`, // SrcBegin `<sourcecode>`, // SrcEnd `</sourcecode>`, // Tail `</root>`, // Identifier "<i>%s</i>", // Comment `<c t="%s">%s</c>`, // StringLiteral "<sl>%s</sl>", // CharLiteral "<cl>%s</cl>", // Operator "<op>%s</op>", // LorG `<op t="lg"><></op>`, // LessEqual `<op t="le"><=</op>`, // GreaterEqual `<op t="ge">>=</op>`, // AndLogical `<op t="aa">&&</op>`, // OrLogical `<op t="oo">||</op>`, // NotEqual `<op t="ne">!=</op>`, // Not `<op t="n">!</op>`, // Number "<n>%s</n>", // Bracket "<br>%s</br>", // SpecialToken "<st>%s</st>", // Shebang "<shebang>%s</shebang>", // Keyword "<k>%s</k>", // HLineBegin "<hl>", // HLineEnd "</hl>", // Filespec "<fs>%s</fs>", ]; static assert(html_tags.length == DocPart.max+1); static assert(xml_tags.length == DocPart.max+1); void syntaxToDoc(string fileName, DocOption options) { auto tags = options & DocOption.HTML ? html_tags : xml_tags; auto sourceText = loadFile(fileName); auto parser = new Parser(sourceText, fileName); auto root = parser.start(); auto lx = parser.lx; auto token = lx.head; char* end = lx.text.ptr; writefln(tags[DocPart.Head]); // Output error messages. if (lx.errors.length || parser.errors.length) { writefln(tags[DocPart.CompBegin]); foreach (error; lx.errors) { writefln(tags[DocPart.Error], "L", lx.fileName, error.loc, "L", xml_escape(error.getMsg)); } foreach (error; parser.errors) { writefln(tags[DocPart.Error], "P", lx.fileName, error.loc, "P", xml_escape(error.getMsg)); } writefln(tags[DocPart.CompEnd]); } writef(tags[DocPart.SrcBegin]); Node[][Token*] beginNodes, endNodes; void populateAAs(Node[] nodes) { foreach (node; nodes) { assert(node !is null); auto begin = node.begin; if (begin) { auto end = node.end; assert(end); beginNodes[begin] ~= node; endNodes[end] ~= node; } if (node.children.length) populateAAs(node.children); } } populateAAs(root.children); char[] getTag(NodeCategory nc) { char[] tag; switch (nc) { alias NodeCategory NC; case NC.Declaration: tag = "d"; break; case NC.Statement: tag = "s"; break; case NC.Expression: tag = "e"; break; case NC.Type: tag = "t"; break; case NC.Other: tag = "o"; break; default: } return tag; } // Traverse linked list and print tokens. while (token.type != TOK.EOF) { token = token.next; // Print whitespace between previous and current token. if (end != token.start) writef("%s", end[0 .. token.start - end]); Node[]* nodes = token in beginNodes; if (nodes) { foreach (node; *nodes) writef(tags[DocPart.SyntaxBegin], getTag(node.category), getShortClassName(node)); } printToken(token, tags); nodes = token in endNodes; if (nodes) { foreach_reverse (node; *nodes) if (options & DocOption.HTML) writef(tags[DocPart.SyntaxEnd]); else writef(tags[DocPart.SyntaxEnd], getTag(node.category)); } end = token.end; } writef(tags[DocPart.SrcEnd], tags[DocPart.Tail]); } void tokensToDoc(string fileName, DocOption options) { auto tags = options & DocOption.HTML ? html_tags : xml_tags; auto sourceText = loadFile(fileName); auto lx = new Lexer(sourceText, fileName); auto token = lx.getTokens(); char* end = lx.text.ptr; writefln(tags[DocPart.Head]); if (lx.errors.length) { writefln(tags[DocPart.CompBegin]); foreach (error; lx.errors) { writefln(tags[DocPart.Error], "L", lx.fileName, error.loc, "L", xml_escape(error.getMsg)); } writefln(tags[DocPart.CompEnd]); } writef(tags[DocPart.SrcBegin]); // Traverse linked list and print tokens. while (token.type != TOK.EOF) { token = token.next; // Print whitespace between previous and current token. if (end != token.start) writef("%s", end[0 .. token.start - end]); printToken(token, tags); end = token.end; } writef(\n, tags[DocPart.SrcEnd], \n, tags[DocPart.Tail]); } void printToken(Token* token, string[] tags) { alias DocPart DP; string srcText = xml_escape(token.srcText); switch(token.type) { case TOK.Identifier: writef(tags[DP.Identifier], srcText); break; case TOK.Comment: string t; switch (token.start[1]) { case '/': t = "l"; break; case '*': t = "b"; break; case '+': t = "n"; break; default: assert(0); } writef(tags[DP.Comment], t, srcText); break; case TOK.String: writef(tags[DP.StringLiteral], srcText); break; case TOK.CharLiteral, TOK.WCharLiteral, TOK.DCharLiteral: writef(tags[DP.CharLiteral], srcText); break; case TOK.Assign, TOK.Equal, TOK.Less, TOK.Greater, TOK.LShiftAssign, TOK.LShift, TOK.RShiftAssign, TOK.RShift, TOK.URShiftAssign, TOK.URShift, TOK.OrAssign, TOK.OrBinary, TOK.AndAssign, TOK.AndBinary, TOK.PlusAssign, TOK.PlusPlus, TOK.Plus, TOK.MinusAssign, TOK.MinusMinus, TOK.Minus, TOK.DivAssign, TOK.Div, TOK.MulAssign, TOK.Mul, TOK.ModAssign, TOK.Mod, TOK.XorAssign, TOK.Xor, TOK.CatAssign, TOK.Tilde, TOK.Unordered, TOK.UorE, TOK.UorG, TOK.UorGorE, TOK.UorL, TOK.UorLorE, TOK.LorEorG: writef(tags[DP.Operator], srcText); break; case TOK.LorG: writef(tags[DP.LorG]); break; case TOK.LessEqual: writef(tags[DP.LessEqual]); break; case TOK.GreaterEqual: writef(tags[DP.GreaterEqual]); break; case TOK.AndLogical: writef(tags[DP.AndLogical]); break; case TOK.OrLogical: writef(tags[DP.OrLogical]); break; case TOK.NotEqual: writef(tags[DP.NotEqual]); break; case TOK.Not: // Check if this is part of a template instantiation. // TODO: comments aren't skipped. if (token.prev.type == TOK.Identifier && token.next.type == TOK.LParen) goto default; writef(tags[DP.Not]); break; case TOK.Int32, TOK.Int64, TOK.Uint32, TOK.Uint64, TOK.Float32, TOK.Float64, TOK.Float80, TOK.Imaginary32, TOK.Imaginary64, TOK.Imaginary80: writef(tags[DP.Number], srcText); break; case TOK.LParen, TOK.RParen, TOK.LBracket, TOK.RBracket, TOK.LBrace, TOK.RBrace: writef(tags[DP.Bracket], srcText); break; case TOK.Shebang: writef(tags[DP.Shebang], srcText); break; case TOK.HashLine: void printWS(char* start, char* end) { if (start != end) writef(start[0 .. end - start]); } writef(tags[DP.HLineBegin]); auto num = token.line_num; if (num is null) { writef(token.srcText); writef(tags[DP.HLineEnd]); break; } // Print whitespace between #line and number auto ptr = token.start; printWS(ptr, num.start); // prints "#line" as well printToken(num, tags); if (token.line_filespec) { auto filespec = token.line_filespec; // Print whitespace between number and filespec printWS(num.end, filespec.start); writef(tags[DP.Filespec], xml_escape(filespec.srcText)); ptr = filespec.end; } else ptr = num.end; // Print remaining whitespace printWS(ptr, token.end); writef(tags[DP.HLineEnd]); break; default: if (token.isKeyword()) writef(tags[DP.Keyword], srcText); else if (token.isSpecialToken) writef(tags[DP.SpecialToken], srcText); else writef("%s", srcText); } }