Mercurial > projects > dil

comparison src/cmd/Highlight.d @ 808:28e1ff1dcfcf

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Renamed generate command to highlight and refactored it.
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Sun, 09 Mar 2008 16:10:25 +0100
parents src/cmd/Generate.d@bcb74c9b895c
children 80eb3251e010
comparison
equal deleted inserted replaced
807:a2880c95eda3 808:28e1ff1dcfcf
1 /++
2 Author: Aziz Köksal
3 License: GPL3
4 +/
5 module cmd.Highlight;
6
7 import dil.ast.DefaultVisitor;
8 import dil.ast.Node,
9 dil.ast.Declaration,
10 dil.ast.Statement,
11 dil.ast.Expression,
12 dil.ast.Types;
13 import dil.lexer.Lexer;
14 import dil.parser.Parser;
15 import dil.semantic.Module;
16 import dil.SourceText;
17 import dil.Information;
18 import SettingsLoader;
19 import Settings;
20 import common;
21
22 import tango.io.GrowBuffer;
23 import tango.io.Print;
24
25 /// The highlight command.
26 struct HighlightCommand
27 {
28 /// Options for the command.
29 enum Option
30 {
31 None = 0,
32 Tokens = 1,
33 Syntax = 1<<1,
34 HTML = 1<<2,
35 XML = 1<<3,
36 PrintLines = 1<<4
37 }
38 alias Option Options;
39
40 Options options; /// Command options.
41 string filePath; /// File path to the module to be highlighted.
42 InfoManager infoMan;
43
44 /// Adds o to the options.
45 void add(Option o)
46 {
47 options |= o;
48 }
49
50 /// Executes the command.
51 void run()
52 {
53 add(HighlightCommand.Option.Tokens);
54 if (!(options & (Option.XML | Option.HTML)))
55 add(Option.XML); // Default to XML.
56
57 auto mapFilePath = options & Option.HTML ? GlobalSettings.htmlMapFile
58 : GlobalSettings.xmlMapFile;
59 auto map = TagMapLoader(infoMan).load(mapFilePath);
60 auto tags = new TagMap(map);
61
62 if (infoMan.hasInfo)
63 return;
64
65 if (options & Option.Syntax)
66 highlightSyntax(filePath, tags, Stdout, options);
67 else
68 highlightTokens(filePath, tags, Stdout, options);
69 }
70 }
71
72 /// Escapes the characters '<', '>' and '&' with named character entities.
73 char[] xml_escape(char[] text)
74 {
75 char[] result;
76 foreach(c; text)
77 switch(c)
78 {
79 case '<': result ~= "&lt;"; break;
80 case '>': result ~= "&gt;"; break;
81 case '&': result ~= "&amp;"; break;
82 default: result ~= c;
83 }
84 if (result.length != text.length)
85 return result;
86 // Nothing escaped. Return original text.
87 delete result;
88 return text;
89 }
90
91 /// Maps tokens to (format) strings.
92 class TagMap
93 {
94 string[string] table;
95 string[TOK.MAX] tokenTable;
96
97 this(string[string] table)
98 {
99 this.table = table;
100 Identifier = this["Identifier", "{0}"];
101 String = this["String", "{0}"];
102 Char = this["Char", "{0}"];
103 Number = this["Number", "{0}"];
104 Keyword = this["Keyword", "{0}"];
105 LineC = this["LineC", "{0}"];
106 BlockC = this["BlockC", "{0}"];
107 NestedC = this["NestedC", "{0}"];
108 Shebang = this["Shebang", "{0}"];
109 HLine = this["HLine", "{0}"];
110 Filespec = this["Filespec", "{0}"];
111 Illegal = this["Illegal", "{0}"];
112 Newline = this["Newline", "{0}"];
113 SpecialToken = this["SpecialToken", "{0}"];
114 Declaration = this["Declaration", "d"];
115 Statement = this["Statement", "s"];
116 Expression = this["Expression", "e"];
117 Type = this["Type", "t"];
118 Other = this["Other", "o"];
119 EOF = this["EOF", ""];
120
121 foreach (i, tokStr; tokToString)
122 if (auto pStr = tokStr in this.table)
123 tokenTable[i] = *pStr;
124 }
125
126 /// Returns the value for str, or 'fallback' if str is not in the table.
127 string opIndex(string str, string fallback = "")
128 {
129 auto p = str in table;
130 if (p)
131 return *p;
132 return fallback;
133 }
134
135 /// Returns the value for tok in O(1) time.
136 string opIndex(TOK tok)
137 {
138 return tokenTable[tok];
139 }
140
141 /// Shortcuts for quick access.
142 string Identifier, String, Char, Number, Keyword, LineC, BlockC,
143 NestedC, Shebang, HLine, Filespec, Illegal, Newline, SpecialToken,
144 Declaration, Statement, Expression, Type, Other, EOF;
145
146 /// Returns the tag for the category 'nc'.
147 string getTag(NodeCategory nc)
148 {
149 string tag;
150 switch (nc)
151 { alias NodeCategory NC;
152 case NC.Declaration: tag = Declaration; break;
153 case NC.Statement: tag = Statement; break;
154 case NC.Expression: tag = Expression; break;
155 case NC.Type: tag = Type; break;
156 case NC.Other: tag = Other; break;
157 default: assert(0);
158 }
159 return tag;
160 }
161 }
162
163 /// Find the last occurrence of object in subject.
164 /// Returns: the index if found, or -1 if not.
165 int rfind(char[] subject, char object)
166 {
167 foreach_reverse(i, c; subject)
168 if (c == object)
169 return i;
170 return -1;
171 }
172
173 /// Returns the short class name of a class descending from Node.$(BR)
174 /// E.g.: dil.ast.Declarations.ClassDeclaration -> Class
175 char[] getShortClassName(Node node)
176 {
177 static char[][] name_table;
178 if (name_table is null)
179 name_table = new char[][NodeKind.max+1]; // Create a new table.
180 // Look up in table.
181 char[] name = name_table[node.kind];
182 if (name !is null)
183 return name; // Return cached name.
184
185 name = node.classinfo.name; // Get the fully qualified name of the class.
186 name = name[rfind(name, '.')+1 .. $]; // Remove package and module name.
187
188 uint suffixLength;
189 switch (node.category)
190 {
191 alias NodeCategory NC;
192 case NC.Declaration:
193 suffixLength = "Declaration".length;
194 break;
195 case NC.Statement:
196 suffixLength = "Statement".length;
197 break;
198 case NC.Expression:
199 suffixLength = "Expression".length;
200 break;
201 case NC.Type:
202 suffixLength = "Type".length;
203 break;
204 case NC.Other:
205 break;
206 default:
207 assert(0);
208 }
209 // Remove common suffix.
210 name = name[0 .. $ - suffixLength];
211 // Store the name in the table.
212 name_table[node.kind] = name;
213 return name;
214 }
215
216 /// Extended token structure.
217 struct TokenEx
218 {
219 Token* token; /// The lexer token.
220 Node[] beginNodes; /// beginNodes[n].begin == token
221 Node[] endNodes; /// endNodes[n].end == token
222 }
223
224 /// Builds an array of TokenEx items.
225 class TokenExBuilder : DefaultVisitor
226 {
227 private TokenEx*[Token*] tokenTable;
228
229 TokenEx[] build(Node root, Token* first)
230 {
231 auto token = first;
232
233 uint count; // Count tokens.
234 for (; token; token = token.next)
235 count++;
236 // Creat the exact number of TokenEx instances.
237 auto toks = new TokenEx[count];
238 token = first;
239 foreach (ref tokEx; toks)
240 {
241 tokEx.token = token;
242 if (!token.isWhitespace)
243 tokenTable[token] = &tokEx;
244 token = token.next;
245 }
246
247 super.visitN(root);
248 tokenTable = null;
249 return toks;
250 }
251
252 TokenEx* getTokenEx()(Token* t)
253 {
254 auto p = t in tokenTable;
255 assert(p, t.srcText~" is not in tokenTable");
256 return *p;
257 }
258
259 // Override dispatch function.
260 override Node dispatch(Node n)
261 {
262 auto begin = n.begin;
263 if (begin)
264 { assert(n.end);
265 auto txbegin = getTokenEx(begin);
266 auto txend = getTokenEx(n.end);
267 txbegin.beginNodes ~= n;
268 txend.endNodes ~= n;
269 }
270 return super.dispatch(n);
271 }
272 }
273
274 void printErrors(Lexer lx, TagMap tags, Print!(char) print)
275 {
276 foreach (e; lx.errors)
277 print.format(tags["LexerError"], e.filePath, e.loc, e.col, xml_escape(e.getMsg));
278 }
279
280 void printErrors(Parser parser, TagMap tags, Print!(char) print)
281 {
282 foreach (e; parser.errors)
283 print.format(tags["ParserError"], e.filePath, e.loc, e.col, xml_escape(e.getMsg));
284 }
285
286 void printLines(uint lines, TagMap tags, Print!(char) print)
287 {
288 auto lineNumberFormat = tags["LineNumber"];
289 for (auto lineNum = 1; lineNum <= lines; lineNum++)
290 print.format(lineNumberFormat, lineNum);
291 }
292
293 /// Highlights the syntax in a source file.
294 void highlightSyntax(string filePath, TagMap tags,
295 Print!(char) print,
296 HighlightCommand.Options options)
297 {
298 auto parser = new Parser(new SourceText(filePath, true));
299 auto root = parser.start();
300 auto lx = parser.lexer;
301
302 auto builder = new TokenExBuilder();
303 auto tokenExList = builder.build(root, lx.firstToken());
304
305 print(tags["DocHead"]);
306 if (lx.errors.length || parser.errors.length)
307 { // Output error messages.
308 print(tags["CompBegin"]);
309 printErrors(lx, tags, print);
310 printErrors(parser, tags, print);
311 print(tags["CompEnd"]);
312 }
313
314 if (options & HighlightCommand.Option.PrintLines)
315 {
316 print(tags["LineNumberBegin"]);
317 printLines(lx.lineNum, tags, print);
318 print(tags["LineNumberEnd"]);
319 }
320
321 print(tags["SourceBegin"]);
322
323 auto tagNodeBegin = tags["NodeBegin"];
324 auto tagNodeEnd = tags["NodeEnd"];
325
326 // Iterate over list of tokens.
327 foreach (ref tokenEx; tokenExList)
328 {
329 auto token = tokenEx.token;
330
331 token.ws && print(token.wsChars); // Print preceding whitespace.
332 if (token.isWhitespace) {
333 printToken(token, tags, print);
334 continue;
335 }
336 // <node>
337 foreach (node; tokenEx.beginNodes)
338 print.format(tagNodeBegin, tags.getTag(node.category), getShortClassName(node));
339 // Token text.
340 printToken(token, tags, print);
341 // </node>
342 if (options & HighlightCommand.Option.HTML)
343 foreach_reverse (node; tokenEx.endNodes)
344 print(tagNodeEnd);
345 else
346 foreach_reverse (node; tokenEx.endNodes)
347 print.format(tagNodeEnd, tags.getTag(node.category));
348 }
349 print(tags["SourceEnd"]);
350 print(tags["DocEnd"]);
351 }
352
353 /// Highlights all tokens of a source file.
354 void highlightTokens(string filePath, TagMap tags,
355 Print!(char) print,
356 HighlightCommand.Options options)
357 {
358 auto lx = new Lexer(new SourceText(filePath, true));
359 lx.scanAll();
360
361 print(tags["DocHead"]);
362 if (lx.errors.length)
363 {
364 print(tags["CompBegin"]);
365 printErrors(lx, tags, print);
366 print(tags["CompEnd"]);
367 }
368
369 if (options & HighlightCommand.Option.PrintLines)
370 {
371 print(tags["LineNumberBegin"]);
372 printLines(lx.lineNum, tags, print);
373 print(tags["LineNumberEnd"]);
374 }
375
376 print(tags["SourceBegin"]);
377 // Traverse linked list and print tokens.
378 for (auto token = lx.firstToken(); token; token = token.next) {
379 token.ws && print(token.wsChars); // Print preceding whitespace.
380 printToken(token, tags, print);
381 }
382 print(tags["SourceEnd"]);
383 print(tags["DocEnd"]);
384 }
385
386 /// A token highlighter designed for DDoc.
387 class TokenHighlighter
388 {
389 TagMap tags;
390 this(InfoManager infoMan, bool useHTML = true)
391 {
392 string filePath = GlobalSettings.htmlMapFile;
393 if (!useHTML)
394 filePath = GlobalSettings.xmlMapFile;
395 auto map = TagMapLoader(infoMan).load(filePath);
396 tags = new TagMap(map);
397 }
398
399 /// Highlights tokens in a DDoc code section.
400 /// Returns: a string with the highlighted tokens (in HTML tags.)
401 string highlight(string text, string filePath)
402 {
403 auto buffer = new GrowBuffer(text.length);
404 auto print = new Print!(char)(Format, buffer);
405
406 auto lx = new Lexer(new SourceText(filePath, text));
407 lx.scanAll();
408
409 // Traverse linked list and print tokens.
410 print("$(D_CODE\n");
411 if (lx.errors.length)
412 { // Output error messages.
413 print(tags["CompBegin"]);
414 printErrors(lx, tags, print);
415 print(tags["CompEnd"]);
416 }
417 // Traverse linked list and print tokens.
418 for (auto token = lx.firstToken(); token; token = token.next) {
419 token.ws && print(token.wsChars); // Print preceding whitespace.
420 printToken(token, tags, print);
421 }
422 print("\n)");
423 return cast(char[])buffer.slice();
424 }
425 }
426
427 /// Prints a token to the stream print.
428 void printToken(Token* token, TagMap tags, Print!(char) print)
429 {
430 switch(token.kind)
431 {
432 case TOK.Identifier:
433 print.format(tags.Identifier, token.srcText);
434 break;
435 case TOK.Comment:
436 string formatStr;
437 switch (token.start[1])
438 {
439 case '/': formatStr = tags.LineC; break;
440 case '*': formatStr = tags.BlockC; break;
441 case '+': formatStr = tags.NestedC; break;
442 default: assert(0);
443 }
444 print.format(formatStr, xml_escape(token.srcText));
445 break;
446 case TOK.String:
447 print.format(tags.String, xml_escape(token.srcText));
448 break;
449 case TOK.CharLiteral:
450 print.format(tags.Char, xml_escape(token.srcText));
451 break;
452 case TOK.Int32, TOK.Int64, TOK.Uint32, TOK.Uint64,
453 TOK.Float32, TOK.Float64, TOK.Float80,
454 TOK.Imaginary32, TOK.Imaginary64, TOK.Imaginary80:
455 print.format(tags.Number, token.srcText);
456 break;
457 case TOK.Shebang:
458 print.format(tags.Shebang, xml_escape(token.srcText));
459 break;
460 case TOK.HashLine:
461 auto formatStr = tags.HLine;
462 // The text to be inserted into formatStr.
463 auto buffer = new GrowBuffer;
464 auto print2 = new Print!(char)(Format, buffer);
465
466 void printWS(char* start, char* end)
467 {
468 start != end && print2(start[0 .. end - start]);
469 }
470
471 auto num = token.tokLineNum;
472 if (num is null)
473 { // Malformed #line
474 print.format(formatStr, token.srcText);
475 break;
476 }
477
478 // Print whitespace between #line and number.
479 printWS(token.start, num.start); // Prints "#line" as well.
480 printToken(num, tags, print2); // Print the number.
481
482 if (auto filespec = token.tokLineFilespec)
483 { // Print whitespace between number and filespec.
484 printWS(num.end, filespec.start);
485 print2.format(tags.Filespec, xml_escape(filespec.srcText));
486 }
487 // Finally print the whole token.
488 print.format(formatStr, cast(char[])buffer.slice());
489 break;
490 case TOK.Illegal:
491 print.format(tags.Illegal, token.srcText());
492 break;
493 case TOK.Newline:
494 print.format(tags.Newline, token.srcText());
495 break;
496 case TOK.EOF:
497 print(tags.EOF);
498 break;
499 default:
500 if (token.isKeyword())
501 print.format(tags.Keyword, token.srcText);
502 else if (token.isSpecialToken)
503 print.format(tags.SpecialToken, token.srcText);
504 else
505 print(tags[token.kind]);
506 }
507 }