Mercurial > projects > dil
annotate trunk/src/cmd/Generate.d @ 388:ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Fix: Token.line_num can be null which could cause a segfault in printToken().
Renamed MID.ExpectedNumberAfterSTLine to ExpectedIntegerAfterSTLine.
Fix: Added a check for TOK.Int32 and TOK.Uint32 after a number was scanned
in scanSpecialTokenSequence().
| author | Aziz K?ksal <aziz.koeksal@gmail.com> |
|---|---|
| date | Wed, 12 Sep 2007 13:43:45 +0200 |
| parents | a92f7826a4fa |
| children | c4bfceab7246 |
| rev | line source |
|---|---|
| 363 | 1 /++ |
| 2 Author: Aziz Köksal | |
| 3 License: GPL3 | |
| 4 +/ | |
| 5 module cmd.Generate; | |
| 6 import dil.SyntaxTree; | |
| 7 import dil.Token; | |
| 8 import dil.Parser, dil.Lexer; | |
| 9 import dil.File; | |
| 10 import std.stdio; | |
| 11 | |
| 12 enum DocOption | |
| 13 { | |
| 14 Tokens, | |
| 15 Syntax = 1<<1, | |
| 16 HTML = 1<<2, | |
| 17 XML = 1<<3 | |
| 18 } | |
| 19 | |
| 364 | 20 void execute(string fileName, DocOption options) |
| 21 { | |
| 22 if (options & DocOption.Syntax) | |
| 23 syntaxToDoc(fileName, options); | |
| 24 else | |
| 25 tokensToDoc(fileName, options); | |
| 26 } | |
| 27 | |
| 363 | 28 char[] xml_escape(char[] text) |
| 29 { | |
| 30 char[] result; | |
| 31 foreach(c; text) | |
| 32 switch(c) | |
| 33 { | |
| 34 case '<': result ~= "<"; break; | |
| 35 case '>': result ~= ">"; break; | |
| 36 case '&': result ~= "&"; break; | |
| 37 default: result ~= c; | |
| 38 } | |
| 39 return result; | |
| 40 } | |
| 41 | |
| 42 char[] getShortClassName(Node n) | |
| 43 { | |
| 44 static char[][] name_table; | |
| 45 if (name_table is null) | |
| 46 name_table = new char[][NodeKind.max+1]; | |
| 47 char[] name = name_table[n.kind]; | |
| 48 if (name !is null) | |
| 49 return name; | |
| 50 | |
| 51 alias std.string.find find; | |
| 52 name = n.classinfo.name; | |
| 53 name = name[find(name, ".")+1 .. $]; // Remove package name | |
| 54 name = name[find(name, ".")+1 .. $]; // Remove module name | |
| 55 char[] remove; | |
| 56 switch (n.category) | |
| 57 { | |
| 58 alias NodeCategory NC; | |
| 59 case NC.Declaration: remove = "Declaration"; break; | |
| 60 case NC.Statement: | |
| 61 if (n.kind == NodeKind.Statements) | |
| 62 return name; | |
| 63 remove = "Statement"; | |
| 64 break; | |
| 65 case NC.Expression: remove = "Expression"; break; | |
| 66 case NC.Type: remove = "Type"; break; | |
| 67 case NC.Other: return name; | |
| 68 default: | |
| 69 } | |
| 70 // Remove common suffix. | |
| 71 auto idx = find(name, remove); | |
| 72 if (idx != -1) | |
| 73 name = name[0 .. idx]; | |
| 74 // Store the name. | |
| 75 name_table[n.kind] = name; | |
| 76 return name; | |
| 77 } | |
| 78 | |
| 79 enum DocPart | |
| 80 { | |
| 81 Head, | |
| 82 CompBegin, | |
| 83 CompEnd, | |
| 84 Error, | |
| 85 SyntaxBegin, | |
| 86 SyntaxEnd, | |
| 87 SrcBegin, | |
| 88 SrcEnd, | |
| 89 Tail, | |
| 90 // Tokens: | |
| 91 Identifier, | |
| 92 Comment, | |
| 93 StringLiteral, | |
| 94 CharLiteral, | |
| 95 Operator, | |
| 96 LorG, | |
| 97 LessEqual, | |
| 98 GreaterEqual, | |
| 99 AndLogical, | |
| 100 OrLogical, | |
| 101 NotEqual, | |
| 102 Not, | |
| 103 Number, | |
| 104 Bracket, | |
| 105 SpecialToken, | |
| 106 Shebang, | |
| 107 Keyword, | |
| 108 HLineBegin, | |
| 109 HLineEnd, | |
| 110 Filespec, | |
| 111 } | |
| 112 | |
| 113 auto html_tags = [ | |
| 114 // Head | |
| 115 `<html>`\n | |
| 116 `<head>`\n | |
| 117 `<meta http-equiv="Content-Type" content="text/html; charset=utf-8">`\n | |
| 118 `<link href="dil_html.css" rel="stylesheet" type="text/css">`\n | |
| 119 `</head>`\n | |
| 120 `<body>`[], | |
| 121 // CompBegin | |
| 122 `<div class="compilerinfo">`, | |
| 123 // CompEnd | |
| 124 `</div>`, | |
| 125 // Error | |
| 126 `<p class="error %s">%s(%d)%s: %s</p>`, | |
| 127 // SyntaxBegin | |
| 128 `<span class="%s %s">`, | |
| 129 // SyntaxEnd | |
| 130 `</span>`, | |
| 131 // SrcBegin | |
| 132 `<pre class="sourcecode">`, | |
| 133 // SrcEnd | |
| 134 `</pre>`, | |
| 135 // Tail | |
| 136 `</html>`, | |
| 137 // Identifier | |
| 138 `<span class="i">%s</span>`, | |
| 139 // Comment | |
| 140 `<span class="c%s">%s</span>`, | |
| 141 // StringLiteral | |
| 142 `<span class="sl">%s</span>`, | |
| 143 // CharLiteral | |
| 144 `<span class="cl">%s</span>`, | |
| 145 // Operator | |
| 146 `<span class="op">%s</span>`, | |
| 147 // LorG | |
| 148 `<span class="oplg"><></span>`, | |
| 149 // LessEqual | |
| 150 `<span class="ople"><=</span>`, | |
| 151 // GreaterEqual | |
| 152 `<span class="opge">>=</span>`, | |
| 153 // AndLogical | |
| 154 `<span class="opaa">&&</span>`, | |
| 155 // OrLogical | |
| 156 `<span class="opoo">||</span>`, | |
| 157 // NotEqual | |
| 158 `<span class="opne">!=</span>`, | |
| 159 // Not | |
| 160 `<span class="opn">!</span>`, | |
| 161 // Number | |
| 162 `<span class="n">%s</span>`, | |
| 163 // Bracket | |
| 164 `<span class="br">%s</span>`, | |
| 165 // SpecialToken | |
| 166 `<span class="st">%s</span>`, | |
| 167 // Shebang | |
| 168 `<span class="shebang">%s</span>`, | |
| 169 // Keyword | |
| 170 `<span class="k">%s</span>`, | |
| 171 // HLineBegin | |
|
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
172 `<span class="hl">`, |
| 363 | 173 // HLineEnd |
| 174 "</span>", | |
| 175 // Filespec | |
| 176 `<span class="fs">%s</span>`, | |
| 177 ]; | |
| 178 | |
| 179 auto xml_tags = [ | |
| 180 // Head | |
| 181 `<?xml version="1.0"?>`\n | |
| 182 `<?xml-stylesheet href="dil_xml.css" type="text/css"?>`\n | |
| 183 `<root>`[], | |
| 184 // CompBegin | |
| 185 `<compilerinfo>`, | |
| 186 // CompEnd | |
| 187 `</compilerinfo>`, | |
| 188 // Error | |
| 189 `<error t="%s">%s(%d)%s: %s</error>`, | |
| 190 // SyntaxBegin | |
| 191 `<%s t="%s">`, | |
| 192 // SyntaxEnd | |
| 193 `</%s>`, | |
| 194 // SrcBegin | |
| 195 `<sourcecode>`, | |
| 196 // SrcEnd | |
| 197 `</sourcecode>`, | |
| 198 // Tail | |
| 199 `</root>`, | |
| 200 // Identifier | |
| 201 "<i>%s</i>", | |
| 202 // Comment | |
| 203 `<c t="%s">%s</c>`, | |
| 204 // StringLiteral | |
| 205 "<sl>%s</sl>", | |
| 206 // CharLiteral | |
| 207 "<cl>%s</cl>", | |
| 208 // Operator | |
| 209 "<op>%s</op>", | |
| 210 // LorG | |
| 211 `<op t="lg"><></op>`, | |
| 212 // LessEqual | |
| 213 `<op t="le"><=</op>`, | |
| 214 // GreaterEqual | |
| 215 `<op t="ge">>=</op>`, | |
| 216 // AndLogical | |
| 217 `<op t="aa">&&</op>`, | |
| 218 // OrLogical | |
| 219 `<op t="oo">||</op>`, | |
| 220 // NotEqual | |
| 221 `<op t="ne">!=</op>`, | |
| 222 // Not | |
| 223 `<op t="n">!</op>`, | |
| 224 // Number | |
| 225 "<n>%s</n>", | |
| 226 // Bracket | |
| 227 "<br>%s</br>", | |
| 228 // SpecialToken | |
| 229 "<st>%s</st>", | |
| 230 // Shebang | |
| 231 "<shebang>%s</shebang>", | |
| 232 // Keyword | |
| 233 "<k>%s</k>", | |
| 234 // HLineBegin | |
|
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
235 "<hl>", |
| 363 | 236 // HLineEnd |
| 237 "</hl>", | |
| 238 // Filespec | |
| 239 "<fs>%s</fs>", | |
| 240 ]; | |
| 241 | |
| 242 static assert(html_tags.length == DocPart.max+1); | |
| 243 static assert(xml_tags.length == DocPart.max+1); | |
| 244 | |
| 245 void syntaxToDoc(string fileName, DocOption options) | |
| 246 { | |
| 247 auto tags = options & DocOption.HTML ? html_tags : xml_tags; | |
| 248 auto sourceText = loadFile(fileName); | |
| 249 auto parser = new Parser(sourceText, fileName); | |
| 368 | 250 auto root = parser.start(); |
| 363 | 251 auto lx = parser.lx; |
| 252 | |
| 253 auto token = lx.head; | |
| 254 char* end = lx.text.ptr; | |
| 255 | |
| 256 writefln(tags[DocPart.Head]); | |
| 257 // Output error messages. | |
| 258 if (lx.errors.length || parser.errors.length) | |
| 259 { | |
| 260 writefln(tags[DocPart.CompBegin]); | |
| 261 foreach (error; lx.errors) | |
| 262 { | |
| 263 writefln(tags[DocPart.Error], "L", lx.fileName, error.loc, "L", xml_escape(error.getMsg)); | |
| 264 } | |
| 265 foreach (error; parser.errors) | |
| 266 { | |
| 267 writefln(tags[DocPart.Error], "P", lx.fileName, error.loc, "P", xml_escape(error.getMsg)); | |
| 268 } | |
| 269 writefln(tags[DocPart.CompEnd]); | |
| 270 } | |
| 271 writef(tags[DocPart.SrcBegin]); | |
| 272 | |
| 273 Node[][Token*] beginNodes, endNodes; | |
| 274 | |
| 275 void populateAAs(Node[] nodes) | |
| 276 { | |
| 277 foreach (node; nodes) | |
| 278 { | |
|
379
a92f7826a4fa
- Fix in parseTemplateArguments_(): calling nested parseType_() instead of parseType() in try_().
aziz
parents:
368
diff
changeset
|
279 assert(node !is null); |
| 363 | 280 auto begin = node.begin; |
| 281 if (begin) | |
| 282 { | |
| 283 auto end = node.end; | |
| 284 assert(end); | |
| 285 beginNodes[begin] ~= node; | |
| 286 endNodes[end] ~= node; | |
| 287 } | |
| 288 if (node.children.length) | |
| 289 populateAAs(node.children); | |
| 290 } | |
| 291 } | |
| 292 populateAAs(root.children); | |
| 293 | |
| 294 char[] getTag(NodeCategory nc) | |
| 295 { | |
| 296 char[] tag; | |
| 297 switch (nc) | |
| 298 { | |
| 299 alias NodeCategory NC; | |
| 300 case NC.Declaration: tag = "d"; break; | |
| 301 case NC.Statement: tag = "s"; break; | |
| 302 case NC.Expression: tag = "e"; break; | |
| 303 case NC.Type: tag = "t"; break; | |
| 304 case NC.Other: tag = "o"; break; | |
| 305 default: | |
| 306 } | |
| 307 return tag; | |
| 308 } | |
| 309 | |
| 310 // Traverse linked list and print tokens. | |
| 311 while (token.type != TOK.EOF) | |
| 312 { | |
| 313 token = token.next; | |
| 314 | |
| 315 // Print whitespace between previous and current token. | |
| 316 if (end != token.start) | |
| 317 writef("%s", end[0 .. token.start - end]); | |
| 318 | |
| 319 Node[]* nodes = token in beginNodes; | |
| 320 | |
| 321 if (nodes) | |
| 322 { | |
| 323 foreach (node; *nodes) | |
| 324 writef(tags[DocPart.SyntaxBegin], getTag(node.category), getShortClassName(node)); | |
| 325 } | |
| 326 | |
| 327 printToken(token, tags); | |
| 328 | |
| 329 nodes = token in endNodes; | |
| 330 | |
| 331 if (nodes) | |
| 332 { | |
| 333 foreach_reverse (node; *nodes) | |
| 334 if (options & DocOption.HTML) | |
| 335 writef(tags[DocPart.SyntaxEnd]); | |
| 336 else | |
| 337 writef(tags[DocPart.SyntaxEnd], getTag(node.category)); | |
| 338 } | |
| 339 | |
| 340 end = token.end; | |
| 341 } | |
| 342 writef(tags[DocPart.SrcEnd], tags[DocPart.Tail]); | |
| 343 } | |
| 344 | |
| 345 void tokensToDoc(string fileName, DocOption options) | |
| 346 { | |
| 347 auto tags = options & DocOption.HTML ? html_tags : xml_tags; | |
| 348 auto sourceText = loadFile(fileName); | |
| 349 auto lx = new Lexer(sourceText, fileName); | |
| 350 | |
| 351 auto token = lx.getTokens(); | |
| 352 char* end = lx.text.ptr; | |
| 353 | |
| 354 writefln(tags[DocPart.Head]); | |
| 355 | |
| 356 if (lx.errors.length) | |
| 357 { | |
| 358 writefln(tags[DocPart.CompBegin]); | |
| 359 foreach (error; lx.errors) | |
| 360 { | |
| 361 writefln(tags[DocPart.Error], "L", lx.fileName, error.loc, "L", xml_escape(error.getMsg)); | |
| 362 } | |
| 363 writefln(tags[DocPart.CompEnd]); | |
| 364 } | |
| 365 writef(tags[DocPart.SrcBegin]); | |
| 366 | |
| 367 // Traverse linked list and print tokens. | |
| 368 while (token.type != TOK.EOF) | |
| 369 { | |
| 370 token = token.next; | |
| 371 | |
| 372 // Print whitespace between previous and current token. | |
| 373 if (end != token.start) | |
| 374 writef("%s", end[0 .. token.start - end]); | |
| 375 printToken(token, tags); | |
| 376 end = token.end; | |
| 377 } | |
| 378 writef(\n, tags[DocPart.SrcEnd], \n, tags[DocPart.Tail]); | |
| 379 } | |
| 380 | |
| 381 void printToken(Token* token, string[] tags) | |
| 382 { | |
| 383 alias DocPart DP; | |
| 384 string srcText = xml_escape(token.srcText); | |
| 385 | |
| 386 switch(token.type) | |
| 387 { | |
| 388 case TOK.Identifier: | |
| 389 writef(tags[DP.Identifier], srcText); | |
| 390 break; | |
| 391 case TOK.Comment: | |
| 392 string t; | |
| 393 switch (token.start[1]) | |
| 394 { | |
| 395 case '/': t = "l"; break; | |
| 396 case '*': t = "b"; break; | |
| 397 case '+': t = "n"; break; | |
| 398 default: | |
| 399 assert(0); | |
| 400 } | |
| 401 writef(tags[DP.Comment], t, srcText); | |
| 402 break; | |
| 403 case TOK.String: | |
| 404 writef(tags[DP.StringLiteral], srcText); | |
| 405 break; | |
| 406 case TOK.CharLiteral, TOK.WCharLiteral, TOK.DCharLiteral: | |
| 407 writef(tags[DP.CharLiteral], srcText); | |
| 408 break; | |
| 409 case TOK.Assign, TOK.Equal, | |
| 410 TOK.Less, TOK.Greater, | |
| 411 TOK.LShiftAssign, TOK.LShift, | |
| 412 TOK.RShiftAssign, TOK.RShift, | |
| 413 TOK.URShiftAssign, TOK.URShift, | |
| 414 TOK.OrAssign, TOK.OrBinary, | |
| 415 TOK.AndAssign, TOK.AndBinary, | |
| 416 TOK.PlusAssign, TOK.PlusPlus, TOK.Plus, | |
| 417 TOK.MinusAssign, TOK.MinusMinus, TOK.Minus, | |
| 418 TOK.DivAssign, TOK.Div, | |
| 419 TOK.MulAssign, TOK.Mul, | |
| 420 TOK.ModAssign, TOK.Mod, | |
| 421 TOK.XorAssign, TOK.Xor, | |
| 422 TOK.CatAssign, | |
| 423 TOK.Tilde, | |
| 424 TOK.Unordered, | |
| 425 TOK.UorE, | |
| 426 TOK.UorG, | |
| 427 TOK.UorGorE, | |
| 428 TOK.UorL, | |
| 429 TOK.UorLorE, | |
| 430 TOK.LorEorG: | |
| 431 writef(tags[DP.Operator], srcText); | |
| 432 break; | |
| 433 case TOK.LorG: | |
| 434 writef(tags[DP.LorG]); | |
| 435 break; | |
| 436 case TOK.LessEqual: | |
| 437 writef(tags[DP.LessEqual]); | |
| 438 break; | |
| 439 case TOK.GreaterEqual: | |
| 440 writef(tags[DP.GreaterEqual]); | |
| 441 break; | |
| 442 case TOK.AndLogical: | |
| 443 writef(tags[DP.AndLogical]); | |
| 444 break; | |
| 445 case TOK.OrLogical: | |
| 446 writef(tags[DP.OrLogical]); | |
| 447 break; | |
| 448 case TOK.NotEqual: | |
| 449 writef(tags[DP.NotEqual]); | |
| 450 break; | |
| 451 case TOK.Not: | |
| 452 // Check if this is part of a template instantiation. | |
| 453 // TODO: comments aren't skipped. | |
| 454 if (token.prev.type == TOK.Identifier && token.next.type == TOK.LParen) | |
| 455 goto default; | |
| 456 writef(tags[DP.Not]); | |
| 457 break; | |
| 458 case TOK.Int32, TOK.Int64, TOK.Uint32, TOK.Uint64, | |
| 459 TOK.Float32, TOK.Float64, TOK.Float80, | |
| 460 TOK.Imaginary32, TOK.Imaginary64, TOK.Imaginary80: | |
| 461 writef(tags[DP.Number], srcText); | |
| 462 break; | |
| 463 case TOK.LParen, TOK.RParen, TOK.LBracket, | |
| 464 TOK.RBracket, TOK.LBrace, TOK.RBrace: | |
| 465 writef(tags[DP.Bracket], srcText); | |
| 466 break; | |
| 467 case TOK.Shebang: | |
| 468 writef(tags[DP.Shebang], srcText); | |
| 469 break; | |
| 470 case TOK.HashLine: | |
| 471 void printWS(char* start, char* end) | |
| 472 { | |
| 473 if (start != end) | |
| 474 writef(start[0 .. end - start]); | |
| 475 } | |
| 476 writef(tags[DP.HLineBegin]); | |
| 477 auto num = token.line_num; | |
|
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
478 if (num is null) |
|
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
479 { |
|
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
480 writef(token.start[0 .. token.end - token.start]); |
|
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
481 writef(tags[DP.HLineEnd]); |
|
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
482 break; |
|
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
483 } |
| 363 | 484 // Print whitespace between #line and number |
|
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
485 auto ptr = token.start; |
|
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
486 printWS(ptr, num.start); // prints "#line" as well |
| 363 | 487 printToken(num, tags); |
| 488 if (token.line_filespec) | |
| 489 { | |
| 490 auto filespec = token.line_filespec; | |
| 491 // Print whitespace between number and filespec | |
| 492 printWS(num.end, filespec.start); | |
| 493 writef(tags[DP.Filespec], xml_escape(filespec.srcText)); | |
| 494 | |
| 495 ptr = filespec.end; | |
| 496 } | |
| 497 else | |
| 498 ptr = num.end; | |
| 499 // Print remaining whitespace | |
| 500 printWS(ptr, token.end); | |
| 501 writef(tags[DP.HLineEnd]); | |
| 502 break; | |
| 503 default: | |
| 504 if (token.isKeyword()) | |
| 505 writef(tags[DP.Keyword], srcText); | |
| 506 else if (token.isSpecialToken) | |
| 507 writef(tags[DP.SpecialToken], srcText); | |
| 508 else | |
| 509 writef("%s", srcText); | |
| 510 } | |
| 511 } |