Mercurial > projects > dil
annotate trunk/src/cmd/Generate.d @ 485:ea8c7459f1c4
Changed a lot of things in the Lexer.
Newlines are tokenized now, instead of being treated as whitespace.
Newline tokens store location info as well, which make quite a few functions
unnecessary. Added a static method getLocation() which returns a Location
instance for any given token. This will also be very useful for finding
the location of AST nodes (through Node.begin,) which is needed for reporting
parser and semantic errors and emitting documentation.
Removed rescanNewlines(), LocState, getState(), restoreState(),
evaluateHashLine() and updateErrorLoc().
Added isUnicodeNewlineChar(), isUnicodeNewline(), isNewline(), isNewlineEnd(),
isEndOfLine(), scanNewline(), getLocation() and error_().
Replaced some clunky expressions with isascii(), isNewlineEnd(), isEndOfLine(),
isUnicodeNewline(), isUnicodeNewlineChar().
Fix in scanNormalStringLiteral(): scanPostfix() must be before label Lreturn.
Fixed Lexer unittest.
Fix in parseDeclarationDefinitionsBlock(): 'token' should be 'begin'.
Added method isMultiline() to Token and added documentation comments.: File name too long
abort: file /home/aziz/dil/trunk/Changed a lot of things in the Lexer.
Newlines are tokenized now, instead of being treated as whitespace.
Newline tokens store location info as well, which make quite a few functions
unnecessary. Added a static method getLocation() which returns a Location
instance for any given token. This will also be very useful for finding
the location of AST nodes (through Node.begin,) which is needed for reporting
parser and semantic errors and emitting documentation.
Removed rescanNewlines(), LocState, getState(), restoreState(),
evaluateHashLine() and updateErrorLoc().
Added isUnicodeNewlineChar(), isUnicodeNewline(), isNewline(), isNewlineEnd(),
isEndOfLine(), scanNewline(), getLocation() and error_().
Replaced some clunky expressions with isascii(), isNewlineEnd(), isEndOfLine(),
isUnicodeNewline(), isUnicodeNewlineChar().
Fix in scanNormalStringLiteral(): scanPostfix() must be before label Lreturn.
Fixed Lexer unittest.
Fix in parseDeclarationDefinitionsBlock(): 'token' should be 'begin'.
Added method isMultiline() to Token and added documentation comments.
| author | Aziz K?ksal <aziz.koeksal@gmail.com> |
|---|---|
| date | Fri, 30 Nov 2007 20:17:29 +0100 |
| parents | ad7977fe315a |
| children | 996041463028 |
| rev | line source |
|---|---|
| 363 | 1 /++ |
| 2 Author: Aziz Köksal | |
| 3 License: GPL3 | |
| 4 +/ | |
| 5 module cmd.Generate; | |
| 6 import dil.SyntaxTree; | |
| 7 import dil.Token; | |
| 8 import dil.Parser, dil.Lexer; | |
| 9 import dil.File; | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
10 import tango.io.Print; |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
11 import common; |
| 363 | 12 |
| 13 enum DocOption | |
| 14 { | |
|
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
15 Empty, |
|
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
16 Tokens = 1, |
| 363 | 17 Syntax = 1<<1, |
|
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
18 HTML = 1<<2, |
|
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
19 XML = 1<<3 |
| 363 | 20 } |
| 21 | |
| 364 | 22 void execute(string fileName, DocOption options) |
| 23 { | |
|
392
bb935c6f9b7a
Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
24 assert(options != DocOption.Empty); |
| 364 | 25 if (options & DocOption.Syntax) |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
26 syntaxToDoc(fileName, Stdout, options); |
| 364 | 27 else |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
28 tokensToDoc(fileName, Stdout, options); |
| 364 | 29 } |
| 30 | |
| 363 | 31 char[] xml_escape(char[] text) |
| 32 { | |
| 33 char[] result; | |
| 34 foreach(c; text) | |
| 35 switch(c) | |
| 36 { | |
| 37 case '<': result ~= "<"; break; | |
| 38 case '>': result ~= ">"; break; | |
| 39 case '&': result ~= "&"; break; | |
| 40 default: result ~= c; | |
| 41 } | |
| 42 return result; | |
| 43 } | |
| 44 | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
45 |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
46 /// Find object in subject and return position. |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
47 /// Returns -1 if no match was found. |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
48 int find(char[] subject, char[] object) |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
49 { |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
50 if (object.length > subject.length) |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
51 return -1; |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
52 foreach (i, c; subject) |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
53 { |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
54 if (c == object[0]) |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
55 { |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
56 if (object.length > (subject.length - i)) |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
57 return -1; |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
58 if (object == subject[i..i+object.length]) |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
59 return i; |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
60 } |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
61 } |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
62 return -1; |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
63 } |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
64 |
| 363 | 65 char[] getShortClassName(Node n) |
| 66 { | |
| 67 static char[][] name_table; | |
| 68 if (name_table is null) | |
| 69 name_table = new char[][NodeKind.max+1]; | |
| 70 char[] name = name_table[n.kind]; | |
| 71 if (name !is null) | |
| 72 return name; | |
| 73 | |
| 74 name = n.classinfo.name; | |
| 75 name = name[find(name, ".")+1 .. $]; // Remove package name | |
| 76 name = name[find(name, ".")+1 .. $]; // Remove module name | |
| 77 char[] remove; | |
| 78 switch (n.category) | |
| 79 { | |
| 80 alias NodeCategory NC; | |
|
485
ea8c7459f1c4
Changed a lot of things in the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
81 case NC.Declaration: |
|
ea8c7459f1c4
Changed a lot of things in the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
82 if (n.kind == NodeKind.Declarations) |
|
ea8c7459f1c4
Changed a lot of things in the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
83 return name; |
|
ea8c7459f1c4
Changed a lot of things in the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
84 remove = "Declaration"; |
|
ea8c7459f1c4
Changed a lot of things in the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
85 break; |
| 363 | 86 case NC.Statement: |
| 87 if (n.kind == NodeKind.Statements) | |
| 88 return name; | |
| 89 remove = "Statement"; | |
| 90 break; | |
| 91 case NC.Expression: remove = "Expression"; break; | |
| 92 case NC.Type: remove = "Type"; break; | |
| 93 case NC.Other: return name; | |
| 94 default: | |
| 95 } | |
| 96 // Remove common suffix. | |
| 97 auto idx = find(name, remove); | |
| 98 if (idx != -1) | |
| 99 name = name[0 .. idx]; | |
| 100 // Store the name. | |
| 101 name_table[n.kind] = name; | |
| 102 return name; | |
| 103 } | |
| 104 | |
| 105 enum DocPart | |
| 106 { | |
| 107 Head, | |
| 108 CompBegin, | |
| 109 CompEnd, | |
| 110 Error, | |
| 111 SyntaxBegin, | |
| 112 SyntaxEnd, | |
| 113 SrcBegin, | |
| 114 SrcEnd, | |
| 115 Tail, | |
| 116 // Tokens: | |
| 117 Identifier, | |
| 118 Comment, | |
| 119 StringLiteral, | |
| 120 CharLiteral, | |
| 121 Operator, | |
| 122 LorG, | |
| 123 LessEqual, | |
| 124 GreaterEqual, | |
| 125 AndLogical, | |
| 126 OrLogical, | |
| 127 NotEqual, | |
| 128 Not, | |
| 129 Number, | |
| 130 Bracket, | |
| 131 SpecialToken, | |
| 132 Shebang, | |
| 133 Keyword, | |
| 134 HLineBegin, | |
| 135 HLineEnd, | |
| 136 Filespec, | |
| 137 } | |
| 138 | |
| 139 auto html_tags = [ | |
| 140 // Head | |
| 141 `<html>`\n | |
| 142 `<head>`\n | |
| 143 `<meta http-equiv="Content-Type" content="text/html; charset=utf-8">`\n | |
| 144 `<link href="dil_html.css" rel="stylesheet" type="text/css">`\n | |
| 145 `</head>`\n | |
| 146 `<body>`[], | |
| 147 // CompBegin | |
| 148 `<div class="compilerinfo">`, | |
| 149 // CompEnd | |
| 150 `</div>`, | |
| 151 // Error | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
152 `<p class="error {0}">{1}({2}){3}: {4}</p>`, |
| 363 | 153 // SyntaxBegin |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
154 `<span class="{0} {1}">`, |
| 363 | 155 // SyntaxEnd |
| 156 `</span>`, | |
| 157 // SrcBegin | |
| 158 `<pre class="sourcecode">`, | |
| 159 // SrcEnd | |
| 160 `</pre>`, | |
| 161 // Tail | |
| 162 `</html>`, | |
| 163 // Identifier | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
164 `<span class="i">{0}</span>`, |
| 363 | 165 // Comment |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
166 `<span class="c{0}">{1}</span>`, |
| 363 | 167 // StringLiteral |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
168 `<span class="sl">{0}</span>`, |
| 363 | 169 // CharLiteral |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
170 `<span class="cl">{0}</span>`, |
| 363 | 171 // Operator |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
172 `<span class="op">{0}</span>`, |
| 363 | 173 // LorG |
| 174 `<span class="oplg"><></span>`, | |
| 175 // LessEqual | |
| 176 `<span class="ople"><=</span>`, | |
| 177 // GreaterEqual | |
| 178 `<span class="opge">>=</span>`, | |
| 179 // AndLogical | |
| 180 `<span class="opaa">&&</span>`, | |
| 181 // OrLogical | |
| 182 `<span class="opoo">||</span>`, | |
| 183 // NotEqual | |
| 184 `<span class="opne">!=</span>`, | |
| 185 // Not | |
| 186 `<span class="opn">!</span>`, | |
| 187 // Number | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
188 `<span class="n">{0}</span>`, |
| 363 | 189 // Bracket |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
190 `<span class="br">{0}</span>`, |
| 363 | 191 // SpecialToken |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
192 `<span class="st">{0}</span>`, |
| 363 | 193 // Shebang |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
194 `<span class="shebang">{0}</span>`, |
| 363 | 195 // Keyword |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
196 `<span class="k">{0}</span>`, |
| 363 | 197 // HLineBegin |
|
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
198 `<span class="hl">`, |
| 363 | 199 // HLineEnd |
| 200 "</span>", | |
| 201 // Filespec | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
202 `<span class="fs">{0}</span>`, |
| 363 | 203 ]; |
| 204 | |
| 205 auto xml_tags = [ | |
| 206 // Head | |
| 207 `<?xml version="1.0"?>`\n | |
| 208 `<?xml-stylesheet href="dil_xml.css" type="text/css"?>`\n | |
| 209 `<root>`[], | |
| 210 // CompBegin | |
| 211 `<compilerinfo>`, | |
| 212 // CompEnd | |
| 213 `</compilerinfo>`, | |
| 214 // Error | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
215 `<error t="{0}">{1}({2}){3}: {4}</error>`, |
| 363 | 216 // SyntaxBegin |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
217 `<{0} t="{1}">`, |
| 363 | 218 // SyntaxEnd |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
219 `</{0}>`, |
| 363 | 220 // SrcBegin |
| 221 `<sourcecode>`, | |
| 222 // SrcEnd | |
| 223 `</sourcecode>`, | |
| 224 // Tail | |
| 225 `</root>`, | |
| 226 // Identifier | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
227 "<i>{0}</i>", |
| 363 | 228 // Comment |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
229 `<c t="{0}">{1}</c>`, |
| 363 | 230 // StringLiteral |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
231 "<sl>{0}</sl>", |
| 363 | 232 // CharLiteral |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
233 "<cl>{0}</cl>", |
| 363 | 234 // Operator |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
235 "<op>{0}</op>", |
| 363 | 236 // LorG |
| 237 `<op t="lg"><></op>`, | |
| 238 // LessEqual | |
| 239 `<op t="le"><=</op>`, | |
| 240 // GreaterEqual | |
| 241 `<op t="ge">>=</op>`, | |
| 242 // AndLogical | |
| 243 `<op t="aa">&&</op>`, | |
| 244 // OrLogical | |
| 245 `<op t="oo">||</op>`, | |
| 246 // NotEqual | |
| 247 `<op t="ne">!=</op>`, | |
| 248 // Not | |
| 249 `<op t="n">!</op>`, | |
| 250 // Number | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
251 "<n>{0}</n>", |
| 363 | 252 // Bracket |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
253 "<br>{0}</br>", |
| 363 | 254 // SpecialToken |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
255 "<st>{0}</st>", |
| 363 | 256 // Shebang |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
257 "<shebang>{0}</shebang>", |
| 363 | 258 // Keyword |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
259 "<k>{0}</k>", |
| 363 | 260 // HLineBegin |
|
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
261 "<hl>", |
| 363 | 262 // HLineEnd |
| 263 "</hl>", | |
| 264 // Filespec | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
265 "<fs>{0}</fs>", |
| 363 | 266 ]; |
| 267 | |
| 268 static assert(html_tags.length == DocPart.max+1); | |
| 269 static assert(xml_tags.length == DocPart.max+1); | |
| 270 | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
271 void syntaxToDoc(string fileName, Print!(char) print, DocOption options) |
| 363 | 272 { |
| 273 auto tags = options & DocOption.HTML ? html_tags : xml_tags; | |
| 274 auto sourceText = loadFile(fileName); | |
| 275 auto parser = new Parser(sourceText, fileName); | |
| 368 | 276 auto root = parser.start(); |
| 363 | 277 auto lx = parser.lx; |
| 278 | |
| 279 auto token = lx.head; | |
| 280 | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
281 print(tags[DocPart.Head]~\n); |
| 363 | 282 // Output error messages. |
| 283 if (lx.errors.length || parser.errors.length) | |
| 284 { | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
285 print(tags[DocPart.CompBegin]~\n); |
| 363 | 286 foreach (error; lx.errors) |
| 287 { | |
|
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
396
diff
changeset
|
288 print.formatln(tags[DocPart.Error], "L", error.filePath, Format("{0},{1}", error.loc, error.col), "L", xml_escape(error.getMsg)); |
| 363 | 289 } |
| 290 foreach (error; parser.errors) | |
| 291 { | |
|
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
396
diff
changeset
|
292 print.formatln(tags[DocPart.Error], "P", error.filePath, Format("{0},{1}", error.loc, error.col), "P", xml_escape(error.getMsg)); |
| 363 | 293 } |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
294 print(tags[DocPart.CompEnd]~\n); |
| 363 | 295 } |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
296 print(tags[DocPart.SrcBegin]); |
| 363 | 297 |
| 298 Node[][Token*] beginNodes, endNodes; | |
| 299 | |
| 300 void populateAAs(Node[] nodes) | |
| 301 { | |
| 302 foreach (node; nodes) | |
| 303 { | |
|
396
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
304 assert(delegate bool(){ |
|
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
305 foreach (child; node.children) |
|
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
306 if (child is null) |
|
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
307 return false; |
|
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
308 return true; |
|
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
309 }() == true, Format("Node '{0}' has a null child", node.classinfo.name) |
|
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
310 ); |
| 363 | 311 auto begin = node.begin; |
| 312 if (begin) | |
| 313 { | |
| 314 auto end = node.end; | |
| 315 assert(end); | |
| 316 beginNodes[begin] ~= node; | |
| 317 endNodes[end] ~= node; | |
| 318 } | |
|
396
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
319 |
| 363 | 320 if (node.children.length) |
| 321 populateAAs(node.children); | |
| 322 } | |
| 323 } | |
|
396
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
324 assert(delegate bool(){ |
|
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
325 foreach (child; root.children) |
|
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
326 if (child is null) |
|
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
327 return false; |
|
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
328 return true; |
|
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
329 }() == true, Format("Root node has a null child") |
|
0a4619735ce9
Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
394
diff
changeset
|
330 ); |
| 363 | 331 populateAAs(root.children); |
| 332 | |
| 333 char[] getTag(NodeCategory nc) | |
| 334 { | |
| 335 char[] tag; | |
| 336 switch (nc) | |
| 337 { | |
| 338 alias NodeCategory NC; | |
| 339 case NC.Declaration: tag = "d"; break; | |
| 340 case NC.Statement: tag = "s"; break; | |
| 341 case NC.Expression: tag = "e"; break; | |
| 342 case NC.Type: tag = "t"; break; | |
| 343 case NC.Other: tag = "o"; break; | |
| 344 default: | |
| 345 } | |
| 346 return tag; | |
| 347 } | |
| 348 | |
| 349 // Traverse linked list and print tokens. | |
| 350 while (token.type != TOK.EOF) | |
| 351 { | |
| 352 token = token.next; | |
| 353 | |
|
394
6440da4adb07
Fixed forward references of enum MID complaints by compiler.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
392
diff
changeset
|
354 // Print whitespace. |
|
6440da4adb07
Fixed forward references of enum MID complaints by compiler.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
392
diff
changeset
|
355 if (token.ws) |
|
6440da4adb07
Fixed forward references of enum MID complaints by compiler.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
392
diff
changeset
|
356 print(token.ws[0..token.start - token.ws]); |
|
6440da4adb07
Fixed forward references of enum MID complaints by compiler.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
392
diff
changeset
|
357 |
| 363 | 358 Node[]* nodes = token in beginNodes; |
| 359 | |
| 360 if (nodes) | |
| 361 { | |
| 362 foreach (node; *nodes) | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
363 print.format(tags[DocPart.SyntaxBegin], getTag(node.category), getShortClassName(node)); |
| 363 | 364 } |
| 365 | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
366 printToken(token, tags, print); |
| 363 | 367 |
| 368 nodes = token in endNodes; | |
| 369 | |
| 370 if (nodes) | |
| 371 { | |
| 372 foreach_reverse (node; *nodes) | |
| 373 if (options & DocOption.HTML) | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
374 print(tags[DocPart.SyntaxEnd]); |
| 363 | 375 else |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
376 print.format(tags[DocPart.SyntaxEnd], getTag(node.category)); |
| 363 | 377 } |
| 378 } | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
379 print(\n~tags[DocPart.SrcEnd])(\n~tags[DocPart.Tail]); |
| 363 | 380 } |
| 381 | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
382 void tokensToDoc(string fileName, Print!(char) print, DocOption options) |
| 363 | 383 { |
| 384 auto tags = options & DocOption.HTML ? html_tags : xml_tags; | |
| 385 auto sourceText = loadFile(fileName); | |
| 386 auto lx = new Lexer(sourceText, fileName); | |
| 387 | |
| 388 auto token = lx.getTokens(); | |
| 389 | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
390 print(tags[DocPart.Head]~\n); |
| 363 | 391 |
| 392 if (lx.errors.length) | |
| 393 { | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
394 print(tags[DocPart.CompBegin]~\n); |
| 363 | 395 foreach (error; lx.errors) |
| 396 { | |
|
422
ad7977fe315a
Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
396
diff
changeset
|
397 print.formatln(tags[DocPart.Error], "L", error.filePath, Format("{0},{1}", error.loc, error.col), "L", xml_escape(error.getMsg)); |
| 363 | 398 } |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
399 print(tags[DocPart.CompEnd]~\n); |
| 363 | 400 } |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
401 print(tags[DocPart.SrcBegin]); |
| 363 | 402 |
| 403 // Traverse linked list and print tokens. | |
| 404 while (token.type != TOK.EOF) | |
| 405 { | |
| 406 token = token.next; | |
|
394
6440da4adb07
Fixed forward references of enum MID complaints by compiler.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
392
diff
changeset
|
407 // Print whitespace. |
|
6440da4adb07
Fixed forward references of enum MID complaints by compiler.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
392
diff
changeset
|
408 if (token.ws) |
|
6440da4adb07
Fixed forward references of enum MID complaints by compiler.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
392
diff
changeset
|
409 print(token.ws[0..token.start - token.ws]); |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
410 printToken(token, tags, print); |
| 363 | 411 } |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
412 print(\n~tags[DocPart.SrcEnd])(\n~tags[DocPart.Tail]); |
| 363 | 413 } |
| 414 | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
415 void printToken(Token* token, string[] tags, Print!(char) print) |
| 363 | 416 { |
| 417 alias DocPart DP; | |
| 418 string srcText = xml_escape(token.srcText); | |
| 419 | |
| 420 switch(token.type) | |
| 421 { | |
| 422 case TOK.Identifier: | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
423 print.format(tags[DP.Identifier], srcText); |
| 363 | 424 break; |
| 425 case TOK.Comment: | |
| 426 string t; | |
| 427 switch (token.start[1]) | |
| 428 { | |
| 429 case '/': t = "l"; break; | |
| 430 case '*': t = "b"; break; | |
| 431 case '+': t = "n"; break; | |
| 432 default: | |
| 433 assert(0); | |
| 434 } | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
435 print.format(tags[DP.Comment], t, srcText); |
| 363 | 436 break; |
| 437 case TOK.String: | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
438 print.format(tags[DP.StringLiteral], srcText); |
| 363 | 439 break; |
| 440 case TOK.CharLiteral, TOK.WCharLiteral, TOK.DCharLiteral: | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
441 print.format(tags[DP.CharLiteral], srcText); |
| 363 | 442 break; |
| 443 case TOK.Assign, TOK.Equal, | |
| 444 TOK.Less, TOK.Greater, | |
| 445 TOK.LShiftAssign, TOK.LShift, | |
| 446 TOK.RShiftAssign, TOK.RShift, | |
| 447 TOK.URShiftAssign, TOK.URShift, | |
| 448 TOK.OrAssign, TOK.OrBinary, | |
| 449 TOK.AndAssign, TOK.AndBinary, | |
| 450 TOK.PlusAssign, TOK.PlusPlus, TOK.Plus, | |
| 451 TOK.MinusAssign, TOK.MinusMinus, TOK.Minus, | |
| 452 TOK.DivAssign, TOK.Div, | |
| 453 TOK.MulAssign, TOK.Mul, | |
| 454 TOK.ModAssign, TOK.Mod, | |
| 455 TOK.XorAssign, TOK.Xor, | |
| 456 TOK.CatAssign, | |
| 457 TOK.Tilde, | |
| 458 TOK.Unordered, | |
| 459 TOK.UorE, | |
| 460 TOK.UorG, | |
| 461 TOK.UorGorE, | |
| 462 TOK.UorL, | |
| 463 TOK.UorLorE, | |
| 464 TOK.LorEorG: | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
465 print.format(tags[DP.Operator], srcText); |
| 363 | 466 break; |
| 467 case TOK.LorG: | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
468 print(tags[DP.LorG]); |
| 363 | 469 break; |
| 470 case TOK.LessEqual: | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
471 print(tags[DP.LessEqual]); |
| 363 | 472 break; |
| 473 case TOK.GreaterEqual: | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
474 print(tags[DP.GreaterEqual]); |
| 363 | 475 break; |
| 476 case TOK.AndLogical: | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
477 print(tags[DP.AndLogical]); |
| 363 | 478 break; |
| 479 case TOK.OrLogical: | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
480 print(tags[DP.OrLogical]); |
| 363 | 481 break; |
| 482 case TOK.NotEqual: | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
483 print(tags[DP.NotEqual]); |
| 363 | 484 break; |
| 485 case TOK.Not: | |
| 486 // Check if this is part of a template instantiation. | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
487 // TODO: comments aren't skipped. Use Token.nextNWS and Token.prevNWS |
| 363 | 488 if (token.prev.type == TOK.Identifier && token.next.type == TOK.LParen) |
| 489 goto default; | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
490 print(tags[DP.Not]); |
| 363 | 491 break; |
| 492 case TOK.Int32, TOK.Int64, TOK.Uint32, TOK.Uint64, | |
| 493 TOK.Float32, TOK.Float64, TOK.Float80, | |
| 494 TOK.Imaginary32, TOK.Imaginary64, TOK.Imaginary80: | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
495 print.format(tags[DP.Number], srcText); |
| 363 | 496 break; |
| 497 case TOK.LParen, TOK.RParen, TOK.LBracket, | |
| 498 TOK.RBracket, TOK.LBrace, TOK.RBrace: | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
499 print.format(tags[DP.Bracket], srcText); |
| 363 | 500 break; |
| 501 case TOK.Shebang: | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
502 print.format(tags[DP.Shebang], srcText); |
| 363 | 503 break; |
| 504 case TOK.HashLine: | |
| 505 void printWS(char* start, char* end) | |
| 506 { | |
| 507 if (start != end) | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
508 print(start[0 .. end - start]); |
| 363 | 509 } |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
510 print(tags[DP.HLineBegin]); |
|
485
ea8c7459f1c4
Changed a lot of things in the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
511 auto num = token.tokLineNum; |
|
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
512 if (num is null) |
|
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
513 { |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
514 print(token.srcText); |
|
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
515 print(tags[DP.HLineEnd]); |
|
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
516 break; |
|
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
517 } |
| 363 | 518 // Print whitespace between #line and number |
|
388
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
519 auto ptr = token.start; |
|
ae154eceba65
Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
379
diff
changeset
|
520 printWS(ptr, num.start); // prints "#line" as well |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
521 printToken(num, tags, print); |
|
485
ea8c7459f1c4
Changed a lot of things in the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
522 if (token.tokLineFilespec) |
| 363 | 523 { |
|
485
ea8c7459f1c4
Changed a lot of things in the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
422
diff
changeset
|
524 auto filespec = token.tokLineFilespec; |
| 363 | 525 // Print whitespace between number and filespec |
| 526 printWS(num.end, filespec.start); | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
527 print.format(tags[DP.Filespec], xml_escape(filespec.srcText)); |
| 363 | 528 |
| 529 ptr = filespec.end; | |
| 530 } | |
| 531 else | |
| 532 ptr = num.end; | |
| 533 // Print remaining whitespace | |
| 534 printWS(ptr, token.end); | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
535 print(tags[DP.HLineEnd]); |
| 363 | 536 break; |
| 537 default: | |
| 538 if (token.isKeyword()) | |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
539 print.format(tags[DP.Keyword], srcText); |
| 363 | 540 else if (token.isSpecialToken) |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
541 print.format(tags[DP.SpecialToken], srcText); |
| 363 | 542 else |
|
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
543 print(srcText); |
| 363 | 544 } |
| 545 } |