Mercurial > projects > ldc
annotate tests/makewebstatistics.d @ 314:8d98e42ece93 trunk
[svn r335] The basics of exception handling are in place.
Still need to make sure calls are turned into invokes everywhere. (NewExpression for instance)
Still some rough edges and corner cases to figure out.
Needs testing!
author | ChristianK |
---|---|
date | Wed, 02 Jul 2008 22:20:18 +0200 |
parents | 5d2f4814bb2e |
children | c9d5c711d65a |
rev | line source |
---|---|
255 | 1 // Based on DSTRESS code by Thomas Kühne |
2 | |
3 module findregressions; | |
4 | |
5 private import std.string; | |
6 private import std.conv; | |
7 private import std.stdio; | |
8 private import std.stream; | |
9 private import std.file; | |
10 private import std.c.stdlib; | |
11 private import std.date; | |
12 private import std.path; | |
13 | |
14 | |
15 enum Result{ | |
16 UNTESTED = 0, | |
17 PASS = 1 << 2, | |
18 XFAIL = 2 << 2, | |
19 XPASS = 3 << 2, | |
20 FAIL = 4 << 2, | |
21 ERROR = 5 << 2, | |
22 BASE_MASK = 7 << 2, | |
23 | |
24 EXT_MASK = 3, | |
25 BAD_MSG = 1, | |
26 BAD_GDB = 2, | |
27 | |
28 MAX = BAD_GDB + BASE_MASK | |
29 } | |
30 | |
31 char[] toString(Result r){ | |
32 switch(r & Result.BASE_MASK){ | |
33 case Result.PASS: return "PASS"; | |
34 case Result.XPASS: return "XPASS"; | |
35 case Result.FAIL: return "FAIL"; | |
36 case Result.XFAIL: return "XFAIL"; | |
37 case Result.ERROR: return "ERROR"; | |
38 case Result.UNTESTED: return "UNTESTED"; | |
39 default: | |
40 break; | |
41 } | |
42 throw new Exception(format("unhandled Result value %s", cast(int)r)); | |
43 } | |
44 | |
45 char[] dateString(){ | |
46 static char[] date; | |
47 if(date is null){ | |
48 auto time = getUTCtime(); | |
49 auto year = YearFromTime(time); | |
50 auto month = MonthFromTime(time); | |
51 auto day = DateFromTime(time); | |
52 date = format("%d-%02d-%02d", year, month+1, day); | |
53 } | |
54 return date; | |
55 } | |
56 | |
57 char[][] unique(char[][] a){ | |
58 char[][] b = a.sort; | |
59 char[][] back; | |
60 | |
61 back ~= b[0]; | |
62 | |
63 size_t ii=0; | |
64 for(size_t i=0; i<b.length; i++){ | |
65 if(back[ii]!=b[i]){ | |
66 back~=b[i]; | |
67 ii++; | |
68 } | |
69 } | |
70 | |
71 return back; | |
72 } | |
73 | |
74 private{ | |
75 version(Windows){ | |
76 import std.c.windows.windows; | |
77 extern(Windows) BOOL GetFileTime(HANDLE hFile, LPFILETIME lpCreationTime, LPFILETIME lpLastAccessTime, LPFILETIME lpLastWriteTime); | |
78 }else version(linux){ | |
79 import std.c.linux.linux; | |
80 version = Unix; | |
81 }else version(Unix){ | |
82 import std.c.unix.unix; | |
83 }else{ | |
84 static assert(0); | |
85 } | |
86 | |
87 alias ulong FStime; | |
88 | |
89 FStime getFStime(char[] fileName){ | |
90 version(Windows){ | |
91 HANDLE h; | |
92 | |
93 if (useWfuncs){ | |
94 wchar* namez = std.utf.toUTF16z(fileName); | |
95 h = CreateFileW(namez,GENERIC_WRITE,0,null,OPEN_ALWAYS, | |
96 FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,cast(HANDLE)null); | |
97 }else{ | |
98 char* namez = toMBSz(fileName); | |
99 h = CreateFileA(namez,GENERIC_WRITE,0,null,OPEN_ALWAYS, | |
100 FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,cast(HANDLE)null); | |
101 } | |
102 | |
103 if (h == INVALID_HANDLE_VALUE) | |
104 goto err; | |
105 | |
106 FILETIME creationTime; | |
107 FILETIME accessTime; | |
108 FILETIME writeTime; | |
109 | |
110 BOOL b = GetFileTime(h, &creationTime, &accessTime, &writeTime); | |
111 if(b==1){ | |
112 long modA = writeTime.dwLowDateTime; | |
113 long modB = writeTime.dwHighDateTime; | |
114 return modA | (modB << (writeTime.dwHighDateTime.sizeof*8)); | |
115 } | |
116 | |
117 err: | |
118 CloseHandle(h); | |
119 throw new Exception("failed to query file modification : "~fileName); | |
120 }else version(Unix){ | |
121 char* namez = toStringz(fileName); | |
122 struct_stat statbuf; | |
123 | |
124 if(stat(namez, &statbuf)){ | |
125 throw new FileException(fileName, getErrno()); | |
126 } | |
127 | |
128 return statbuf.st_mtime; | |
129 }else{ | |
130 static assert(0); | |
131 } | |
132 } | |
133 } | |
134 | |
135 char[] cleanFileName(char[] file){ | |
136 char[] back; | |
137 bool hadSep; | |
138 | |
139 foreach(char c; file){ | |
140 if(c == '/' || c == '\\'){ | |
141 if(!hadSep){ | |
142 back ~= '/'; | |
143 hadSep = true; | |
144 } | |
145 }else{ | |
146 back ~= c; | |
147 hadSep = false; | |
148 } | |
149 } | |
150 | |
151 size_t start = 0; | |
152 while(back[start] <= ' ' && start < back.length){ | |
153 start++; | |
154 } | |
155 | |
156 size_t end = back.length-1; | |
157 while(back[end] <= ' ' && end >= start){ | |
158 end--; | |
159 } | |
160 | |
161 back = back[start .. end+1]; | |
162 | |
163 return back; | |
164 } | |
165 | |
166 class Test{ | |
167 char[] name; | |
168 char[] file; | |
169 Result r; | |
170 | |
171 this(char[] file){ | |
172 this.file = file; | |
173 | |
174 int start = rfind(file, "/"); | |
175 if(start<0){ | |
176 start = 0; | |
177 }else{ | |
178 start += 1; | |
179 } | |
180 | |
181 int end = rfind(file, "."); | |
182 if(end < start){ | |
183 end = file.length; | |
184 } | |
185 | |
186 name = file[start .. end]; | |
187 } | |
188 } | |
189 | |
190 | |
191 class Log{ | |
192 Test[char[]] tests; | |
193 | |
194 char[] id; | |
195 | |
196 int[Result] counts; | |
197 | |
198 this(char[] id, char[] file){ | |
199 this.id = id; | |
200 counts = [ | |
201 Result.PASS: 0, | |
202 Result.FAIL: 0, | |
203 Result.XPASS: 0, | |
204 Result.XFAIL: 0, | |
205 Result.ERROR: 0 ]; | |
206 | |
207 writefln("parsing: %s", file); | |
208 FStime logTime = getFStime(file); | |
209 Stream source = new BufferedFile(file, FileMode.In); | |
210 while(!source.eof()){ | |
211 add(source.readLine()); | |
212 } | |
213 dropBogusResults(logTime, "dstress"); | |
214 } | |
215 | |
216 | |
217 void dropBogusResults(FStime recordTime, char[] testRoot){ | |
218 uint totalCount = tests.length; | |
219 | |
220 char[][] sourcesTests = tests.keys; | |
221 foreach(char[] source; sourcesTests){ | |
222 if(find(source, "complex/") < 0){ | |
223 try{ | |
224 FStime caseTime = getFStime(testRoot~std.path.sep~source); | |
225 if(caseTime > recordTime){ | |
226 debug(drop) fwritefln(stderr, "dropped: %s", source); | |
227 counts[tests[source].r & Result.BASE_MASK]--; | |
228 tests.remove(source); | |
229 } | |
230 }catch(Exception e){ | |
231 debug(drop) fwritefln(stderr, "dropped: %s", source); | |
232 counts[tests[source].r & Result.BASE_MASK]--; | |
233 tests.remove(source); | |
234 } | |
235 } | |
236 // asm-filter | |
237 int i = find(source, "asm_p"); | |
238 if(i >= 0){ | |
239 counts[tests[source].r & Result.BASE_MASK]--; | |
240 tests.remove(source); | |
241 } | |
242 } | |
243 tests.rehash; | |
244 | |
245 writefln("dropped %s outdated tests (%s remaining)", totalCount - tests.length, tests.length); | |
246 } | |
247 | |
248 | |
249 bool add(char[] line){ | |
250 const char[] SUB = "Torture-Sub-"; | |
251 const char[] TORTURE = "Torture:"; | |
252 | |
253 line = strip(line); | |
254 int id = -1; | |
255 Result r = Result.UNTESTED; | |
256 | |
257 if(line.length > SUB.length && line[0 .. SUB.length] == SUB){ | |
258 line = line[SUB.length .. $]; | |
259 id = 0; | |
260 while(line[id] >= '0' && line[id] <= '9'){ | |
261 id++; | |
262 } | |
263 int start = id; | |
264 id = std.conv.toUint(line[0 .. id]); | |
265 | |
266 while(line[start] != '-'){ | |
267 start++; | |
268 } | |
269 line = line[start+1 .. $]; | |
270 } | |
271 | |
272 char[][] token = split(line); | |
273 if(token.length < 2){ | |
274 return false; | |
275 } | |
276 char[] file = strip(token[1]); | |
277 | |
278 switch(token[0]){ | |
279 case "PASS:": | |
280 r = Result.PASS; break; | |
281 case "FAIL:": | |
282 r = Result.FAIL; break; | |
283 case "XPASS:": | |
284 r = Result.XPASS; break; | |
285 case "XFAIL:": | |
286 r = Result.XFAIL; break; | |
287 case "ERROR:": | |
288 r = Result.ERROR; break; | |
289 default:{ | |
290 if(token[0] == TORTURE){ | |
291 throw new Exception("not yet handled: "~line); | |
292 }else if(id > -1){ | |
293 throw new Exception(format("bug in SUB line: (%s) %s", id, line)); | |
294 } | |
295 } | |
296 } | |
297 | |
298 if(r != Result.UNTESTED){ | |
299 if(std.string.find(line, "bad error message") > -1){ | |
300 r |= Result.BAD_MSG; | |
301 } | |
302 if(std.string.find(line, "bad debugger message") > -1){ | |
303 r |= Result.BAD_MSG; | |
304 } | |
305 | |
306 file = cleanFileName(file); | |
307 | |
308 if(id >= 0){ | |
309 // update sub | |
310 id--; | |
311 | |
312 Test* test = file in tests; | |
313 | |
314 if(test is null){ | |
315 Test t = new Test(file); | |
316 tests[file] = t; | |
317 t.r = r; | |
318 counts[r & Result.BASE_MASK]++; | |
319 }else{ | |
320 if(test.r != Result.UNTESTED){ | |
321 test.r = Result.UNTESTED; | |
322 } | |
323 test.r = r; | |
324 } | |
325 } | |
326 return true; | |
327 } | |
328 return false; | |
329 } | |
330 } | |
331 | |
332 | |
266 | 333 char[] basedir = "web"; |
334 bool regenerate = false; | |
335 | |
255 | 336 int main(char[][] args){ |
337 | |
266 | 338 if(args.length < 3 || (args[1] == "--regenerate" && args.length < 4)){ |
339 fwritefln(stderr, "%s [--regenerate] <reference-log> <log> <log> ...", args[0]); | |
340 fwritefln(stderr, "bash example: %s reference/dmd-something $(ls reference/llvmdc*)", args[0]); | |
255 | 341 return 1; |
342 } | |
343 | |
266 | 344 char[] reference; |
257 | 345 char[][] files; |
346 if(args[1] == "--regenerate") { | |
347 regenerate = true; | |
266 | 348 reference = args[2]; |
349 files = args[3..$] ~ reference; | |
257 | 350 } else { |
266 | 351 reference = args[1]; |
352 files = args[2..$] ~ reference; | |
257 | 353 } |
354 | |
255 | 355 // make sure base path exists |
356 if(std.file.exists(basedir) && !std.file.isdir(basedir)) | |
357 throw new Exception(basedir ~ " is not a directory!"); | |
358 else if(!std.file.exists(basedir)) | |
359 std.file.mkdir(basedir); | |
360 | |
361 | |
362 Log[char[]] logs; | |
363 | |
266 | 364 // emit per-log data |
365 foreach(char[] file; files) | |
366 generateLogStatistics(file, logs); | |
367 | |
368 // differences between logs | |
369 foreach(int i, char[] file; files[1 .. $]) | |
370 generateChangeStatistics(files[1+i], files[1+i-1], logs); | |
371 | |
372 // differences between reference and logs | |
373 foreach(char[] file; files[0..$-1]) | |
374 generateChangeStatistics(file, reference, logs); | |
375 | |
376 // collect all the stats.base files into a large table | |
377 BufferedFile index = new BufferedFile(std.path.join(basedir, "index.html"), FileMode.OutNew); | |
378 scope(exit) index.close(); | |
379 index.writefln(` | |
380 <html><body> | |
381 <table style="border-collapse:collapse; text-align:center;"> | |
382 <colgroup> | |
383 <col style="border-right: medium solid black;"> | |
384 <col style="background-color: #AAFFAA;"> | |
385 <col style="background-color: #AAFFAA; border-right: thin solid black;"> | |
386 <col style="background-color: #FFAAAA;"> | |
387 <col style="background-color: #FFAAAA;"> | |
388 <col style="background-color: #FFAAAA;"> | |
389 <col style="border-left: medium solid black;"> | |
390 </colgroup> | |
391 <tr style="border-bottom: medium solid black;"> | |
392 <th>name</th> | |
393 <th style="padding-left:1em;padding-right:1em;">PASS</th> | |
394 <th style="padding-left:1em;padding-right:1em;">XFAIL</th> | |
395 <th style="padding-left:1em;padding-right:1em;">FAIL</th> | |
396 <th style="padding-left:1em;padding-right:1em;">XPASS</th> | |
397 <th style="padding-left:1em;padding-right:1em;">ERROR</th> | |
398 <th style="padding-left:1em;padding-right:1em;">comparison to ` ~ std.path.getBaseName(reference) ~ `</th> | |
399 </tr> | |
400 `); | |
255 | 401 |
266 | 402 for(int i = files.length - 1; i >= 0; --i) { |
403 auto file = files[i]; | |
404 index.writefln(`<tr>`); | |
405 char[] id = std.path.getBaseName(file); | |
406 char[] statsname = std.path.join(std.path.join(basedir, id), "stats.base"); | |
407 index.writef(cast(char[])std.file.read(statsname)); | |
408 | |
409 if(i != files.length - 1) { | |
410 index.writefln(`<td>`); | |
411 char[] refid = std.path.getBaseName(reference); | |
412 statsname = std.path.join(std.path.join(basedir, refid ~ "-to-" ~ id), "stats.base"); | |
413 index.writef(cast(char[])std.file.read(statsname)); | |
414 index.writefln(`</td></tr>`); | |
415 } else { | |
416 index.writefln(`<td></td></tr>`); | |
417 } | |
418 | |
419 if(i == 0) { | |
420 continue; | |
421 } | |
422 | |
423 index.writefln(`<tr><td></td>`); | |
424 index.writefln(`<td style="background-color:white;" colspan="5">`); | |
425 char[] newid = std.path.getBaseName(files[i-1]); | |
426 statsname = std.path.join(std.path.join(basedir, newid ~ "-to-" ~ id), "stats.base"); | |
427 index.writef(cast(char[])std.file.read(statsname)); | |
428 index.writefln(`</td><td></td></tr>`); | |
429 } | |
430 | |
431 index.writefln(`</table></body></html>`); | |
432 | |
433 return 0; | |
434 } | |
435 | |
436 void generateLogStatistics(char[] file, ref Log[char[]] logs) | |
437 { | |
438 char[] id = std.path.getBaseName(file); | |
439 char[] dirname = std.path.join(basedir, id); | |
440 | |
441 if(std.file.exists(dirname)) { | |
442 if(std.file.isdir(dirname)) { | |
443 if(!regenerate) { | |
444 writefln("Directory ", dirname, " already exists, skipping..."); | |
445 return; | |
255 | 446 } |
447 } | |
448 else | |
266 | 449 throw new Exception(dirname ~ " is not a directory!"); |
450 } | |
451 else | |
452 std.file.mkdir(dirname); | |
255 | 453 |
266 | 454 // parse etc. |
455 Log log = new Log(id, file); | |
456 logs[id] = log; | |
255 | 457 |
266 | 458 // write status |
459 { | |
255 | 460 BufferedFile makeFile(char[] name) { |
261
5723b7385c25
[svn r279] fixed bug in makewebstatistics, regenerated output
ChristianK
parents:
257
diff
changeset
|
461 return new BufferedFile(std.path.join(dirname, name), FileMode.OutNew); |
255 | 462 } |
463 BufferedFile[Result] resultsfile = [ | |
464 Result.PASS: makeFile("pass.html"), | |
465 Result.FAIL: makeFile("fail.html"), | |
466 Result.XPASS: makeFile("xpass.html"), | |
467 Result.XFAIL: makeFile("xfail.html"), | |
468 Result.ERROR: makeFile("error.html") ]; | |
266 | 469 |
255 | 470 scope(exit) { |
471 foreach(file; resultsfile) | |
472 file.close(); | |
473 } | |
266 | 474 |
475 foreach(file; resultsfile) | |
476 file.writefln(`<html><body>`); | |
477 | |
255 | 478 foreach(tkey; log.tests.keys.sort) { |
479 auto test = log.tests[tkey]; | |
480 auto result = test.r & Result.BASE_MASK; | |
266 | 481 resultsfile[result].writefln(test.name, " in ", test.file, "<br>"); |
255 | 482 } |
483 | |
266 | 484 foreach(file; resultsfile) |
485 file.writefln(`</body></html>`); | |
486 } | |
255 | 487 |
266 | 488 BufferedFile stats = new BufferedFile(std.path.join(dirname, "stats.base"), FileMode.OutNew); |
489 scope(exit) stats.close(); | |
490 stats.writefln(`<td style="padding-right:1em; text-align:left;">`, id, `</td>`); | |
491 stats.writefln(`<td><a href="`, std.path.join(log.id, "pass.html"), `">`, log.counts[Result.PASS], `</a></td>`); | |
492 stats.writefln(`<td><a href="`, std.path.join(log.id, "xfail.html"), `">`, log.counts[Result.XFAIL], `</a></td>`); | |
493 stats.writefln(`<td><a href="`, std.path.join(log.id, "fail.html"), `">`, log.counts[Result.FAIL], `</a></td>`); | |
494 stats.writefln(`<td><a href="`, std.path.join(log.id, "xpass.html"), `">`, log.counts[Result.XPASS], `</a></td>`); | |
495 stats.writefln(`<td><a href="`, std.path.join(log.id, "error.html"), `">`, log.counts[Result.ERROR], `</a></td>`); | |
496 } | |
255 | 497 |
266 | 498 void generateChangeStatistics(char[] file1, char[] file2, ref Log[char[]] logs) |
499 { | |
500 char[] newid = std.path.getBaseName(file1); | |
501 char[] oldid = std.path.getBaseName(file2); | |
502 | |
503 char[] dirname = std.path.join(basedir, oldid ~ "-to-" ~ newid); | |
504 | |
505 if(std.file.exists(dirname)) { | |
506 if(std.file.isdir(dirname)) { | |
507 if(!regenerate) { | |
508 writefln("Directory ", dirname, " already exists, skipping..."); | |
509 return; | |
255 | 510 } |
511 } | |
512 else | |
266 | 513 throw new Exception(dirname ~ " is not a directory!"); |
514 } | |
515 else | |
516 std.file.mkdir(dirname); | |
255 | 517 |
266 | 518 // parse etc. |
519 Log newLog, oldLog; | |
520 Log getOrParse(char[] id, char[] file) { | |
521 if(id in logs) | |
522 return logs[id]; | |
523 else { | |
524 Log tmp = new Log(id, file); | |
525 logs[id] = tmp; | |
526 return tmp; | |
255 | 527 } |
266 | 528 } |
529 newLog = getOrParse(newid, file1); | |
530 oldLog = getOrParse(oldid, file2); | |
255 | 531 |
266 | 532 int nRegressions, nImprovements, nChanges; |
533 | |
534 { | |
261
5723b7385c25
[svn r279] fixed bug in makewebstatistics, regenerated output
ChristianK
parents:
257
diff
changeset
|
535 auto regressionsFile = new BufferedFile(std.path.join(dirname, "regressions.html"), FileMode.OutNew); |
255 | 536 scope(exit) regressionsFile.close(); |
266 | 537 regressionsFile.writefln(`<html><body>`); |
538 | |
261
5723b7385c25
[svn r279] fixed bug in makewebstatistics, regenerated output
ChristianK
parents:
257
diff
changeset
|
539 auto improvementsFile = new BufferedFile(std.path.join(dirname, "improvements.html"), FileMode.OutNew); |
255 | 540 scope(exit) improvementsFile.close(); |
266 | 541 improvementsFile.writefln(`<html><body>`); |
542 | |
261
5723b7385c25
[svn r279] fixed bug in makewebstatistics, regenerated output
ChristianK
parents:
257
diff
changeset
|
543 auto changesFile = new BufferedFile(std.path.join(dirname, "changes.html"), FileMode.OutNew); |
255 | 544 scope(exit) changesFile.close(); |
266 | 545 changesFile.writefln(`<html><body>`); |
546 | |
255 | 547 BufferedFile targetFile; |
266 | 548 |
255 | 549 foreach(Test t; newLog.tests.values){ |
550 Test* oldT = t.file in oldLog.tests; | |
551 | |
552 if(oldT !is null){ | |
553 if(oldT.r == t.r) | |
554 continue; | |
555 else if(oldT.r < t.r && oldT.r && oldT.r <= Result.XFAIL){ | |
556 targetFile = regressionsFile; | |
557 nRegressions++; | |
558 } | |
559 else if(t.r < oldT.r && t.r && t.r <= Result.XFAIL){ | |
560 targetFile = improvementsFile; | |
561 nImprovements++; | |
562 } | |
563 else { | |
564 targetFile = changesFile; | |
565 nChanges++; | |
566 } | |
266 | 567 targetFile.writefln(toString(oldT.r), " -> ", toString(t.r), " : ", t.name, " in ", t.file, "<br>"); |
255 | 568 } |
266 | 569 } |
255 | 570 |
266 | 571 regressionsFile.writefln(`</body></html>`); |
572 improvementsFile.writefln(`</body></html>`); | |
573 changesFile.writefln(`</body></html>`); | |
255 | 574 } |
575 | |
266 | 576 BufferedFile stats = new BufferedFile(std.path.join(dirname, "stats.base"), FileMode.OutNew); |
577 scope(exit) stats.close(); | |
578 auto dir = oldid ~ "-to-" ~ newid; | |
579 stats.writefln(`<a href="`, std.path.join(dir, "improvements.html"), `">Improvements: `, nImprovements, `</a>, `); | |
580 stats.writefln(`<a href="`, std.path.join(dir, "regressions.html"), `">Regressions: `, nRegressions, `</a>, `); | |
581 stats.writefln(`<a href="`, std.path.join(dir, "changes.html"), `">Changes: `, nChanges, `</a>`); | |
255 | 582 } |