Mercurial > projects > ldc

comparison tests/makewebstatistics.d @ 255:4d14a1c84be7 trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[svn r272] some rough html statistics for the tests
author ChristianK
date Thu, 12 Jun 2008 18:19:54 +0200
parents
children 26127a48bc09
comparison
equal deleted inserted replaced
254:8187884566fa 255:4d14a1c84be7
1 // Based on DSTRESS code by Thomas Kühne
2
3 module findregressions;
4
5 private import std.string;
6 private import std.conv;
7 private import std.stdio;
8 private import std.stream;
9 private import std.file;
10 private import std.c.stdlib;
11 private import std.date;
12 private import std.path;
13
14
15 enum Result{
16 UNTESTED = 0,
17 PASS = 1 << 2,
18 XFAIL = 2 << 2,
19 XPASS = 3 << 2,
20 FAIL = 4 << 2,
21 ERROR = 5 << 2,
22 BASE_MASK = 7 << 2,
23
24 EXT_MASK = 3,
25 BAD_MSG = 1,
26 BAD_GDB = 2,
27
28 MAX = BAD_GDB + BASE_MASK
29 }
30
31 char[] toString(Result r){
32 switch(r & Result.BASE_MASK){
33 case Result.PASS: return "PASS";
34 case Result.XPASS: return "XPASS";
35 case Result.FAIL: return "FAIL";
36 case Result.XFAIL: return "XFAIL";
37 case Result.ERROR: return "ERROR";
38 case Result.UNTESTED: return "UNTESTED";
39 default:
40 break;
41 }
42 throw new Exception(format("unhandled Result value %s", cast(int)r));
43 }
44
45 char[] dateString(){
46 static char[] date;
47 if(date is null){
48 auto time = getUTCtime();
49 auto year = YearFromTime(time);
50 auto month = MonthFromTime(time);
51 auto day = DateFromTime(time);
52 date = format("%d-%02d-%02d", year, month+1, day);
53 }
54 return date;
55 }
56
57 char[][] unique(char[][] a){
58 char[][] b = a.sort;
59 char[][] back;
60
61 back ~= b[0];
62
63 size_t ii=0;
64 for(size_t i=0; i<b.length; i++){
65 if(back[ii]!=b[i]){
66 back~=b[i];
67 ii++;
68 }
69 }
70
71 return back;
72 }
73
74 private{
75 version(Windows){
76 import std.c.windows.windows;
77 extern(Windows) BOOL GetFileTime(HANDLE hFile, LPFILETIME lpCreationTime, LPFILETIME lpLastAccessTime, LPFILETIME lpLastWriteTime);
78 }else version(linux){
79 import std.c.linux.linux;
80 version = Unix;
81 }else version(Unix){
82 import std.c.unix.unix;
83 }else{
84 static assert(0);
85 }
86
87 alias ulong FStime;
88
89 FStime getFStime(char[] fileName){
90 version(Windows){
91 HANDLE h;
92
93 if (useWfuncs){
94 wchar* namez = std.utf.toUTF16z(fileName);
95 h = CreateFileW(namez,GENERIC_WRITE,0,null,OPEN_ALWAYS,
96 FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,cast(HANDLE)null);
97 }else{
98 char* namez = toMBSz(fileName);
99 h = CreateFileA(namez,GENERIC_WRITE,0,null,OPEN_ALWAYS,
100 FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,cast(HANDLE)null);
101 }
102
103 if (h == INVALID_HANDLE_VALUE)
104 goto err;
105
106 FILETIME creationTime;
107 FILETIME accessTime;
108 FILETIME writeTime;
109
110 BOOL b = GetFileTime(h, &creationTime, &accessTime, &writeTime);
111 if(b==1){
112 long modA = writeTime.dwLowDateTime;
113 long modB = writeTime.dwHighDateTime;
114 return modA | (modB << (writeTime.dwHighDateTime.sizeof*8));
115 }
116
117 err:
118 CloseHandle(h);
119 throw new Exception("failed to query file modification : "~fileName);
120 }else version(Unix){
121 char* namez = toStringz(fileName);
122 struct_stat statbuf;
123
124 if(stat(namez, &statbuf)){
125 throw new FileException(fileName, getErrno());
126 }
127
128 return statbuf.st_mtime;
129 }else{
130 static assert(0);
131 }
132 }
133 }
134
135 char[] cleanFileName(char[] file){
136 char[] back;
137 bool hadSep;
138
139 foreach(char c; file){
140 if(c == '/' || c == '\\'){
141 if(!hadSep){
142 back ~= '/';
143 hadSep = true;
144 }
145 }else{
146 back ~= c;
147 hadSep = false;
148 }
149 }
150
151 size_t start = 0;
152 while(back[start] <= ' ' && start < back.length){
153 start++;
154 }
155
156 size_t end = back.length-1;
157 while(back[end] <= ' ' && end >= start){
158 end--;
159 }
160
161 back = back[start .. end+1];
162
163 return back;
164 }
165
166 class Test{
167 char[] name;
168 char[] file;
169 Result r;
170
171 this(char[] file){
172 this.file = file;
173
174 int start = rfind(file, "/");
175 if(start<0){
176 start = 0;
177 }else{
178 start += 1;
179 }
180
181 int end = rfind(file, ".");
182 if(end < start){
183 end = file.length;
184 }
185
186 name = file[start .. end];
187 }
188 }
189
190
191 class Log{
192 Test[char[]] tests;
193
194 char[] id;
195
196 int[Result] counts;
197
198 this(char[] id, char[] file){
199 this.id = id;
200 counts = [
201 Result.PASS: 0,
202 Result.FAIL: 0,
203 Result.XPASS: 0,
204 Result.XFAIL: 0,
205 Result.ERROR: 0 ];
206
207 writefln("parsing: %s", file);
208 FStime logTime = getFStime(file);
209 Stream source = new BufferedFile(file, FileMode.In);
210 while(!source.eof()){
211 add(source.readLine());
212 }
213 dropBogusResults(logTime, "dstress");
214 }
215
216
217 void dropBogusResults(FStime recordTime, char[] testRoot){
218 uint totalCount = tests.length;
219
220 char[][] sourcesTests = tests.keys;
221 foreach(char[] source; sourcesTests){
222 if(find(source, "complex/") < 0){
223 try{
224 FStime caseTime = getFStime(testRoot~std.path.sep~source);
225 if(caseTime > recordTime){
226 debug(drop) fwritefln(stderr, "dropped: %s", source);
227 counts[tests[source].r & Result.BASE_MASK]--;
228 tests.remove(source);
229 }
230 }catch(Exception e){
231 debug(drop) fwritefln(stderr, "dropped: %s", source);
232 counts[tests[source].r & Result.BASE_MASK]--;
233 tests.remove(source);
234 }
235 }
236 // asm-filter
237 int i = find(source, "asm_p");
238 if(i >= 0){
239 counts[tests[source].r & Result.BASE_MASK]--;
240 tests.remove(source);
241 }
242 }
243 tests.rehash;
244
245 writefln("dropped %s outdated tests (%s remaining)", totalCount - tests.length, tests.length);
246 }
247
248
249 bool add(char[] line){
250 const char[] SUB = "Torture-Sub-";
251 const char[] TORTURE = "Torture:";
252
253 line = strip(line);
254 int id = -1;
255 Result r = Result.UNTESTED;
256
257 if(line.length > SUB.length && line[0 .. SUB.length] == SUB){
258 line = line[SUB.length .. $];
259 id = 0;
260 while(line[id] >= '0' && line[id] <= '9'){
261 id++;
262 }
263 int start = id;
264 id = std.conv.toUint(line[0 .. id]);
265
266 while(line[start] != '-'){
267 start++;
268 }
269 line = line[start+1 .. $];
270 }
271
272 char[][] token = split(line);
273 if(token.length < 2){
274 return false;
275 }
276 char[] file = strip(token[1]);
277
278 switch(token[0]){
279 case "PASS:":
280 r = Result.PASS; break;
281 case "FAIL:":
282 r = Result.FAIL; break;
283 case "XPASS:":
284 r = Result.XPASS; break;
285 case "XFAIL:":
286 r = Result.XFAIL; break;
287 case "ERROR:":
288 r = Result.ERROR; break;
289 default:{
290 if(token[0] == TORTURE){
291 throw new Exception("not yet handled: "~line);
292 }else if(id > -1){
293 throw new Exception(format("bug in SUB line: (%s) %s", id, line));
294 }
295 }
296 }
297
298 if(r != Result.UNTESTED){
299 if(std.string.find(line, "bad error message") > -1){
300 r |= Result.BAD_MSG;
301 }
302 if(std.string.find(line, "bad debugger message") > -1){
303 r |= Result.BAD_MSG;
304 }
305
306 file = cleanFileName(file);
307
308 if(id >= 0){
309 // update sub
310 id--;
311
312 Test* test = file in tests;
313
314 if(test is null){
315 Test t = new Test(file);
316 tests[file] = t;
317 t.r = r;
318 counts[r & Result.BASE_MASK]++;
319 }else{
320 if(test.r != Result.UNTESTED){
321 test.r = Result.UNTESTED;
322 }
323 test.r = r;
324 }
325 }
326 return true;
327 }
328 return false;
329 }
330 }
331
332
333 int main(char[][] args){
334
335 if(args.length < 2){
336 fwritefln(stderr, "%s <log> <log> ...", args[0]);
337 fwritefln(stderr, "bash example: %s $(ls reference/llvmdc*)", args[0]);
338 return 1;
339 }
340
341 // make sure base path exists
342 char[] basedir = "web";
343 if(std.file.exists(basedir) && !std.file.isdir(basedir))
344 throw new Exception(basedir ~ " is not a directory!");
345 else if(!std.file.exists(basedir))
346 std.file.mkdir(basedir);
347
348
349 Log[char[]] logs;
350
351 // parse log and emit per-log data if necessary
352 foreach(char[] file; args[1 .. $]){
353 char[] id = std.path.getBaseName(file);
354 char[] dirname = std.path.join(basedir, id);
355
356 if(std.file.exists(dirname)) {
357 if(std.file.isdir(dirname)) {
358 writefln("Directory ", dirname, " already exists, skipping...");
359 continue;
360 }
361 else
362 throw new Exception(dirname ~ " is not a directory!");
363 }
364 else
365 std.file.mkdir(dirname);
366
367 // parse etc.
368 Log log = new Log(id, file);
369 logs[id] = log;
370
371 // write status
372 BufferedFile makeFile(char[] name) {
373 return new BufferedFile(std.path.join(dirname, name), FileMode.Out);
374 }
375 BufferedFile[Result] resultsfile = [
376 Result.PASS: makeFile("pass.html"),
377 Result.FAIL: makeFile("fail.html"),
378 Result.XPASS: makeFile("xpass.html"),
379 Result.XFAIL: makeFile("xfail.html"),
380 Result.ERROR: makeFile("error.html") ];
381
382 scope(exit) {
383 foreach(file; resultsfile)
384 file.close();
385 }
386
387
388 foreach(tkey; log.tests.keys.sort) {
389 auto test = log.tests[tkey];
390 auto result = test.r & Result.BASE_MASK;
391 resultsfile[result].writefln(test.name, " in ", test.file, "<br>");
392 }
393
394
395 BufferedFile stats = new BufferedFile(std.path.join(dirname, "stats.base"), FileMode.Out);
396 scope(exit) stats.close();
397 stats.writefln(`<tr>`);
398 stats.writefln(`<td style="padding-right:1em; text-align:left;">`, id, `</td>`);
399 stats.writefln(`<td><a href="`, std.path.join(log.id, "pass.html"), `">`, log.counts[Result.PASS], `</a></td>`);
400 stats.writefln(`<td><a href="`, std.path.join(log.id, "xfail.html"), `">`, log.counts[Result.XFAIL], `</a></td>`);
401 stats.writefln(`<td><a href="`, std.path.join(log.id, "fail.html"), `">`, log.counts[Result.FAIL], `</a></td>`);
402 stats.writefln(`<td><a href="`, std.path.join(log.id, "xpass.html"), `">`, log.counts[Result.XPASS], `</a></td>`);
403 stats.writefln(`<td><a href="`, std.path.join(log.id, "error.html"), `">`, log.counts[Result.ERROR], `</a></td>`);
404 stats.writefln(`</tr>`);
405 }
406
407 // differences between logs
408 foreach(int i, char[] file; args[2 .. $]){
409 char[] newid = std.path.getBaseName(args[2+i]);
410 char[] oldid = std.path.getBaseName(args[2+i-1]);
411
412 char[] dirname = std.path.join(basedir, oldid ~ "-to-" ~ newid);
413
414 if(std.file.exists(dirname)) {
415 if(std.file.isdir(dirname)) {
416 writefln("Directory ", dirname, " already exists, skipping...");
417 continue;
418 }
419 else
420 throw new Exception(dirname ~ " is not a directory!");
421 }
422 else
423 std.file.mkdir(dirname);
424
425 // parse etc.
426 Log newLog, oldLog;
427 Log getOrParse(char[] id, char[] file) {
428 if(id in logs)
429 return logs[id];
430 else {
431 Log tmp = new Log(id, file);
432 logs[id] = tmp;
433 return tmp;
434 }
435 }
436 newLog = getOrParse(newid, args[2+i]);
437 oldLog = getOrParse(oldid, args[2+i-1]);
438
439 int nRegressions, nImprovements, nChanges;
440 auto regressionsFile = new BufferedFile(std.path.join(dirname, "regressions.html"), FileMode.Out);
441 scope(exit) regressionsFile.close();
442 auto improvementsFile = new BufferedFile(std.path.join(dirname, "improvements.html"), FileMode.Out);
443 scope(exit) improvementsFile.close();
444 auto changesFile = new BufferedFile(std.path.join(dirname, "changes.html"), FileMode.Out);
445 scope(exit) changesFile.close();
446 BufferedFile targetFile;
447
448 foreach(Test t; newLog.tests.values){
449 Test* oldT = t.file in oldLog.tests;
450
451 if(oldT !is null){
452 if(oldT.r == t.r)
453 continue;
454 else if(oldT.r < t.r && oldT.r && oldT.r <= Result.XFAIL){
455 targetFile = regressionsFile;
456 nRegressions++;
457 }
458 else if(t.r < oldT.r && t.r && t.r <= Result.XFAIL){
459 targetFile = improvementsFile;
460 nImprovements++;
461 }
462 else {
463 targetFile = changesFile;
464 nChanges++;
465 }
466 targetFile.writefln(toString(oldT.r), " -> ", toString(t.r), " : ", t.name, " in ", t.file, "<br>");
467 }
468 }
469
470 BufferedFile stats = new BufferedFile(std.path.join(dirname, "stats.base"), FileMode.Out);
471 scope(exit) stats.close();
472 auto dir = oldid ~ "-to-" ~ newid;
473 stats.writefln(`<tr><td></td>`);
474 stats.writefln(`<td style="background-color:white;" colspan="5">`);
475 stats.writefln(`<a href="`, std.path.join(dir, "improvements.html"), `">Improvements: `, nImprovements, `</a>, `);
476 stats.writefln(`<a href="`, std.path.join(dir, "regressions.html"), `">Regressions: `, nRegressions, `</a>, `);
477 stats.writefln(`<a href="`, std.path.join(dir, "changes.html"), `">Changes: `, nChanges, `</a></td>`);
478 stats.writefln(`</tr>`);
479 }
480
481 // collect all the stats.base files into a large table
482 BufferedFile index = new BufferedFile(std.path.join(basedir, "index.html"), FileMode.Out);
483 scope(exit) index.close();
484 index.writefln(`
485 <table style="border-collapse:collapse; text-align:center;">
486 <colgroup>
487 <col style="border-right: medium solid black;">
488 <col style="background-color: #AAFFAA;">
489 <col style="background-color: #AAFFAA; border-right: thin solid black;">
490 <col style="background-color: #FFAAAA;">
491 <col style="background-color: #FFAAAA;">
492 <col style="background-color: #FFAAAA;">
493 </colgroup>
494 <tr style="border-bottom: medium solid black;">
495 <th>name</th>
496 <th style="padding-left:1em;padding-right:1em;">PASS</th>
497 <th style="padding-left:1em;padding-right:1em;">XFAIL</th>
498 <th style="padding-left:1em;padding-right:1em;">FAIL</th>
499 <th style="padding-left:1em;padding-right:1em;">XPASS</th>
500 <th style="padding-left:1em;padding-right:1em;">ERROR</th>
501 </tr>
502 `);
503
504 for(int i = args.length - 1; i >= 1; --i) {
505 auto file = args[i];
506 char[] id = std.path.getBaseName(file);
507 char[] statsname = std.path.join(std.path.join(basedir, id), "stats.base");
508 index.writef(cast(char[])std.file.read(statsname));
509
510 if(i == 1)
511 continue;
512
513 char[] newid = std.path.getBaseName(args[i-1]);
514 statsname = std.path.join(std.path.join(basedir, newid ~ "-to-" ~ id), "stats.base");
515 index.writef(cast(char[])std.file.read(statsname));
516 }
517
518 index.writefln(`</table>`);
519
520 return 0;
521 }
522