255
|
1 // Based on DSTRESS code by Thomas Kühne
|
|
2
|
|
3 module findregressions;
|
|
4
|
|
5 private import std.string;
|
|
6 private import std.conv;
|
|
7 private import std.stdio;
|
|
8 private import std.stream;
|
|
9 private import std.file;
|
|
10 private import std.c.stdlib;
|
|
11 private import std.date;
|
|
12 private import std.path;
|
|
13
|
|
14
|
|
15 enum Result{
|
|
16 UNTESTED = 0,
|
|
17 PASS = 1 << 2,
|
|
18 XFAIL = 2 << 2,
|
|
19 XPASS = 3 << 2,
|
|
20 FAIL = 4 << 2,
|
|
21 ERROR = 5 << 2,
|
|
22 BASE_MASK = 7 << 2,
|
|
23
|
|
24 EXT_MASK = 3,
|
|
25 BAD_MSG = 1,
|
|
26 BAD_GDB = 2,
|
|
27
|
|
28 MAX = BAD_GDB + BASE_MASK
|
|
29 }
|
|
30
|
|
31 char[] toString(Result r){
|
|
32 switch(r & Result.BASE_MASK){
|
|
33 case Result.PASS: return "PASS";
|
|
34 case Result.XPASS: return "XPASS";
|
|
35 case Result.FAIL: return "FAIL";
|
|
36 case Result.XFAIL: return "XFAIL";
|
|
37 case Result.ERROR: return "ERROR";
|
|
38 case Result.UNTESTED: return "UNTESTED";
|
|
39 default:
|
|
40 break;
|
|
41 }
|
|
42 throw new Exception(format("unhandled Result value %s", cast(int)r));
|
|
43 }
|
|
44
|
|
45 char[] dateString(){
|
|
46 static char[] date;
|
|
47 if(date is null){
|
|
48 auto time = getUTCtime();
|
|
49 auto year = YearFromTime(time);
|
|
50 auto month = MonthFromTime(time);
|
|
51 auto day = DateFromTime(time);
|
|
52 date = format("%d-%02d-%02d", year, month+1, day);
|
|
53 }
|
|
54 return date;
|
|
55 }
|
|
56
|
|
57 char[][] unique(char[][] a){
|
|
58 char[][] b = a.sort;
|
|
59 char[][] back;
|
|
60
|
|
61 back ~= b[0];
|
|
62
|
|
63 size_t ii=0;
|
|
64 for(size_t i=0; i<b.length; i++){
|
|
65 if(back[ii]!=b[i]){
|
|
66 back~=b[i];
|
|
67 ii++;
|
|
68 }
|
|
69 }
|
|
70
|
|
71 return back;
|
|
72 }
|
|
73
|
|
74 private{
|
|
75 version(Windows){
|
|
76 import std.c.windows.windows;
|
|
77 extern(Windows) BOOL GetFileTime(HANDLE hFile, LPFILETIME lpCreationTime, LPFILETIME lpLastAccessTime, LPFILETIME lpLastWriteTime);
|
|
78 }else version(linux){
|
|
79 import std.c.linux.linux;
|
|
80 version = Unix;
|
|
81 }else version(Unix){
|
|
82 import std.c.unix.unix;
|
|
83 }else{
|
|
84 static assert(0);
|
|
85 }
|
|
86
|
|
87 alias ulong FStime;
|
|
88
|
|
89 FStime getFStime(char[] fileName){
|
|
90 version(Windows){
|
|
91 HANDLE h;
|
|
92
|
|
93 if (useWfuncs){
|
|
94 wchar* namez = std.utf.toUTF16z(fileName);
|
|
95 h = CreateFileW(namez,GENERIC_WRITE,0,null,OPEN_ALWAYS,
|
|
96 FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,cast(HANDLE)null);
|
|
97 }else{
|
|
98 char* namez = toMBSz(fileName);
|
|
99 h = CreateFileA(namez,GENERIC_WRITE,0,null,OPEN_ALWAYS,
|
|
100 FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,cast(HANDLE)null);
|
|
101 }
|
|
102
|
|
103 if (h == INVALID_HANDLE_VALUE)
|
|
104 goto err;
|
|
105
|
|
106 FILETIME creationTime;
|
|
107 FILETIME accessTime;
|
|
108 FILETIME writeTime;
|
|
109
|
|
110 BOOL b = GetFileTime(h, &creationTime, &accessTime, &writeTime);
|
|
111 if(b==1){
|
|
112 long modA = writeTime.dwLowDateTime;
|
|
113 long modB = writeTime.dwHighDateTime;
|
|
114 return modA | (modB << (writeTime.dwHighDateTime.sizeof*8));
|
|
115 }
|
|
116
|
|
117 err:
|
|
118 CloseHandle(h);
|
|
119 throw new Exception("failed to query file modification : "~fileName);
|
|
120 }else version(Unix){
|
|
121 char* namez = toStringz(fileName);
|
|
122 struct_stat statbuf;
|
|
123
|
|
124 if(stat(namez, &statbuf)){
|
|
125 throw new FileException(fileName, getErrno());
|
|
126 }
|
|
127
|
|
128 return statbuf.st_mtime;
|
|
129 }else{
|
|
130 static assert(0);
|
|
131 }
|
|
132 }
|
|
133 }
|
|
134
|
|
135 char[] cleanFileName(char[] file){
|
|
136 char[] back;
|
|
137 bool hadSep;
|
|
138
|
|
139 foreach(char c; file){
|
|
140 if(c == '/' || c == '\\'){
|
|
141 if(!hadSep){
|
|
142 back ~= '/';
|
|
143 hadSep = true;
|
|
144 }
|
|
145 }else{
|
|
146 back ~= c;
|
|
147 hadSep = false;
|
|
148 }
|
|
149 }
|
|
150
|
|
151 size_t start = 0;
|
|
152 while(back[start] <= ' ' && start < back.length){
|
|
153 start++;
|
|
154 }
|
|
155
|
|
156 size_t end = back.length-1;
|
|
157 while(back[end] <= ' ' && end >= start){
|
|
158 end--;
|
|
159 }
|
|
160
|
|
161 back = back[start .. end+1];
|
|
162
|
|
163 return back;
|
|
164 }
|
|
165
|
|
166 class Test{
|
|
167 char[] name;
|
|
168 char[] file;
|
|
169 Result r;
|
|
170
|
|
171 this(char[] file){
|
|
172 this.file = file;
|
|
173
|
|
174 int start = rfind(file, "/");
|
|
175 if(start<0){
|
|
176 start = 0;
|
|
177 }else{
|
|
178 start += 1;
|
|
179 }
|
|
180
|
|
181 int end = rfind(file, ".");
|
|
182 if(end < start){
|
|
183 end = file.length;
|
|
184 }
|
|
185
|
|
186 name = file[start .. end];
|
|
187 }
|
|
188 }
|
|
189
|
|
190
|
|
191 class Log{
|
|
192 Test[char[]] tests;
|
|
193
|
|
194 char[] id;
|
|
195
|
|
196 int[Result] counts;
|
|
197
|
|
198 this(char[] id, char[] file){
|
|
199 this.id = id;
|
|
200 counts = [
|
|
201 Result.PASS: 0,
|
|
202 Result.FAIL: 0,
|
|
203 Result.XPASS: 0,
|
|
204 Result.XFAIL: 0,
|
|
205 Result.ERROR: 0 ];
|
|
206
|
|
207 writefln("parsing: %s", file);
|
|
208 FStime logTime = getFStime(file);
|
|
209 Stream source = new BufferedFile(file, FileMode.In);
|
|
210 while(!source.eof()){
|
|
211 add(source.readLine());
|
|
212 }
|
|
213 dropBogusResults(logTime, "dstress");
|
|
214 }
|
|
215
|
|
216
|
|
217 void dropBogusResults(FStime recordTime, char[] testRoot){
|
|
218 uint totalCount = tests.length;
|
|
219
|
|
220 char[][] sourcesTests = tests.keys;
|
|
221 foreach(char[] source; sourcesTests){
|
|
222 if(find(source, "complex/") < 0){
|
|
223 try{
|
|
224 FStime caseTime = getFStime(testRoot~std.path.sep~source);
|
|
225 if(caseTime > recordTime){
|
|
226 debug(drop) fwritefln(stderr, "dropped: %s", source);
|
|
227 counts[tests[source].r & Result.BASE_MASK]--;
|
|
228 tests.remove(source);
|
|
229 }
|
|
230 }catch(Exception e){
|
|
231 debug(drop) fwritefln(stderr, "dropped: %s", source);
|
|
232 counts[tests[source].r & Result.BASE_MASK]--;
|
|
233 tests.remove(source);
|
|
234 }
|
|
235 }
|
|
236 // asm-filter
|
|
237 int i = find(source, "asm_p");
|
|
238 if(i >= 0){
|
|
239 counts[tests[source].r & Result.BASE_MASK]--;
|
|
240 tests.remove(source);
|
|
241 }
|
|
242 }
|
|
243 tests.rehash;
|
|
244
|
|
245 writefln("dropped %s outdated tests (%s remaining)", totalCount - tests.length, tests.length);
|
|
246 }
|
|
247
|
|
248
|
|
249 bool add(char[] line){
|
|
250 const char[] SUB = "Torture-Sub-";
|
|
251 const char[] TORTURE = "Torture:";
|
|
252
|
|
253 line = strip(line);
|
|
254 int id = -1;
|
|
255 Result r = Result.UNTESTED;
|
|
256
|
|
257 if(line.length > SUB.length && line[0 .. SUB.length] == SUB){
|
|
258 line = line[SUB.length .. $];
|
|
259 id = 0;
|
|
260 while(line[id] >= '0' && line[id] <= '9'){
|
|
261 id++;
|
|
262 }
|
|
263 int start = id;
|
|
264 id = std.conv.toUint(line[0 .. id]);
|
|
265
|
|
266 while(line[start] != '-'){
|
|
267 start++;
|
|
268 }
|
|
269 line = line[start+1 .. $];
|
|
270 }
|
|
271
|
|
272 char[][] token = split(line);
|
|
273 if(token.length < 2){
|
|
274 return false;
|
|
275 }
|
|
276 char[] file = strip(token[1]);
|
|
277
|
|
278 switch(token[0]){
|
|
279 case "PASS:":
|
|
280 r = Result.PASS; break;
|
|
281 case "FAIL:":
|
|
282 r = Result.FAIL; break;
|
|
283 case "XPASS:":
|
|
284 r = Result.XPASS; break;
|
|
285 case "XFAIL:":
|
|
286 r = Result.XFAIL; break;
|
|
287 case "ERROR:":
|
|
288 r = Result.ERROR; break;
|
|
289 default:{
|
|
290 if(token[0] == TORTURE){
|
|
291 throw new Exception("not yet handled: "~line);
|
|
292 }else if(id > -1){
|
|
293 throw new Exception(format("bug in SUB line: (%s) %s", id, line));
|
|
294 }
|
|
295 }
|
|
296 }
|
|
297
|
|
298 if(r != Result.UNTESTED){
|
|
299 if(std.string.find(line, "bad error message") > -1){
|
|
300 r |= Result.BAD_MSG;
|
|
301 }
|
|
302 if(std.string.find(line, "bad debugger message") > -1){
|
|
303 r |= Result.BAD_MSG;
|
|
304 }
|
|
305
|
|
306 file = cleanFileName(file);
|
|
307
|
|
308 if(id >= 0){
|
|
309 // update sub
|
|
310 id--;
|
|
311
|
|
312 Test* test = file in tests;
|
|
313
|
|
314 if(test is null){
|
|
315 Test t = new Test(file);
|
|
316 tests[file] = t;
|
|
317 t.r = r;
|
|
318 counts[r & Result.BASE_MASK]++;
|
|
319 }else{
|
|
320 if(test.r != Result.UNTESTED){
|
|
321 test.r = Result.UNTESTED;
|
|
322 }
|
|
323 test.r = r;
|
|
324 }
|
|
325 }
|
|
326 return true;
|
|
327 }
|
|
328 return false;
|
|
329 }
|
|
330 }
|
|
331
|
|
332
|
|
333 int main(char[][] args){
|
|
334
|
|
335 if(args.length < 2){
|
|
336 fwritefln(stderr, "%s <log> <log> ...", args[0]);
|
|
337 fwritefln(stderr, "bash example: %s $(ls reference/llvmdc*)", args[0]);
|
|
338 return 1;
|
|
339 }
|
|
340
|
|
341 // make sure base path exists
|
|
342 char[] basedir = "web";
|
|
343 if(std.file.exists(basedir) && !std.file.isdir(basedir))
|
|
344 throw new Exception(basedir ~ " is not a directory!");
|
|
345 else if(!std.file.exists(basedir))
|
|
346 std.file.mkdir(basedir);
|
|
347
|
|
348
|
|
349 Log[char[]] logs;
|
|
350
|
|
351 // parse log and emit per-log data if necessary
|
|
352 foreach(char[] file; args[1 .. $]){
|
|
353 char[] id = std.path.getBaseName(file);
|
|
354 char[] dirname = std.path.join(basedir, id);
|
|
355
|
|
356 if(std.file.exists(dirname)) {
|
|
357 if(std.file.isdir(dirname)) {
|
|
358 writefln("Directory ", dirname, " already exists, skipping...");
|
|
359 continue;
|
|
360 }
|
|
361 else
|
|
362 throw new Exception(dirname ~ " is not a directory!");
|
|
363 }
|
|
364 else
|
|
365 std.file.mkdir(dirname);
|
|
366
|
|
367 // parse etc.
|
|
368 Log log = new Log(id, file);
|
|
369 logs[id] = log;
|
|
370
|
|
371 // write status
|
|
372 BufferedFile makeFile(char[] name) {
|
|
373 return new BufferedFile(std.path.join(dirname, name), FileMode.Out);
|
|
374 }
|
|
375 BufferedFile[Result] resultsfile = [
|
|
376 Result.PASS: makeFile("pass.html"),
|
|
377 Result.FAIL: makeFile("fail.html"),
|
|
378 Result.XPASS: makeFile("xpass.html"),
|
|
379 Result.XFAIL: makeFile("xfail.html"),
|
|
380 Result.ERROR: makeFile("error.html") ];
|
|
381
|
|
382 scope(exit) {
|
|
383 foreach(file; resultsfile)
|
|
384 file.close();
|
|
385 }
|
|
386
|
|
387
|
|
388 foreach(tkey; log.tests.keys.sort) {
|
|
389 auto test = log.tests[tkey];
|
|
390 auto result = test.r & Result.BASE_MASK;
|
|
391 resultsfile[result].writefln(test.name, " in ", test.file, "<br>");
|
|
392 }
|
|
393
|
|
394
|
|
395 BufferedFile stats = new BufferedFile(std.path.join(dirname, "stats.base"), FileMode.Out);
|
|
396 scope(exit) stats.close();
|
|
397 stats.writefln(`<tr>`);
|
|
398 stats.writefln(`<td style="padding-right:1em; text-align:left;">`, id, `</td>`);
|
|
399 stats.writefln(`<td><a href="`, std.path.join(log.id, "pass.html"), `">`, log.counts[Result.PASS], `</a></td>`);
|
|
400 stats.writefln(`<td><a href="`, std.path.join(log.id, "xfail.html"), `">`, log.counts[Result.XFAIL], `</a></td>`);
|
|
401 stats.writefln(`<td><a href="`, std.path.join(log.id, "fail.html"), `">`, log.counts[Result.FAIL], `</a></td>`);
|
|
402 stats.writefln(`<td><a href="`, std.path.join(log.id, "xpass.html"), `">`, log.counts[Result.XPASS], `</a></td>`);
|
|
403 stats.writefln(`<td><a href="`, std.path.join(log.id, "error.html"), `">`, log.counts[Result.ERROR], `</a></td>`);
|
|
404 stats.writefln(`</tr>`);
|
|
405 }
|
|
406
|
|
407 // differences between logs
|
|
408 foreach(int i, char[] file; args[2 .. $]){
|
|
409 char[] newid = std.path.getBaseName(args[2+i]);
|
|
410 char[] oldid = std.path.getBaseName(args[2+i-1]);
|
|
411
|
|
412 char[] dirname = std.path.join(basedir, oldid ~ "-to-" ~ newid);
|
|
413
|
|
414 if(std.file.exists(dirname)) {
|
|
415 if(std.file.isdir(dirname)) {
|
|
416 writefln("Directory ", dirname, " already exists, skipping...");
|
|
417 continue;
|
|
418 }
|
|
419 else
|
|
420 throw new Exception(dirname ~ " is not a directory!");
|
|
421 }
|
|
422 else
|
|
423 std.file.mkdir(dirname);
|
|
424
|
|
425 // parse etc.
|
|
426 Log newLog, oldLog;
|
|
427 Log getOrParse(char[] id, char[] file) {
|
|
428 if(id in logs)
|
|
429 return logs[id];
|
|
430 else {
|
|
431 Log tmp = new Log(id, file);
|
|
432 logs[id] = tmp;
|
|
433 return tmp;
|
|
434 }
|
|
435 }
|
|
436 newLog = getOrParse(newid, args[2+i]);
|
|
437 oldLog = getOrParse(oldid, args[2+i-1]);
|
|
438
|
|
439 int nRegressions, nImprovements, nChanges;
|
|
440 auto regressionsFile = new BufferedFile(std.path.join(dirname, "regressions.html"), FileMode.Out);
|
|
441 scope(exit) regressionsFile.close();
|
|
442 auto improvementsFile = new BufferedFile(std.path.join(dirname, "improvements.html"), FileMode.Out);
|
|
443 scope(exit) improvementsFile.close();
|
|
444 auto changesFile = new BufferedFile(std.path.join(dirname, "changes.html"), FileMode.Out);
|
|
445 scope(exit) changesFile.close();
|
|
446 BufferedFile targetFile;
|
|
447
|
|
448 foreach(Test t; newLog.tests.values){
|
|
449 Test* oldT = t.file in oldLog.tests;
|
|
450
|
|
451 if(oldT !is null){
|
|
452 if(oldT.r == t.r)
|
|
453 continue;
|
|
454 else if(oldT.r < t.r && oldT.r && oldT.r <= Result.XFAIL){
|
|
455 targetFile = regressionsFile;
|
|
456 nRegressions++;
|
|
457 }
|
|
458 else if(t.r < oldT.r && t.r && t.r <= Result.XFAIL){
|
|
459 targetFile = improvementsFile;
|
|
460 nImprovements++;
|
|
461 }
|
|
462 else {
|
|
463 targetFile = changesFile;
|
|
464 nChanges++;
|
|
465 }
|
|
466 targetFile.writefln(toString(oldT.r), " -> ", toString(t.r), " : ", t.name, " in ", t.file, "<br>");
|
|
467 }
|
|
468 }
|
|
469
|
|
470 BufferedFile stats = new BufferedFile(std.path.join(dirname, "stats.base"), FileMode.Out);
|
|
471 scope(exit) stats.close();
|
|
472 auto dir = oldid ~ "-to-" ~ newid;
|
|
473 stats.writefln(`<tr><td></td>`);
|
|
474 stats.writefln(`<td style="background-color:white;" colspan="5">`);
|
|
475 stats.writefln(`<a href="`, std.path.join(dir, "improvements.html"), `">Improvements: `, nImprovements, `</a>, `);
|
|
476 stats.writefln(`<a href="`, std.path.join(dir, "regressions.html"), `">Regressions: `, nRegressions, `</a>, `);
|
|
477 stats.writefln(`<a href="`, std.path.join(dir, "changes.html"), `">Changes: `, nChanges, `</a></td>`);
|
|
478 stats.writefln(`</tr>`);
|
|
479 }
|
|
480
|
|
481 // collect all the stats.base files into a large table
|
|
482 BufferedFile index = new BufferedFile(std.path.join(basedir, "index.html"), FileMode.Out);
|
|
483 scope(exit) index.close();
|
|
484 index.writefln(`
|
|
485 <table style="border-collapse:collapse; text-align:center;">
|
|
486 <colgroup>
|
|
487 <col style="border-right: medium solid black;">
|
|
488 <col style="background-color: #AAFFAA;">
|
|
489 <col style="background-color: #AAFFAA; border-right: thin solid black;">
|
|
490 <col style="background-color: #FFAAAA;">
|
|
491 <col style="background-color: #FFAAAA;">
|
|
492 <col style="background-color: #FFAAAA;">
|
|
493 </colgroup>
|
|
494 <tr style="border-bottom: medium solid black;">
|
|
495 <th>name</th>
|
|
496 <th style="padding-left:1em;padding-right:1em;">PASS</th>
|
|
497 <th style="padding-left:1em;padding-right:1em;">XFAIL</th>
|
|
498 <th style="padding-left:1em;padding-right:1em;">FAIL</th>
|
|
499 <th style="padding-left:1em;padding-right:1em;">XPASS</th>
|
|
500 <th style="padding-left:1em;padding-right:1em;">ERROR</th>
|
|
501 </tr>
|
|
502 `);
|
|
503
|
|
504 for(int i = args.length - 1; i >= 1; --i) {
|
|
505 auto file = args[i];
|
|
506 char[] id = std.path.getBaseName(file);
|
|
507 char[] statsname = std.path.join(std.path.join(basedir, id), "stats.base");
|
|
508 index.writef(cast(char[])std.file.read(statsname));
|
|
509
|
|
510 if(i == 1)
|
|
511 continue;
|
|
512
|
|
513 char[] newid = std.path.getBaseName(args[i-1]);
|
|
514 statsname = std.path.join(std.path.join(basedir, newid ~ "-to-" ~ id), "stats.base");
|
|
515 index.writef(cast(char[])std.file.read(statsname));
|
|
516 }
|
|
517
|
|
518 index.writefln(`</table>`);
|
|
519
|
|
520 return 0;
|
|
521 }
|
|
522
|