Mercurial > projects > ldc

comparison tests/makewebstatistics.d @ 266:5d2f4814bb2e trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[svn r287] update to test statistics generation
author ChristianK
date Sun, 15 Jun 2008 12:58:48 +0200
parents 5723b7385c25
children c9d5c711d65a
comparison
equal deleted inserted replaced
265:455eb10696f2 266:5d2f4814bb2e
328 return false; 328 return false;
329 } 329 }
330 } 330 }
331 331
332 332
333 char[] basedir = "web";
334 bool regenerate = false;
335
333 int main(char[][] args){ 336 int main(char[][] args){
334 337
335 if(args.length < 2){ 338 if(args.length < 3 || (args[1] == "--regenerate" && args.length < 4)){
336 fwritefln(stderr, "%s [--regenerate] <log> <log> ...", args[0]); 339 fwritefln(stderr, "%s [--regenerate] <reference-log> <log> <log> ...", args[0]);
337 fwritefln(stderr, "bash example: %s $(ls reference/llvmdc*)", args[0]); 340 fwritefln(stderr, "bash example: %s reference/dmd-something $(ls reference/llvmdc*)", args[0]);
338 return 1; 341 return 1;
339 } 342 }
340 343
341 bool regenerate = false; 344 char[] reference;
342 char[][] files; 345 char[][] files;
343 if(args[1] == "--regenerate") { 346 if(args[1] == "--regenerate") {
344 regenerate = true; 347 regenerate = true;
345 files = args[2..$]; 348 reference = args[2];
349 files = args[3..$] ~ reference;
346 } else { 350 } else {
347 files = args[1..$]; 351 reference = args[1];
352 files = args[2..$] ~ reference;
348 } 353 }
349 354
350 // make sure base path exists 355 // make sure base path exists
351 char[] basedir = "web";
352 if(std.file.exists(basedir) && !std.file.isdir(basedir)) 356 if(std.file.exists(basedir) && !std.file.isdir(basedir))
353 throw new Exception(basedir ~ " is not a directory!"); 357 throw new Exception(basedir ~ " is not a directory!");
354 else if(!std.file.exists(basedir)) 358 else if(!std.file.exists(basedir))
355 std.file.mkdir(basedir); 359 std.file.mkdir(basedir);
356 360
357 361
358 Log[char[]] logs; 362 Log[char[]] logs;
359 363
360 // parse log and emit per-log data if necessary 364 // emit per-log data
361 foreach(char[] file; files){ 365 foreach(char[] file; files)
366 generateLogStatistics(file, logs);
367
368 // differences between logs
369 foreach(int i, char[] file; files[1 .. $])
370 generateChangeStatistics(files[1+i], files[1+i-1], logs);
371
372 // differences between reference and logs
373 foreach(char[] file; files[0..$-1])
374 generateChangeStatistics(file, reference, logs);
375
376 // collect all the stats.base files into a large table
377 BufferedFile index = new BufferedFile(std.path.join(basedir, "index.html"), FileMode.OutNew);
378 scope(exit) index.close();
379 index.writefln(`
380 <html><body>
381 <table style="border-collapse:collapse; text-align:center;">
382 <colgroup>
383 <col style="border-right: medium solid black;">
384 <col style="background-color: #AAFFAA;">
385 <col style="background-color: #AAFFAA; border-right: thin solid black;">
386 <col style="background-color: #FFAAAA;">
387 <col style="background-color: #FFAAAA;">
388 <col style="background-color: #FFAAAA;">
389 <col style="border-left: medium solid black;">
390 </colgroup>
391 <tr style="border-bottom: medium solid black;">
392 <th>name</th>
393 <th style="padding-left:1em;padding-right:1em;">PASS</th>
394 <th style="padding-left:1em;padding-right:1em;">XFAIL</th>
395 <th style="padding-left:1em;padding-right:1em;">FAIL</th>
396 <th style="padding-left:1em;padding-right:1em;">XPASS</th>
397 <th style="padding-left:1em;padding-right:1em;">ERROR</th>
398 <th style="padding-left:1em;padding-right:1em;">comparison to ` ~ std.path.getBaseName(reference) ~ `</th>
399 </tr>
400 `);
401
402 for(int i = files.length - 1; i >= 0; --i) {
403 auto file = files[i];
404 index.writefln(`<tr>`);
362 char[] id = std.path.getBaseName(file); 405 char[] id = std.path.getBaseName(file);
363 char[] dirname = std.path.join(basedir, id); 406 char[] statsname = std.path.join(std.path.join(basedir, id), "stats.base");
364 407 index.writef(cast(char[])std.file.read(statsname));
365 if(std.file.exists(dirname)) { 408
366 if(std.file.isdir(dirname)) { 409 if(i != files.length - 1) {
367 if(!regenerate) { 410 index.writefln(`<td>`);
368 writefln("Directory ", dirname, " already exists, skipping..."); 411 char[] refid = std.path.getBaseName(reference);
369 continue; 412 statsname = std.path.join(std.path.join(basedir, refid ~ "-to-" ~ id), "stats.base");
370 } 413 index.writef(cast(char[])std.file.read(statsname));
371 } 414 index.writefln(`</td></tr>`);
372 else 415 } else {
373 throw new Exception(dirname ~ " is not a directory!"); 416 index.writefln(`<td></td></tr>`);
417 }
418
419 if(i == 0) {
420 continue;
421 }
422
423 index.writefln(`<tr><td></td>`);
424 index.writefln(`<td style="background-color:white;" colspan="5">`);
425 char[] newid = std.path.getBaseName(files[i-1]);
426 statsname = std.path.join(std.path.join(basedir, newid ~ "-to-" ~ id), "stats.base");
427 index.writef(cast(char[])std.file.read(statsname));
428 index.writefln(`</td><td></td></tr>`);
429 }
430
431 index.writefln(`</table></body></html>`);
432
433 return 0;
434 }
435
436 void generateLogStatistics(char[] file, ref Log[char[]] logs)
437 {
438 char[] id = std.path.getBaseName(file);
439 char[] dirname = std.path.join(basedir, id);
440
441 if(std.file.exists(dirname)) {
442 if(std.file.isdir(dirname)) {
443 if(!regenerate) {
444 writefln("Directory ", dirname, " already exists, skipping...");
445 return;
446 }
374 } 447 }
375 else 448 else
376 std.file.mkdir(dirname); 449 throw new Exception(dirname ~ " is not a directory!");
377 450 }
378 // parse etc. 451 else
379 Log log = new Log(id, file); 452 std.file.mkdir(dirname);
380 logs[id] = log; 453
381 454 // parse etc.
382 // write status 455 Log log = new Log(id, file);
456 logs[id] = log;
457
458 // write status
459 {
383 BufferedFile makeFile(char[] name) { 460 BufferedFile makeFile(char[] name) {
384 return new BufferedFile(std.path.join(dirname, name), FileMode.OutNew); 461 return new BufferedFile(std.path.join(dirname, name), FileMode.OutNew);
385 } 462 }
386 BufferedFile[Result] resultsfile = [ 463 BufferedFile[Result] resultsfile = [
387 Result.PASS: makeFile("pass.html"), 464 Result.PASS: makeFile("pass.html"),
388 Result.FAIL: makeFile("fail.html"), 465 Result.FAIL: makeFile("fail.html"),
389 Result.XPASS: makeFile("xpass.html"), 466 Result.XPASS: makeFile("xpass.html"),
390 Result.XFAIL: makeFile("xfail.html"), 467 Result.XFAIL: makeFile("xfail.html"),
391 Result.ERROR: makeFile("error.html") ]; 468 Result.ERROR: makeFile("error.html") ];
392 469
393 scope(exit) { 470 scope(exit) {
394 foreach(file; resultsfile) 471 foreach(file; resultsfile)
395 file.close(); 472 file.close();
396 } 473 }
397 474
398 475 foreach(file; resultsfile)
476 file.writefln(`<html><body>`);
477
399 foreach(tkey; log.tests.keys.sort) { 478 foreach(tkey; log.tests.keys.sort) {
400 auto test = log.tests[tkey]; 479 auto test = log.tests[tkey];
401 auto result = test.r & Result.BASE_MASK; 480 auto result = test.r & Result.BASE_MASK;
402 resultsfile[result].writefln(test.name, " in ", test.file); 481 resultsfile[result].writefln(test.name, " in ", test.file, "<br>");
403 } 482 }
404 483
405 484 foreach(file; resultsfile)
406 BufferedFile stats = new BufferedFile(std.path.join(dirname, "stats.base"), FileMode.OutNew); 485 file.writefln(`</body></html>`);
407 scope(exit) stats.close(); 486 }
408 stats.writefln(`<tr>`); 487
409 stats.writefln(`<td style="padding-right:1em; text-align:left;">`, id, `</td>`); 488 BufferedFile stats = new BufferedFile(std.path.join(dirname, "stats.base"), FileMode.OutNew);
410 stats.writefln(`<td><a href="`, std.path.join(log.id, "pass.html"), `">`, log.counts[Result.PASS], `</a></td>`); 489 scope(exit) stats.close();
411 stats.writefln(`<td><a href="`, std.path.join(log.id, "xfail.html"), `">`, log.counts[Result.XFAIL], `</a></td>`); 490 stats.writefln(`<td style="padding-right:1em; text-align:left;">`, id, `</td>`);
412 stats.writefln(`<td><a href="`, std.path.join(log.id, "fail.html"), `">`, log.counts[Result.FAIL], `</a></td>`); 491 stats.writefln(`<td><a href="`, std.path.join(log.id, "pass.html"), `">`, log.counts[Result.PASS], `</a></td>`);
413 stats.writefln(`<td><a href="`, std.path.join(log.id, "xpass.html"), `">`, log.counts[Result.XPASS], `</a></td>`); 492 stats.writefln(`<td><a href="`, std.path.join(log.id, "xfail.html"), `">`, log.counts[Result.XFAIL], `</a></td>`);
414 stats.writefln(`<td><a href="`, std.path.join(log.id, "error.html"), `">`, log.counts[Result.ERROR], `</a></td>`); 493 stats.writefln(`<td><a href="`, std.path.join(log.id, "fail.html"), `">`, log.counts[Result.FAIL], `</a></td>`);
415 stats.writefln(`</tr>`); 494 stats.writefln(`<td><a href="`, std.path.join(log.id, "xpass.html"), `">`, log.counts[Result.XPASS], `</a></td>`);
416 } 495 stats.writefln(`<td><a href="`, std.path.join(log.id, "error.html"), `">`, log.counts[Result.ERROR], `</a></td>`);
417 496 }
418 // differences between logs 497
419 foreach(int i, char[] file; files[1 .. $]){ 498 void generateChangeStatistics(char[] file1, char[] file2, ref Log[char[]] logs)
420 char[] newid = std.path.getBaseName(files[1+i]); 499 {
421 char[] oldid = std.path.getBaseName(files[1+i-1]); 500 char[] newid = std.path.getBaseName(file1);
422 501 char[] oldid = std.path.getBaseName(file2);
423 char[] dirname = std.path.join(basedir, oldid ~ "-to-" ~ newid); 502
424 503 char[] dirname = std.path.join(basedir, oldid ~ "-to-" ~ newid);
425 if(std.file.exists(dirname)) { 504
426 if(std.file.isdir(dirname)) { 505 if(std.file.exists(dirname)) {
427 if(!regenerate) { 506 if(std.file.isdir(dirname)) {
428 writefln("Directory ", dirname, " already exists, skipping..."); 507 if(!regenerate) {
429 continue; 508 writefln("Directory ", dirname, " already exists, skipping...");
430 } 509 return;
431 } 510 }
432 else
433 throw new Exception(dirname ~ " is not a directory!");
434 } 511 }
435 else 512 else
436 std.file.mkdir(dirname); 513 throw new Exception(dirname ~ " is not a directory!");
437 514 }
438 // parse etc. 515 else
439 Log newLog, oldLog; 516 std.file.mkdir(dirname);
440 Log getOrParse(char[] id, char[] file) { 517
441 if(id in logs) 518 // parse etc.
442 return logs[id]; 519 Log newLog, oldLog;
443 else { 520 Log getOrParse(char[] id, char[] file) {
444 Log tmp = new Log(id, file); 521 if(id in logs)
445 logs[id] = tmp; 522 return logs[id];
446 return tmp; 523 else {
447 } 524 Log tmp = new Log(id, file);
448 } 525 logs[id] = tmp;
449 newLog = getOrParse(newid, files[1+i]); 526 return tmp;
450 oldLog = getOrParse(oldid, files[1+i-1]); 527 }
451 528 }
452 int nRegressions, nImprovements, nChanges; 529 newLog = getOrParse(newid, file1);
530 oldLog = getOrParse(oldid, file2);
531
532 int nRegressions, nImprovements, nChanges;
533
534 {
453 auto regressionsFile = new BufferedFile(std.path.join(dirname, "regressions.html"), FileMode.OutNew); 535 auto regressionsFile = new BufferedFile(std.path.join(dirname, "regressions.html"), FileMode.OutNew);
454 scope(exit) regressionsFile.close(); 536 scope(exit) regressionsFile.close();
537 regressionsFile.writefln(`<html><body>`);
538
455 auto improvementsFile = new BufferedFile(std.path.join(dirname, "improvements.html"), FileMode.OutNew); 539 auto improvementsFile = new BufferedFile(std.path.join(dirname, "improvements.html"), FileMode.OutNew);
456 scope(exit) improvementsFile.close(); 540 scope(exit) improvementsFile.close();
541 improvementsFile.writefln(`<html><body>`);
542
457 auto changesFile = new BufferedFile(std.path.join(dirname, "changes.html"), FileMode.OutNew); 543 auto changesFile = new BufferedFile(std.path.join(dirname, "changes.html"), FileMode.OutNew);
458 scope(exit) changesFile.close(); 544 scope(exit) changesFile.close();
545 changesFile.writefln(`<html><body>`);
546
459 BufferedFile targetFile; 547 BufferedFile targetFile;
460 548
461 foreach(Test t; newLog.tests.values){ 549 foreach(Test t; newLog.tests.values){
462 Test* oldT = t.file in oldLog.tests; 550 Test* oldT = t.file in oldLog.tests;
463 551
464 if(oldT !is null){ 552 if(oldT !is null){
465 if(oldT.r == t.r) 553 if(oldT.r == t.r)
474 } 562 }
475 else { 563 else {
476 targetFile = changesFile; 564 targetFile = changesFile;
477 nChanges++; 565 nChanges++;
478 } 566 }
479 targetFile.writefln(toString(oldT.r), " -> ", toString(t.r), " : ", t.name, " in ", t.file); 567 targetFile.writefln(toString(oldT.r), " -> ", toString(t.r), " : ", t.name, " in ", t.file, "<br>");
480 } 568 }
481 } 569 }
482 570
483 BufferedFile stats = new BufferedFile(std.path.join(dirname, "stats.base"), FileMode.OutNew); 571 regressionsFile.writefln(`</body></html>`);
484 scope(exit) stats.close(); 572 improvementsFile.writefln(`</body></html>`);
485 auto dir = oldid ~ "-to-" ~ newid; 573 changesFile.writefln(`</body></html>`);
486 stats.writefln(`<tr><td></td>`); 574 }
487 stats.writefln(`<td style="background-color:white;" colspan="5">`); 575
488 stats.writefln(`<a href="`, std.path.join(dir, "improvements.html"), `">Improvements: `, nImprovements, `</a>, `); 576 BufferedFile stats = new BufferedFile(std.path.join(dirname, "stats.base"), FileMode.OutNew);
489 stats.writefln(`<a href="`, std.path.join(dir, "regressions.html"), `">Regressions: `, nRegressions, `</a>, `); 577 scope(exit) stats.close();
490 stats.writefln(`<a href="`, std.path.join(dir, "changes.html"), `">Changes: `, nChanges, `</a></td>`); 578 auto dir = oldid ~ "-to-" ~ newid;
491 stats.writefln(`</tr>`); 579 stats.writefln(`<a href="`, std.path.join(dir, "improvements.html"), `">Improvements: `, nImprovements, `</a>, `);
492 } 580 stats.writefln(`<a href="`, std.path.join(dir, "regressions.html"), `">Regressions: `, nRegressions, `</a>, `);
493 581 stats.writefln(`<a href="`, std.path.join(dir, "changes.html"), `">Changes: `, nChanges, `</a>`);
494 // collect all the stats.base files into a large table 582 }
495 BufferedFile index = new BufferedFile(std.path.join(basedir, "index.html"), FileMode.OutNew);
496 scope(exit) index.close();
497 index.writefln(`
498 <html><body>
499 <table style="border-collapse:collapse; text-align:center;">
500 <colgroup>
501 <col style="border-right: medium solid black;">
502 <col style="background-color: #AAFFAA;">
503 <col style="background-color: #AAFFAA; border-right: thin solid black;">
504 <col style="background-color: #FFAAAA;">
505 <col style="background-color: #FFAAAA;">
506 <col style="background-color: #FFAAAA;">
507 </colgroup>
508 <tr style="border-bottom: medium solid black;">
509 <th>name</th>
510 <th style="padding-left:1em;padding-right:1em;">PASS</th>
511 <th style="padding-left:1em;padding-right:1em;">XFAIL</th>
512 <th style="padding-left:1em;padding-right:1em;">FAIL</th>
513 <th style="padding-left:1em;padding-right:1em;">XPASS</th>
514 <th style="padding-left:1em;padding-right:1em;">ERROR</th>
515 </tr>
516 `);
517
518 for(int i = files.length - 1; i >= 0; --i) {
519 auto file = files[i];
520 char[] id = std.path.getBaseName(file);
521 char[] statsname = std.path.join(std.path.join(basedir, id), "stats.base");
522 index.writef(cast(char[])std.file.read(statsname));
523
524 if(i == 0)
525 continue;
526
527 char[] newid = std.path.getBaseName(files[i-1]);
528 statsname = std.path.join(std.path.join(basedir, newid ~ "-to-" ~ id), "stats.base");
529 index.writef(cast(char[])std.file.read(statsname));
530 }
531
532 index.writefln(`</table></body></html>`);
533
534 return 0;
535 }
536