Mercurial > projects > ldc

comparison tests/findregressions.d @ 186:395223f9875e trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[svn r202] added start of dstress-based test suite to tests/
author ChristianK
date Thu, 08 May 2008 22:32:22 +0200
parents
children 648409a7fb0c
comparison
equal deleted inserted replaced
185:89e21eeaf4c4 186:395223f9875e
1 // Based on DSTRESS code by Thomas Kühne
2
3 module findregressions;
4
5 private import std.string;
6 private import std.conv;
7 private import std.stdio;
8 private import std.stream;
9 private import std.file;
10 private import std.c.stdlib;
11 private import std.date;
12
13
14 enum Result{
15 UNTESTED = 0,
16 PASS = 1 << 2,
17 XFAIL = 2 << 2,
18 XPASS = 3 << 2,
19 FAIL = 4 << 2,
20 ERROR = 5 << 2,
21 BASE_MASK = 7 << 2,
22
23 EXT_MASK = 3,
24 BAD_MSG = 1,
25 BAD_GDB = 2,
26
27 MAX = BAD_GDB + BASE_MASK
28 }
29
30 char[] toString(Result r){
31 switch(r & Result.BASE_MASK){
32 case Result.PASS: return "PASS";
33 case Result.XPASS: return "XPASS";
34 case Result.FAIL: return "FAIL";
35 case Result.XFAIL: return "XFAIL";
36 case Result.ERROR: return "ERROR";
37 case Result.UNTESTED: return "UNTESTED";
38 default:
39 break;
40 }
41 throw new Exception(format("unhandled Result value %s", cast(int)r));
42 }
43
44 char[] dateString(){
45 static char[] date;
46 if(date is null){
47 auto time = getUTCtime();
48 auto year = YearFromTime(time);
49 auto month = MonthFromTime(time);
50 auto day = DateFromTime(time);
51 date = format("%d-%02d-%02d", year, month+1, day);
52 }
53 return date;
54 }
55
56 char[][] unique(char[][] a){
57 char[][] b = a.sort;
58 char[][] back;
59
60 back ~= b[0];
61
62 size_t ii=0;
63 for(size_t i=0; i<b.length; i++){
64 if(back[ii]!=b[i]){
65 back~=b[i];
66 ii++;
67 }
68 }
69
70 return back;
71 }
72
73 private{
74 version(Windows){
75 import std.c.windows.windows;
76 extern(Windows) BOOL GetFileTime(HANDLE hFile, LPFILETIME lpCreationTime, LPFILETIME lpLastAccessTime, LPFILETIME lpLastWriteTime);
77 }else version(linux){
78 import std.c.linux.linux;
79 version = Unix;
80 }else version(Unix){
81 import std.c.unix.unix;
82 }else{
83 static assert(0);
84 }
85
86 alias ulong FStime;
87
88 FStime getFStime(char[] fileName){
89 version(Windows){
90 HANDLE h;
91
92 if (useWfuncs){
93 wchar* namez = std.utf.toUTF16z(fileName);
94 h = CreateFileW(namez,GENERIC_WRITE,0,null,OPEN_ALWAYS,
95 FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,cast(HANDLE)null);
96 }else{
97 char* namez = toMBSz(fileName);
98 h = CreateFileA(namez,GENERIC_WRITE,0,null,OPEN_ALWAYS,
99 FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,cast(HANDLE)null);
100 }
101
102 if (h == INVALID_HANDLE_VALUE)
103 goto err;
104
105 FILETIME creationTime;
106 FILETIME accessTime;
107 FILETIME writeTime;
108
109 BOOL b = GetFileTime(h, &creationTime, &accessTime, &writeTime);
110 if(b==1){
111 long modA = writeTime.dwLowDateTime;
112 long modB = writeTime.dwHighDateTime;
113 return modA | (modB << (writeTime.dwHighDateTime.sizeof*8));
114 }
115
116 err:
117 CloseHandle(h);
118 throw new Exception("failed to query file modification : "~fileName);
119 }else version(Unix){
120 char* namez = toStringz(fileName);
121 struct_stat statbuf;
122
123 if(stat(namez, &statbuf)){
124 throw new FileException(fileName, getErrno());
125 }
126
127 return statbuf.st_mtime;
128 }else{
129 static assert(0);
130 }
131 }
132 }
133
134 char[] cleanFileName(char[] file){
135 char[] back;
136 bool hadSep;
137
138 foreach(char c; file){
139 if(c == '/' || c == '\\'){
140 if(!hadSep){
141 back ~= '/';
142 hadSep = true;
143 }
144 }else{
145 back ~= c;
146 hadSep = false;
147 }
148 }
149
150 size_t start = 0;
151 while(back[start] <= ' ' && start < back.length){
152 start++;
153 }
154
155 size_t end = back.length-1;
156 while(back[end] <= ' ' && end >= start){
157 end--;
158 }
159
160 back = back[start .. end+1];
161
162 return back;
163 }
164
165 class Test{
166 char[] name;
167 char[] file;
168 Result r;
169
170 this(char[] file){
171 this.file = file;
172
173 int start = rfind(file, "/");
174 if(start<0){
175 start = 0;
176 }else{
177 start += 1;
178 }
179
180 int end = rfind(file, ".");
181 if(end < start){
182 end = file.length;
183 }
184
185 name = file[start .. end];
186 }
187 }
188
189
190 class Log{
191 Test[char[]] tests;
192
193 char[] id;
194
195 this(char[] id){
196 this.id = id;
197 }
198
199
200 void dropBogusResults(FStime recordTime, char[] testRoot){
201 uint totalCount = tests.length;
202
203 char[][] sourcesTests = tests.keys;
204 foreach(char[] source; sourcesTests){
205 if(find(source, "complex/") < 0){
206 try{
207 FStime caseTime = getFStime(testRoot~std.path.sep~source);
208 if(caseTime > recordTime){
209 debug(drop) fwritefln(stderr, "dropped: %s", source);
210 tests.remove(source);
211 }
212 }catch(Exception e){
213 debug(drop) fwritefln(stderr, "dropped: %s", source);
214 tests.remove(source);
215 }
216 }
217 // asm-filter
218 int i = find(source, "asm_p");
219 if(i >= 0){
220 tests.remove(source);
221 }
222 }
223 tests.rehash;
224
225 writefln("dropped %s outdated tests (%s remaining)", totalCount - tests.length, tests.length);
226 }
227
228
229 bool add(char[] line){
230 const char[] SUB = "Torture-Sub-";
231 const char[] TORTURE = "Torture:";
232
233 line = strip(line);
234 int id = -1;
235 Result r = Result.UNTESTED;
236
237 if(line.length > SUB.length && line[0 .. SUB.length] == SUB){
238 line = line[SUB.length .. $];
239 id = 0;
240 while(line[id] >= '0' && line[id] <= '9'){
241 id++;
242 }
243 int start = id;
244 id = std.conv.toUint(line[0 .. id]);
245
246 while(line[start] != '-'){
247 start++;
248 }
249 line = line[start+1 .. $];
250 }
251
252 char[][] token = split(line);
253 if(token.length < 2){
254 return false;
255 }
256 char[] file = strip(token[1]);
257
258 switch(token[0]){
259 case "PASS:":
260 r = Result.PASS; break;
261 case "FAIL:":
262 r = Result.FAIL; break;
263 case "XPASS:":
264 r = Result.XPASS; break;
265 case "XFAIL:":
266 r = Result.XFAIL; break;
267 case "ERROR:":
268 r = Result.ERROR; break;
269 default:{
270 if(token[0] == TORTURE){
271 throw new Exception("not yet handled: "~line);
272 }else if(id > -1){
273 throw new Exception(format("bug in SUB line: (%s) %s", id, line));
274 }
275 }
276 }
277
278 if(r != Result.UNTESTED){
279 if(std.string.find(line, "bad error message") > -1){
280 r |= Result.BAD_MSG;
281 }
282 if(std.string.find(line, "bad debugger message") > -1){
283 r |= Result.BAD_MSG;
284 }
285
286 file = cleanFileName(file);
287
288 if(id >= 0){
289 // update sub
290 id--;
291
292 Test* test = file in tests;
293
294 if(test is null){
295 Test t = new Test(file);
296 tests[file] = t;
297 t.r = r;
298 }else{
299 if(test.r != Result.UNTESTED){
300 test.r = Result.UNTESTED;
301 }
302 test.r = r;
303 }
304 }
305 return true;
306 }
307 return false;
308 }
309 }
310
311
312 int main(char[][] args){
313
314 if(args.length < 2){
315 fwritefln(stderr, "%s <old log> <new log>", args[0]);
316 return 1;
317 }
318
319
320 Log[] logs;
321
322 foreach(size_t id, char[] file; args[1 .. $]){
323 writefln("parsing: %s", file);
324 FStime logTime = getFStime(file);
325 debug fwritefln(stderr, "sourceTime: %s", logTime);
326
327 Log l= new Log(file);
328 Stream source = new BufferedFile(file, FileMode.In);
329 while(!source.eof()){
330 l.add(source.readLine());
331 }
332
333 l.dropBogusResults(logTime, "dstress");
334
335 logs ~= l;
336 }
337
338 Log oldLog = logs[0];
339 Log newLog = logs[1];
340
341 foreach(Test t; newLog.tests.values){
342 Test* oldT = t.file in oldLog.tests;
343
344 if(oldT !is null){
345 if(oldT.r == t.r)
346 continue;
347 else if(oldT.r < t.r && oldT.r && oldT.r <= Result.XFAIL){
348 writef("Regression ");
349 }
350 else if(t.r < oldT.r && t.r && t.r <= Result.XFAIL){
351 writef("Improvement ");
352 }
353 else {
354 writef("Change ");
355 }
356 writefln(toString(oldT.r), " -> ", toString(t.r), " : ", t.name, " in ", t.file);
357 }
358 }
359
360
361 return 0;
362 }
363