|
132
|
1 /*******************************************************************************
|
|
|
2
|
|
|
3 copyright: Copyright (C) 2007 Daniel Keep. All rights reserved.
|
|
|
4
|
|
|
5 license: BSD style: $(LICENSE)
|
|
|
6
|
|
|
7 version: Initial release: July 2007
|
|
|
8
|
|
|
9 author: Daniel Keep
|
|
|
10
|
|
|
11 history: Added support for "window bits", needed for Zip support.
|
|
|
12
|
|
|
13 *******************************************************************************/
|
|
|
14
|
|
|
15 module tango.io.compress.ZlibStream;
|
|
|
16
|
|
|
17 private import tango.io.compress.c.zlib;
|
|
|
18
|
|
|
19 private import tango.stdc.stringz : fromUtf8z;
|
|
|
20
|
|
|
21 private import tango.core.Exception : IOException;
|
|
|
22
|
|
|
23 private import tango.io.Conduit : InputFilter, OutputFilter;
|
|
|
24
|
|
|
25 private import tango.io.model.IConduit : InputStream, OutputStream, IConduit;
|
|
|
26
|
|
|
27
|
|
|
28 /* This constant controls the size of the input/output buffers we use
|
|
|
29 * internally. This should be a fairly sane value (it's suggested by the zlib
|
|
|
30 * documentation), that should only need changing for memory-constrained
|
|
|
31 * platforms/use cases.
|
|
|
32 *
|
|
|
33 * An alternative would be to make the chunk size a template parameter to the
|
|
|
34 * filters themselves, but Tango already has more than enough template
|
|
|
35 * parameters getting in the way :)
|
|
|
36 */
|
|
|
37
|
|
|
38 private const CHUNKSIZE = 256 * 1024;
|
|
|
39
|
|
|
40 /*******************************************************************************
|
|
|
41
|
|
|
42 This input filter can be used to perform decompression of zlib streams.
|
|
|
43
|
|
|
44 *******************************************************************************/
|
|
|
45
|
|
|
46 class ZlibInput : InputFilter
|
|
|
47 {
|
|
|
48 private
|
|
|
49 {
|
|
|
50 /* Used to make sure we don't try to perform operations on a dead
|
|
|
51 * stream. */
|
|
|
52 bool zs_valid = false;
|
|
|
53
|
|
|
54 z_stream zs;
|
|
|
55 ubyte[] in_chunk;
|
|
|
56 }
|
|
|
57
|
|
|
58 /***************************************************************************
|
|
|
59
|
|
|
60 Constructs a new zlib decompression filter. You need to pass in the
|
|
|
61 stream that the decompression filter will read from. If you are using
|
|
|
62 this filter with a conduit, the idiom to use is:
|
|
|
63
|
|
|
64 ---
|
|
|
65 auto input = new ZlibInput(myConduit.input));
|
|
|
66 input.read(myContent);
|
|
|
67 ---
|
|
|
68
|
|
|
69 The optional windowBits parameter is the base two logarithm of the
|
|
|
70 window size, and should be in the range 8-15, defaulting to 15 if not
|
|
|
71 specified. Additionally, the windowBits parameter may be negative to
|
|
|
72 indicate that zlib should omit the standard zlib header and trailer,
|
|
|
73 with the window size being -windowBits.
|
|
|
74
|
|
|
75 ***************************************************************************/
|
|
|
76
|
|
|
77 this(InputStream stream)
|
|
|
78 {
|
|
|
79 super (stream);
|
|
|
80 in_chunk = new ubyte[CHUNKSIZE];
|
|
|
81
|
|
|
82 // Allocate inflate state
|
|
|
83 with( zs )
|
|
|
84 {
|
|
|
85 zalloc = null;
|
|
|
86 zfree = null;
|
|
|
87 opaque = null;
|
|
|
88 avail_in = 0;
|
|
|
89 next_in = null;
|
|
|
90 }
|
|
|
91
|
|
|
92 auto ret = inflateInit(&zs);
|
|
|
93 if( ret != Z_OK )
|
|
|
94 throw new ZlibException(ret);
|
|
|
95
|
|
|
96 zs_valid = true;
|
|
|
97 }
|
|
|
98
|
|
|
99 /// ditto
|
|
|
100 this(InputStream stream, int windowBits)
|
|
|
101 {
|
|
|
102 super (stream);
|
|
|
103 in_chunk = new ubyte[CHUNKSIZE];
|
|
|
104
|
|
|
105 // Allocate inflate state
|
|
|
106 with( zs )
|
|
|
107 {
|
|
|
108 zalloc = null;
|
|
|
109 zfree = null;
|
|
|
110 opaque = null;
|
|
|
111 avail_in = 0;
|
|
|
112 next_in = null;
|
|
|
113 }
|
|
|
114
|
|
|
115 auto ret = inflateInit2(&zs, windowBits);
|
|
|
116 if( ret != Z_OK )
|
|
|
117 throw new ZlibException(ret);
|
|
|
118
|
|
|
119 zs_valid = true;
|
|
|
120 }
|
|
|
121
|
|
|
122 ~this()
|
|
|
123 {
|
|
|
124 if( zs_valid )
|
|
|
125 kill_zs();
|
|
|
126 }
|
|
|
127
|
|
|
128 /***************************************************************************
|
|
|
129
|
|
|
130 Decompresses data from the underlying conduit into a target array.
|
|
|
131
|
|
|
132 Returns the number of bytes stored into dst, which may be less than
|
|
|
133 requested.
|
|
|
134
|
|
|
135 ***************************************************************************/
|
|
|
136
|
|
|
137 override uint read(void[] dst)
|
|
|
138 {
|
|
|
139 if( !zs_valid )
|
|
|
140 return IConduit.Eof;
|
|
|
141
|
|
|
142 // Check to see if we've run out of input data. If we have, get some
|
|
|
143 // more.
|
|
|
144 if( zs.avail_in == 0 )
|
|
|
145 {
|
|
|
146 auto len = host.read(in_chunk);
|
|
|
147 if( len == IConduit.Eof )
|
|
|
148 return IConduit.Eof;
|
|
|
149
|
|
|
150 zs.avail_in = len;
|
|
|
151 zs.next_in = in_chunk.ptr;
|
|
|
152 }
|
|
|
153
|
|
|
154 // We'll tell zlib to inflate straight into the target array.
|
|
|
155 zs.avail_out = dst.length;
|
|
|
156 zs.next_out = cast(ubyte*)dst.ptr;
|
|
|
157 auto ret = inflate(&zs, Z_NO_FLUSH);
|
|
|
158
|
|
|
159 switch( ret )
|
|
|
160 {
|
|
|
161 case Z_NEED_DICT:
|
|
|
162 // Whilst not technically an error, this should never happen
|
|
|
163 // for general-use code, so treat it as an error.
|
|
|
164 case Z_DATA_ERROR:
|
|
|
165 case Z_MEM_ERROR:
|
|
|
166 kill_zs();
|
|
|
167 throw new ZlibException(ret);
|
|
|
168
|
|
|
169 case Z_STREAM_END:
|
|
|
170 // zlib stream is finished; kill the stream so we don't try to
|
|
|
171 // read from it again.
|
|
|
172 kill_zs();
|
|
|
173 break;
|
|
|
174
|
|
|
175 default:
|
|
|
176 }
|
|
|
177
|
|
|
178 return dst.length - zs.avail_out;
|
|
|
179 }
|
|
|
180
|
|
|
181 /***************************************************************************
|
|
|
182
|
|
|
183 Clear any buffered content. No-op.
|
|
|
184
|
|
|
185 ***************************************************************************/
|
|
|
186
|
|
|
187 override InputStream clear()
|
|
|
188 {
|
|
|
189 check_valid();
|
|
|
190
|
|
|
191 // TODO: What should this method do? We don't do any heap allocation,
|
|
|
192 // so there's really nothing to clear... For now, just invalidate the
|
|
|
193 // stream...
|
|
|
194 kill_zs();
|
|
|
195
|
|
|
196 super.clear();
|
|
|
197 return this;
|
|
|
198 }
|
|
|
199
|
|
|
200 // This function kills the stream: it deallocates the internal state, and
|
|
|
201 // unsets the zs_valid flag.
|
|
|
202 private void kill_zs()
|
|
|
203 {
|
|
|
204 check_valid();
|
|
|
205
|
|
|
206 inflateEnd(&zs);
|
|
|
207 zs_valid = false;
|
|
|
208 }
|
|
|
209
|
|
|
210 // Asserts that the stream is still valid and usable (except that this
|
|
|
211 // check doesn't get elided with -release).
|
|
|
212 private void check_valid()
|
|
|
213 {
|
|
|
214 if( !zs_valid )
|
|
|
215 throw new ZlibClosedException;
|
|
|
216 }
|
|
|
217 }
|
|
|
218
|
|
|
219 /*******************************************************************************
|
|
|
220
|
|
|
221 This output filter can be used to perform compression of data into a zlib
|
|
|
222 stream.
|
|
|
223
|
|
|
224 *******************************************************************************/
|
|
|
225
|
|
|
226 class ZlibOutput : OutputFilter
|
|
|
227 {
|
|
|
228 /***************************************************************************
|
|
|
229
|
|
|
230 This enumeration represents several pre-defined compression levels.
|
|
|
231
|
|
|
232 None instructs zlib to perform no compression whatsoever, and simply
|
|
|
233 store the data stream. Note that this actually expands the stream
|
|
|
234 slightly to accommodate the zlib stream metadata.
|
|
|
235
|
|
|
236 Fast instructs zlib to perform a minimal amount of compression, Best
|
|
|
237 indicates that you want the maximum level of compression and Normal
|
|
|
238 (the default level) is a compromise between the two. The exact
|
|
|
239 compression level Normal represents is determined by the underlying
|
|
|
240 zlib library, but is typically level 6.
|
|
|
241
|
|
|
242 Any integer between -1 and 9 inclusive may be used as a level,
|
|
|
243 although the symbols in this enumeration should suffice for most
|
|
|
244 use-cases.
|
|
|
245
|
|
|
246 ***************************************************************************/
|
|
|
247
|
|
|
248 enum Level : int
|
|
|
249 {
|
|
|
250 Normal = -1,
|
|
|
251 None = 0,
|
|
|
252 Fast = 1,
|
|
|
253 Best = 9
|
|
|
254 }
|
|
|
255
|
|
|
256 private
|
|
|
257 {
|
|
|
258 bool zs_valid = false;
|
|
|
259 z_stream zs;
|
|
|
260 ubyte[] out_chunk;
|
|
|
261 size_t _written = 0;
|
|
|
262 }
|
|
|
263
|
|
|
264 /***************************************************************************
|
|
|
265
|
|
|
266 Constructs a new zlib compression filter. You need to pass in the
|
|
|
267 stream that the compression filter will write to. If you are using
|
|
|
268 this filter with a conduit, the idiom to use is:
|
|
|
269
|
|
|
270 ---
|
|
|
271 auto output = new ZlibOutput(myConduit.output);
|
|
|
272 output.write(myContent);
|
|
|
273 ---
|
|
|
274
|
|
|
275 The optional windowBits parameter is the base two logarithm of the
|
|
|
276 window size, and should be in the range 8-15, defaulting to 15 if not
|
|
|
277 specified. Additionally, the windowBits parameter may be negative to
|
|
|
278 indicate that zlib should omit the standard zlib header and trailer,
|
|
|
279 with the window size being -windowBits.
|
|
|
280
|
|
|
281 ***************************************************************************/
|
|
|
282
|
|
|
283 this(OutputStream stream, Level level = Level.Normal)
|
|
|
284 {
|
|
|
285 super(stream);
|
|
|
286 out_chunk = new ubyte[CHUNKSIZE];
|
|
|
287
|
|
|
288 // Allocate deflate state
|
|
|
289 with( zs )
|
|
|
290 {
|
|
|
291 zalloc = null;
|
|
|
292 zfree = null;
|
|
|
293 opaque = null;
|
|
|
294 }
|
|
|
295
|
|
|
296 auto ret = deflateInit(&zs, level);
|
|
|
297 if( ret != Z_OK )
|
|
|
298 throw new ZlibException(ret);
|
|
|
299
|
|
|
300 zs_valid = true;
|
|
|
301 }
|
|
|
302
|
|
|
303 /// ditto
|
|
|
304 this(OutputStream stream, Level level, int windowBits)
|
|
|
305 {
|
|
|
306 super(stream);
|
|
|
307 out_chunk = new ubyte[CHUNKSIZE];
|
|
|
308
|
|
|
309 // Allocate deflate state
|
|
|
310 with( zs )
|
|
|
311 {
|
|
|
312 zalloc = null;
|
|
|
313 zfree = null;
|
|
|
314 opaque = null;
|
|
|
315 }
|
|
|
316
|
|
|
317 auto ret = deflateInit2(&zs, level, Z_DEFLATED, windowBits, 8,
|
|
|
318 Z_DEFAULT_STRATEGY);
|
|
|
319 if( ret != Z_OK )
|
|
|
320 throw new ZlibException(ret);
|
|
|
321
|
|
|
322 zs_valid = true;
|
|
|
323 }
|
|
|
324
|
|
|
325 ~this()
|
|
|
326 {
|
|
|
327 if( zs_valid )
|
|
|
328 kill_zs();
|
|
|
329 }
|
|
|
330
|
|
|
331 /***************************************************************************
|
|
|
332
|
|
|
333 Compresses the given data to the underlying conduit.
|
|
|
334
|
|
|
335 Returns the number of bytes from src that were compressed; write
|
|
|
336 should always consume all data provided to it, although it may not be
|
|
|
337 immediately written to the underlying output stream.
|
|
|
338
|
|
|
339 ***************************************************************************/
|
|
|
340
|
|
|
341 override uint write(void[] src)
|
|
|
342 {
|
|
|
343 check_valid();
|
|
|
344 scope(failure) kill_zs();
|
|
|
345
|
|
|
346 zs.avail_in = src.length;
|
|
|
347 zs.next_in = cast(ubyte*)src.ptr;
|
|
|
348
|
|
|
349 do
|
|
|
350 {
|
|
|
351 zs.avail_out = out_chunk.length;
|
|
|
352 zs.next_out = out_chunk.ptr;
|
|
|
353
|
|
|
354 auto ret = deflate(&zs, Z_NO_FLUSH);
|
|
|
355 if( ret == Z_STREAM_ERROR )
|
|
|
356 throw new ZlibException(ret);
|
|
|
357
|
|
|
358 // Push the compressed bytes out to the stream, until it's either
|
|
|
359 // written them all, or choked.
|
|
|
360 auto have = out_chunk.length-zs.avail_out;
|
|
|
361 auto out_buffer = out_chunk[0..have];
|
|
|
362 do
|
|
|
363 {
|
|
|
364 auto w = host.write(out_buffer);
|
|
|
365 if( w == IConduit.Eof )
|
|
|
366 return w;
|
|
|
367
|
|
|
368 out_buffer = out_buffer[w..$];
|
|
|
369 _written += w;
|
|
|
370 }
|
|
|
371 while( out_buffer.length > 0 );
|
|
|
372 }
|
|
|
373 // Loop while we are still using up the whole output buffer
|
|
|
374 while( zs.avail_out == 0 );
|
|
|
375
|
|
|
376 assert( zs.avail_in == 0, "failed to compress all provided data" );
|
|
|
377
|
|
|
378 return src.length;
|
|
|
379 }
|
|
|
380
|
|
|
381 /***************************************************************************
|
|
|
382
|
|
|
383 This read-only property returns the number of compressed bytes that
|
|
|
384 have been written to the underlying stream. Following a call to
|
|
|
385 either close or commit, this will contain the total compressed size of
|
|
|
386 the input data stream.
|
|
|
387
|
|
|
388 ***************************************************************************/
|
|
|
389
|
|
|
390 size_t written()
|
|
|
391 {
|
|
|
392 return _written;
|
|
|
393 }
|
|
|
394
|
|
|
395 /***************************************************************************
|
|
|
396
|
|
|
397 commit the output
|
|
|
398
|
|
|
399 ***************************************************************************/
|
|
|
400
|
|
|
401 override void close()
|
|
|
402 {
|
|
|
403 // Only commit if the stream is still open.
|
|
|
404 if( zs_valid ) commit;
|
|
|
405
|
|
|
406 super.close;
|
|
|
407 }
|
|
|
408
|
|
|
409 /***************************************************************************
|
|
|
410
|
|
|
411 Purge any buffered content. Calling this will implicitly end the zlib
|
|
|
412 stream, so it should not be called until you are finished compressing
|
|
|
413 data. Any calls to either write or commit after a compression filter
|
|
|
414 has been committed will throw an exception.
|
|
|
415
|
|
|
416 ***************************************************************************/
|
|
|
417
|
|
|
418 void commit()
|
|
|
419 {
|
|
|
420 check_valid();
|
|
|
421 scope(failure) kill_zs();
|
|
|
422
|
|
|
423 zs.avail_in = 0;
|
|
|
424 zs.next_in = null;
|
|
|
425
|
|
|
426 bool finished = false;
|
|
|
427
|
|
|
428 do
|
|
|
429 {
|
|
|
430 zs.avail_out = out_chunk.length;
|
|
|
431 zs.next_out = out_chunk.ptr;
|
|
|
432
|
|
|
433 auto ret = deflate(&zs, Z_FINISH);
|
|
|
434 switch( ret )
|
|
|
435 {
|
|
|
436 case Z_OK:
|
|
|
437 // Keep going
|
|
|
438 break;
|
|
|
439
|
|
|
440 case Z_STREAM_END:
|
|
|
441 // We're done!
|
|
|
442 finished = true;
|
|
|
443 break;
|
|
|
444
|
|
|
445 default:
|
|
|
446 throw new ZlibException(ret);
|
|
|
447 }
|
|
|
448
|
|
|
449 auto have = out_chunk.length - zs.avail_out;
|
|
|
450 auto out_buffer = out_chunk[0..have];
|
|
|
451 if( have > 0 )
|
|
|
452 {
|
|
|
453 do
|
|
|
454 {
|
|
|
455 auto w = host.write(out_buffer);
|
|
|
456 if( w == IConduit.Eof )
|
|
|
457 return w;
|
|
|
458
|
|
|
459 out_buffer = out_buffer[w..$];
|
|
|
460 _written += w;
|
|
|
461 }
|
|
|
462 while( out_buffer.length > 0 );
|
|
|
463 }
|
|
|
464 }
|
|
|
465 while( !finished );
|
|
|
466
|
|
|
467 kill_zs();
|
|
|
468 }
|
|
|
469
|
|
|
470 // This function kills the stream: it deallocates the internal state, and
|
|
|
471 // unsets the zs_valid flag.
|
|
|
472 private void kill_zs()
|
|
|
473 {
|
|
|
474 check_valid();
|
|
|
475
|
|
|
476 deflateEnd(&zs);
|
|
|
477 zs_valid = false;
|
|
|
478 }
|
|
|
479
|
|
|
480 // Asserts that the stream is still valid and usable (except that this
|
|
|
481 // check doesn't get elided with -release).
|
|
|
482 private void check_valid()
|
|
|
483 {
|
|
|
484 if( !zs_valid )
|
|
|
485 throw new ZlibClosedException;
|
|
|
486 }
|
|
|
487 }
|
|
|
488
|
|
|
489 /*******************************************************************************
|
|
|
490
|
|
|
491 This exception is thrown if you attempt to perform a read, write or flush
|
|
|
492 operation on a closed zlib filter stream. This can occur if the input
|
|
|
493 stream has finished, or an output stream was flushed.
|
|
|
494
|
|
|
495 *******************************************************************************/
|
|
|
496
|
|
|
497 class ZlibClosedException : IOException
|
|
|
498 {
|
|
|
499 this()
|
|
|
500 {
|
|
|
501 super("cannot operate on closed zlib stream");
|
|
|
502 }
|
|
|
503 }
|
|
|
504
|
|
|
505 /*******************************************************************************
|
|
|
506
|
|
|
507 This exception is thrown when an error occurs in the underlying zlib
|
|
|
508 library. Where possible, it will indicate both the name of the error, and
|
|
|
509 any textural message zlib has provided.
|
|
|
510
|
|
|
511 *******************************************************************************/
|
|
|
512
|
|
|
513 class ZlibException : IOException
|
|
|
514 {
|
|
|
515 this(int code)
|
|
|
516 {
|
|
|
517 super(codeName(code));
|
|
|
518 }
|
|
|
519
|
|
|
520 this(int code, char* msg)
|
|
|
521 {
|
|
|
522 super(codeName(code)~": "~fromUtf8z(msg));
|
|
|
523 }
|
|
|
524
|
|
|
525 protected char[] codeName(int code)
|
|
|
526 {
|
|
|
527 char[] name;
|
|
|
528
|
|
|
529 switch( code )
|
|
|
530 {
|
|
|
531 case Z_OK: name = "Z_OK"; break;
|
|
|
532 case Z_STREAM_END: name = "Z_STREAM_END"; break;
|
|
|
533 case Z_NEED_DICT: name = "Z_NEED_DICT"; break;
|
|
|
534 case Z_ERRNO: name = "Z_ERRNO"; break;
|
|
|
535 case Z_STREAM_ERROR: name = "Z_STREAM_ERROR"; break;
|
|
|
536 case Z_DATA_ERROR: name = "Z_DATA_ERROR"; break;
|
|
|
537 case Z_MEM_ERROR: name = "Z_MEM_ERROR"; break;
|
|
|
538 case Z_BUF_ERROR: name = "Z_BUF_ERROR"; break;
|
|
|
539 case Z_VERSION_ERROR: name = "Z_VERSION_ERROR"; break;
|
|
|
540 default: name = "Z_UNKNOWN";
|
|
|
541 }
|
|
|
542
|
|
|
543 return name;
|
|
|
544 }
|
|
|
545 }
|
|
|
546
|
|
|
547 /* *****************************************************************************
|
|
|
548
|
|
|
549 This section contains a simple unit test for this module. It is hidden
|
|
|
550 behind a version statement because it introduces additional dependencies.
|
|
|
551
|
|
|
552 ***************************************************************************** */
|
|
|
553
|
|
|
554 debug(UnitTest) {
|
|
|
555
|
|
|
556 import tango.io.GrowBuffer : GrowBuffer;
|
|
|
557
|
|
|
558 unittest
|
|
|
559 {
|
|
|
560 // One ring to rule them all, one ring to find them,
|
|
|
561 // One ring to bring them all and in the darkness bind them.
|
|
|
562 const char[] message =
|
|
|
563 "Ash nazg durbatulûk, ash nazg gimbatul, "
|
|
|
564 "ash nazg thrakatulûk, agh burzum-ishi krimpatul.";
|
|
|
565
|
|
|
566 // This compressed data was created using Python 2.5's built in zlib
|
|
|
567 // module, with the default compression level.
|
|
|
568 const ubyte[] message_z = [
|
|
|
569 0x78,0x9c,0x73,0x2c,0xce,0x50,0xc8,0x4b,
|
|
|
570 0xac,0x4a,0x57,0x48,0x29,0x2d,0x4a,0x4a,
|
|
|
571 0x2c,0x29,0xcd,0x39,0xbc,0x3b,0x5b,0x47,
|
|
|
572 0x21,0x11,0x26,0x9a,0x9e,0x99,0x0b,0x16,
|
|
|
573 0x45,0x12,0x2a,0xc9,0x28,0x4a,0xcc,0x46,
|
|
|
574 0xa8,0x4c,0xcf,0x50,0x48,0x2a,0x2d,0xaa,
|
|
|
575 0x2a,0xcd,0xd5,0xcd,0x2c,0xce,0xc8,0x54,
|
|
|
576 0xc8,0x2e,0xca,0xcc,0x2d,0x00,0xc9,0xea,
|
|
|
577 0x01,0x00,0x1f,0xe3,0x22,0x99];
|
|
|
578
|
|
|
579 scope cond_z = new GrowBuffer;
|
|
|
580 scope comp = new ZlibOutput(cond_z);
|
|
|
581 comp.write (message);
|
|
|
582 comp.close;
|
|
|
583
|
|
|
584 assert( comp.written == message_z.length );
|
|
|
585
|
|
|
586 assert( message_z == cast(ubyte[])(cond_z.slice) );
|
|
|
587
|
|
|
588 scope decomp = new ZlibInput(cond_z);
|
|
|
589 auto buffer = new ubyte[256];
|
|
|
590 buffer = buffer[0 .. decomp.read(buffer)];
|
|
|
591
|
|
|
592 assert( cast(ubyte[])message == buffer );
|
|
|
593 }
|
|
|
594 }
|
