Mercurial > projects > ldc

comparison tango/tango/io/compress/ZlibStream.d @ 132:1700239cab2e trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[svn r136] MAJOR UNSTABLE UPDATE!!! Initial commit after moving to Tango instead of Phobos. Lots of bugfixes... This build is not suitable for most things.
author lindquist
date Fri, 11 Jan 2008 17:57:40 +0100
parents
children
comparison
equal deleted inserted replaced
131:5825d48b27d1 132:1700239cab2e
1 /*******************************************************************************
2
3 copyright: Copyright (C) 2007 Daniel Keep. All rights reserved.
4
5 license: BSD style: $(LICENSE)
6
7 version: Initial release: July 2007
8
9 author: Daniel Keep
10
11 history: Added support for "window bits", needed for Zip support.
12
13 *******************************************************************************/
14
15 module tango.io.compress.ZlibStream;
16
17 private import tango.io.compress.c.zlib;
18
19 private import tango.stdc.stringz : fromUtf8z;
20
21 private import tango.core.Exception : IOException;
22
23 private import tango.io.Conduit : InputFilter, OutputFilter;
24
25 private import tango.io.model.IConduit : InputStream, OutputStream, IConduit;
26
27
28 /* This constant controls the size of the input/output buffers we use
29 * internally. This should be a fairly sane value (it's suggested by the zlib
30 * documentation), that should only need changing for memory-constrained
31 * platforms/use cases.
32 *
33 * An alternative would be to make the chunk size a template parameter to the
34 * filters themselves, but Tango already has more than enough template
35 * parameters getting in the way :)
36 */
37
38 private const CHUNKSIZE = 256 * 1024;
39
40 /*******************************************************************************
41
42 This input filter can be used to perform decompression of zlib streams.
43
44 *******************************************************************************/
45
46 class ZlibInput : InputFilter
47 {
48 private
49 {
50 /* Used to make sure we don't try to perform operations on a dead
51 * stream. */
52 bool zs_valid = false;
53
54 z_stream zs;
55 ubyte[] in_chunk;
56 }
57
58 /***************************************************************************
59
60 Constructs a new zlib decompression filter. You need to pass in the
61 stream that the decompression filter will read from. If you are using
62 this filter with a conduit, the idiom to use is:
63
64 ---
65 auto input = new ZlibInput(myConduit.input));
66 input.read(myContent);
67 ---
68
69 The optional windowBits parameter is the base two logarithm of the
70 window size, and should be in the range 8-15, defaulting to 15 if not
71 specified. Additionally, the windowBits parameter may be negative to
72 indicate that zlib should omit the standard zlib header and trailer,
73 with the window size being -windowBits.
74
75 ***************************************************************************/
76
77 this(InputStream stream)
78 {
79 super (stream);
80 in_chunk = new ubyte[CHUNKSIZE];
81
82 // Allocate inflate state
83 with( zs )
84 {
85 zalloc = null;
86 zfree = null;
87 opaque = null;
88 avail_in = 0;
89 next_in = null;
90 }
91
92 auto ret = inflateInit(&zs);
93 if( ret != Z_OK )
94 throw new ZlibException(ret);
95
96 zs_valid = true;
97 }
98
99 /// ditto
100 this(InputStream stream, int windowBits)
101 {
102 super (stream);
103 in_chunk = new ubyte[CHUNKSIZE];
104
105 // Allocate inflate state
106 with( zs )
107 {
108 zalloc = null;
109 zfree = null;
110 opaque = null;
111 avail_in = 0;
112 next_in = null;
113 }
114
115 auto ret = inflateInit2(&zs, windowBits);
116 if( ret != Z_OK )
117 throw new ZlibException(ret);
118
119 zs_valid = true;
120 }
121
122 ~this()
123 {
124 if( zs_valid )
125 kill_zs();
126 }
127
128 /***************************************************************************
129
130 Decompresses data from the underlying conduit into a target array.
131
132 Returns the number of bytes stored into dst, which may be less than
133 requested.
134
135 ***************************************************************************/
136
137 override uint read(void[] dst)
138 {
139 if( !zs_valid )
140 return IConduit.Eof;
141
142 // Check to see if we've run out of input data. If we have, get some
143 // more.
144 if( zs.avail_in == 0 )
145 {
146 auto len = host.read(in_chunk);
147 if( len == IConduit.Eof )
148 return IConduit.Eof;
149
150 zs.avail_in = len;
151 zs.next_in = in_chunk.ptr;
152 }
153
154 // We'll tell zlib to inflate straight into the target array.
155 zs.avail_out = dst.length;
156 zs.next_out = cast(ubyte*)dst.ptr;
157 auto ret = inflate(&zs, Z_NO_FLUSH);
158
159 switch( ret )
160 {
161 case Z_NEED_DICT:
162 // Whilst not technically an error, this should never happen
163 // for general-use code, so treat it as an error.
164 case Z_DATA_ERROR:
165 case Z_MEM_ERROR:
166 kill_zs();
167 throw new ZlibException(ret);
168
169 case Z_STREAM_END:
170 // zlib stream is finished; kill the stream so we don't try to
171 // read from it again.
172 kill_zs();
173 break;
174
175 default:
176 }
177
178 return dst.length - zs.avail_out;
179 }
180
181 /***************************************************************************
182
183 Clear any buffered content. No-op.
184
185 ***************************************************************************/
186
187 override InputStream clear()
188 {
189 check_valid();
190
191 // TODO: What should this method do? We don't do any heap allocation,
192 // so there's really nothing to clear... For now, just invalidate the
193 // stream...
194 kill_zs();
195
196 super.clear();
197 return this;
198 }
199
200 // This function kills the stream: it deallocates the internal state, and
201 // unsets the zs_valid flag.
202 private void kill_zs()
203 {
204 check_valid();
205
206 inflateEnd(&zs);
207 zs_valid = false;
208 }
209
210 // Asserts that the stream is still valid and usable (except that this
211 // check doesn't get elided with -release).
212 private void check_valid()
213 {
214 if( !zs_valid )
215 throw new ZlibClosedException;
216 }
217 }
218
219 /*******************************************************************************
220
221 This output filter can be used to perform compression of data into a zlib
222 stream.
223
224 *******************************************************************************/
225
226 class ZlibOutput : OutputFilter
227 {
228 /***************************************************************************
229
230 This enumeration represents several pre-defined compression levels.
231
232 None instructs zlib to perform no compression whatsoever, and simply
233 store the data stream. Note that this actually expands the stream
234 slightly to accommodate the zlib stream metadata.
235
236 Fast instructs zlib to perform a minimal amount of compression, Best
237 indicates that you want the maximum level of compression and Normal
238 (the default level) is a compromise between the two. The exact
239 compression level Normal represents is determined by the underlying
240 zlib library, but is typically level 6.
241
242 Any integer between -1 and 9 inclusive may be used as a level,
243 although the symbols in this enumeration should suffice for most
244 use-cases.
245
246 ***************************************************************************/
247
248 enum Level : int
249 {
250 Normal = -1,
251 None = 0,
252 Fast = 1,
253 Best = 9
254 }
255
256 private
257 {
258 bool zs_valid = false;
259 z_stream zs;
260 ubyte[] out_chunk;
261 size_t _written = 0;
262 }
263
264 /***************************************************************************
265
266 Constructs a new zlib compression filter. You need to pass in the
267 stream that the compression filter will write to. If you are using
268 this filter with a conduit, the idiom to use is:
269
270 ---
271 auto output = new ZlibOutput(myConduit.output);
272 output.write(myContent);
273 ---
274
275 The optional windowBits parameter is the base two logarithm of the
276 window size, and should be in the range 8-15, defaulting to 15 if not
277 specified. Additionally, the windowBits parameter may be negative to
278 indicate that zlib should omit the standard zlib header and trailer,
279 with the window size being -windowBits.
280
281 ***************************************************************************/
282
283 this(OutputStream stream, Level level = Level.Normal)
284 {
285 super(stream);
286 out_chunk = new ubyte[CHUNKSIZE];
287
288 // Allocate deflate state
289 with( zs )
290 {
291 zalloc = null;
292 zfree = null;
293 opaque = null;
294 }
295
296 auto ret = deflateInit(&zs, level);
297 if( ret != Z_OK )
298 throw new ZlibException(ret);
299
300 zs_valid = true;
301 }
302
303 /// ditto
304 this(OutputStream stream, Level level, int windowBits)
305 {
306 super(stream);
307 out_chunk = new ubyte[CHUNKSIZE];
308
309 // Allocate deflate state
310 with( zs )
311 {
312 zalloc = null;
313 zfree = null;
314 opaque = null;
315 }
316
317 auto ret = deflateInit2(&zs, level, Z_DEFLATED, windowBits, 8,
318 Z_DEFAULT_STRATEGY);
319 if( ret != Z_OK )
320 throw new ZlibException(ret);
321
322 zs_valid = true;
323 }
324
325 ~this()
326 {
327 if( zs_valid )
328 kill_zs();
329 }
330
331 /***************************************************************************
332
333 Compresses the given data to the underlying conduit.
334
335 Returns the number of bytes from src that were compressed; write
336 should always consume all data provided to it, although it may not be
337 immediately written to the underlying output stream.
338
339 ***************************************************************************/
340
341 override uint write(void[] src)
342 {
343 check_valid();
344 scope(failure) kill_zs();
345
346 zs.avail_in = src.length;
347 zs.next_in = cast(ubyte*)src.ptr;
348
349 do
350 {
351 zs.avail_out = out_chunk.length;
352 zs.next_out = out_chunk.ptr;
353
354 auto ret = deflate(&zs, Z_NO_FLUSH);
355 if( ret == Z_STREAM_ERROR )
356 throw new ZlibException(ret);
357
358 // Push the compressed bytes out to the stream, until it's either
359 // written them all, or choked.
360 auto have = out_chunk.length-zs.avail_out;
361 auto out_buffer = out_chunk[0..have];
362 do
363 {
364 auto w = host.write(out_buffer);
365 if( w == IConduit.Eof )
366 return w;
367
368 out_buffer = out_buffer[w..$];
369 _written += w;
370 }
371 while( out_buffer.length > 0 );
372 }
373 // Loop while we are still using up the whole output buffer
374 while( zs.avail_out == 0 );
375
376 assert( zs.avail_in == 0, "failed to compress all provided data" );
377
378 return src.length;
379 }
380
381 /***************************************************************************
382
383 This read-only property returns the number of compressed bytes that
384 have been written to the underlying stream. Following a call to
385 either close or commit, this will contain the total compressed size of
386 the input data stream.
387
388 ***************************************************************************/
389
390 size_t written()
391 {
392 return _written;
393 }
394
395 /***************************************************************************
396
397 commit the output
398
399 ***************************************************************************/
400
401 override void close()
402 {
403 // Only commit if the stream is still open.
404 if( zs_valid ) commit;
405
406 super.close;
407 }
408
409 /***************************************************************************
410
411 Purge any buffered content. Calling this will implicitly end the zlib
412 stream, so it should not be called until you are finished compressing
413 data. Any calls to either write or commit after a compression filter
414 has been committed will throw an exception.
415
416 ***************************************************************************/
417
418 void commit()
419 {
420 check_valid();
421 scope(failure) kill_zs();
422
423 zs.avail_in = 0;
424 zs.next_in = null;
425
426 bool finished = false;
427
428 do
429 {
430 zs.avail_out = out_chunk.length;
431 zs.next_out = out_chunk.ptr;
432
433 auto ret = deflate(&zs, Z_FINISH);
434 switch( ret )
435 {
436 case Z_OK:
437 // Keep going
438 break;
439
440 case Z_STREAM_END:
441 // We're done!
442 finished = true;
443 break;
444
445 default:
446 throw new ZlibException(ret);
447 }
448
449 auto have = out_chunk.length - zs.avail_out;
450 auto out_buffer = out_chunk[0..have];
451 if( have > 0 )
452 {
453 do
454 {
455 auto w = host.write(out_buffer);
456 if( w == IConduit.Eof )
457 return w;
458
459 out_buffer = out_buffer[w..$];
460 _written += w;
461 }
462 while( out_buffer.length > 0 );
463 }
464 }
465 while( !finished );
466
467 kill_zs();
468 }
469
470 // This function kills the stream: it deallocates the internal state, and
471 // unsets the zs_valid flag.
472 private void kill_zs()
473 {
474 check_valid();
475
476 deflateEnd(&zs);
477 zs_valid = false;
478 }
479
480 // Asserts that the stream is still valid and usable (except that this
481 // check doesn't get elided with -release).
482 private void check_valid()
483 {
484 if( !zs_valid )
485 throw new ZlibClosedException;
486 }
487 }
488
489 /*******************************************************************************
490
491 This exception is thrown if you attempt to perform a read, write or flush
492 operation on a closed zlib filter stream. This can occur if the input
493 stream has finished, or an output stream was flushed.
494
495 *******************************************************************************/
496
497 class ZlibClosedException : IOException
498 {
499 this()
500 {
501 super("cannot operate on closed zlib stream");
502 }
503 }
504
505 /*******************************************************************************
506
507 This exception is thrown when an error occurs in the underlying zlib
508 library. Where possible, it will indicate both the name of the error, and
509 any textural message zlib has provided.
510
511 *******************************************************************************/
512
513 class ZlibException : IOException
514 {
515 this(int code)
516 {
517 super(codeName(code));
518 }
519
520 this(int code, char* msg)
521 {
522 super(codeName(code)~": "~fromUtf8z(msg));
523 }
524
525 protected char[] codeName(int code)
526 {
527 char[] name;
528
529 switch( code )
530 {
531 case Z_OK: name = "Z_OK"; break;
532 case Z_STREAM_END: name = "Z_STREAM_END"; break;
533 case Z_NEED_DICT: name = "Z_NEED_DICT"; break;
534 case Z_ERRNO: name = "Z_ERRNO"; break;
535 case Z_STREAM_ERROR: name = "Z_STREAM_ERROR"; break;
536 case Z_DATA_ERROR: name = "Z_DATA_ERROR"; break;
537 case Z_MEM_ERROR: name = "Z_MEM_ERROR"; break;
538 case Z_BUF_ERROR: name = "Z_BUF_ERROR"; break;
539 case Z_VERSION_ERROR: name = "Z_VERSION_ERROR"; break;
540 default: name = "Z_UNKNOWN";
541 }
542
543 return name;
544 }
545 }
546
547 /* *****************************************************************************
548
549 This section contains a simple unit test for this module. It is hidden
550 behind a version statement because it introduces additional dependencies.
551
552 ***************************************************************************** */
553
554 debug(UnitTest) {
555
556 import tango.io.GrowBuffer : GrowBuffer;
557
558 unittest
559 {
560 // One ring to rule them all, one ring to find them,
561 // One ring to bring them all and in the darkness bind them.
562 const char[] message =
563 "Ash nazg durbatulûk, ash nazg gimbatul, "
564 "ash nazg thrakatulûk, agh burzum-ishi krimpatul.";
565
566 // This compressed data was created using Python 2.5's built in zlib
567 // module, with the default compression level.
568 const ubyte[] message_z = [
569 0x78,0x9c,0x73,0x2c,0xce,0x50,0xc8,0x4b,
570 0xac,0x4a,0x57,0x48,0x29,0x2d,0x4a,0x4a,
571 0x2c,0x29,0xcd,0x39,0xbc,0x3b,0x5b,0x47,
572 0x21,0x11,0x26,0x9a,0x9e,0x99,0x0b,0x16,
573 0x45,0x12,0x2a,0xc9,0x28,0x4a,0xcc,0x46,
574 0xa8,0x4c,0xcf,0x50,0x48,0x2a,0x2d,0xaa,
575 0x2a,0xcd,0xd5,0xcd,0x2c,0xce,0xc8,0x54,
576 0xc8,0x2e,0xca,0xcc,0x2d,0x00,0xc9,0xea,
577 0x01,0x00,0x1f,0xe3,0x22,0x99];
578
579 scope cond_z = new GrowBuffer;
580 scope comp = new ZlibOutput(cond_z);
581 comp.write (message);
582 comp.close;
583
584 assert( comp.written == message_z.length );
585
586 assert( message_z == cast(ubyte[])(cond_z.slice) );
587
588 scope decomp = new ZlibInput(cond_z);
589 auto buffer = new ubyte[256];
590 buffer = buffer[0 .. decomp.read(buffer)];
591
592 assert( cast(ubyte[])message == buffer );
593 }
594 }