Mercurial > projects > ldc

comparison lphobos/std/mmfile.d @ 473:373489eeaf90

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Applied downs' lphobos update
author Tomas Lindquist Olsen <tomas.l.olsen@gmail.com>
date Mon, 04 Aug 2008 19:28:49 +0200
parents
children 88e23f8c2354
comparison
equal deleted inserted replaced
472:15c804b6ce77 473:373489eeaf90
1 /* Copyright 2004-2005 by Digital Mars
2 * Written by Walter Bright and Matthew Wilson
3 *
4 * This software is provided 'as-is', without any express or implied
5 * warranty. In no event will the authors be held liable for any damages
6 * arising from the use of this software.
7 *
8 * Permission is granted to anyone to use this software for any purpose,
9 * including commercial applications, and to alter it and redistribute it
10 * freely, in both source and binary form, subject to the following
11 * restrictions:
12 *
13 * - The origin of this software must not be misrepresented; you must not
14 * claim that you wrote the original software. If you use this software
15 * in a product, an acknowledgment in the product documentation would be
16 * appreciated but is not required.
17 * - Altered source versions must be plainly marked as such, and must not
18 * be misrepresented as being the original software.
19 * - This notice may not be removed or altered from any source
20 * distribution.
21 *
22 */
23
24 /**
25 * Read and write memory mapped files.
26 * Macros:
27 * WIKI=Phobos/StdMmfile
28 */
29
30 /* NOTE: This file has been patched from the original DMD distribution to
31 work with the GDC compiler.
32
33 Modified by David Friedman, September 2004
34 */
35
36 module std.mmfile;
37
38 private import std.c.stdio;
39 private import std.c.stdlib;
40 private import std.string;
41
42 //debug = MMFILE;
43
44 version (Win32)
45 {
46 private import std.c.windows.windows;
47 private import std.utf;
48
49 private uint dwVersion;
50
51 static this()
52 { // http://msdn.microsoft.com/library/default.asp?url=/library/en-us/sysinfo/base/getversion.asp
53 dwVersion = GetVersion();
54 }
55
56 private const bool Have_MMFile = true; // private for now...
57 }
58 else version (Unix)
59 {
60 version (/*GNU_Unix_Have_MMap*/Unix)
61 {
62 version(linux) {
63 import std.c.linux.linux;
64 alias std.c.linux.linux unix;
65 } else {
66 private import std.c.unix.unix;
67 alias std.c.unix.unix unix;
68 }
69
70 version = unix_mm;
71 private const bool Have_MMFile = true;
72 }
73 else
74 {
75 private const bool Have_MMFile = false;
76 }
77
78 }
79 else
80 {
81 private const bool Have_MMFile = false;
82 // Can't simply fail because std.stream imports this module.
83 //static assert(0);
84 }
85
86 static if (Have_MMFile)
87 {
88 private import std.file;
89 private import std.path;
90 }
91
92 /**
93 * MmFile objects control the memory mapped file resource.
94 */
95 class MmFile
96 {
97 /**
98 * The mode the memory mapped file is opened with.
99 */
100 enum Mode
101 { Read, /// read existing file
102 ReadWriteNew, /// delete existing file, write new file
103 ReadWrite, /// read/write existing file, create if not existing
104 ReadCopyOnWrite, /// read/write existing file, copy on write
105 }
106
107 /**
108 * Open memory mapped file filename for reading.
109 * File is closed when the object instance is deleted.
110 * Throws:
111 * std.file.FileException
112 */
113 this(char[] filename)
114 {
115 this(filename, Mode.Read, 0, null);
116 }
117
118 /**
119 * Open memory mapped file filename in mode.
120 * File is closed when the object instance is deleted.
121 * Params:
122 * filename = name of the file.
123 * If null, an anonymous file mapping is created.
124 * mode = access mode defined above.
125 * size = the size of the file. If 0, it is taken to be the
126 * size of the existing file.
127 * address = the preferred address to map the file to,
128 * although the system is not required to honor it.
129 * If null, the system selects the most convenient address.
130 * window = preferred block size of the amount of data to map at one time
131 * with 0 meaning map the entire file. The window size must be a
132 * multiple of the memory allocation page size.
133 * Throws:
134 * std.file.FileException
135 */
136 this(char[] filename, Mode mode, ulong size, void* address,
137 size_t window = 0)
138 {
139 this.filename = filename;
140 this.mMode = mode;
141 this.window = window;
142 this.address = address;
143
144 version (Win32)
145 {
146 void* p;
147 uint dwDesiredAccess2;
148 uint dwShareMode;
149 uint dwCreationDisposition;
150 uint flProtect;
151
152 if (dwVersion & 0x80000000 && (dwVersion & 0xFF) == 3)
153 {
154 throw new FileException(filename,
155 "Win32s does not implement mm files");
156 }
157
158 switch (mode)
159 {
160 case Mode.Read:
161 dwDesiredAccess2 = GENERIC_READ;
162 dwShareMode = FILE_SHARE_READ;
163 dwCreationDisposition = OPEN_EXISTING;
164 flProtect = PAGE_READONLY;
165 dwDesiredAccess = FILE_MAP_READ;
166 break;
167
168 case Mode.ReadWriteNew:
169 assert(size != 0);
170 dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
171 dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
172 dwCreationDisposition = CREATE_ALWAYS;
173 flProtect = PAGE_READWRITE;
174 dwDesiredAccess = FILE_MAP_WRITE;
175 break;
176
177 case Mode.ReadWrite:
178 dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
179 dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
180 dwCreationDisposition = OPEN_ALWAYS;
181 flProtect = PAGE_READWRITE;
182 dwDesiredAccess = FILE_MAP_WRITE;
183 break;
184
185 case Mode.ReadCopyOnWrite:
186 if (dwVersion & 0x80000000)
187 {
188 throw new FileException(filename,
189 "Win9x does not implement copy on write");
190 }
191 dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
192 dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
193 dwCreationDisposition = OPEN_EXISTING;
194 flProtect = PAGE_WRITECOPY;
195 dwDesiredAccess = FILE_MAP_COPY;
196 break;
197
198 default:
199 assert(0);
200 }
201
202 if (filename)
203 {
204 if (useWfuncs)
205 {
206 auto namez = std.utf.toUTF16z(filename);
207 hFile = CreateFileW(namez,
208 dwDesiredAccess2,
209 dwShareMode,
210 null,
211 dwCreationDisposition,
212 FILE_ATTRIBUTE_NORMAL,
213 cast(HANDLE)null);
214 }
215 else
216 {
217 auto namez = std.file.toMBSz(filename);
218 hFile = CreateFileA(namez,
219 dwDesiredAccess2,
220 dwShareMode,
221 null,
222 dwCreationDisposition,
223 FILE_ATTRIBUTE_NORMAL,
224 cast(HANDLE)null);
225 }
226 if (hFile == INVALID_HANDLE_VALUE)
227 goto err1;
228 }
229 else
230 hFile = null;
231
232 int hi = cast(int)(size>>32);
233 hFileMap = CreateFileMappingA(hFile, null, flProtect, hi, cast(uint)size, null);
234 if (hFileMap == null) // mapping failed
235 goto err1;
236
237 if (size == 0)
238 {
239 uint sizehi;
240 uint sizelow = GetFileSize(hFile,&sizehi);
241 size = (cast(ulong)sizehi << 32) + sizelow;
242 }
243 this.size = size;
244
245 size_t initial_map = (window && 2*window<size)? 2*window : cast(size_t)size;
246 p = MapViewOfFileEx(hFileMap, dwDesiredAccess, 0, 0, initial_map, address);
247 if (!p) goto err1;
248 data = p[0 .. initial_map];
249
250 debug (MMFILE) printf("MmFile.this(): p = %p, size = %d\n", p, size);
251 return;
252
253 err1:
254 if (hFileMap != null)
255 CloseHandle(hFileMap);
256 hFileMap = null;
257
258 if (hFile != INVALID_HANDLE_VALUE)
259 CloseHandle(hFile);
260 hFile = INVALID_HANDLE_VALUE;
261
262 errNo();
263 }
264 else version (unix_mm)
265 {
266 char* namez = toStringz(filename);
267 void* p;
268 int oflag;
269 int fmode;
270
271 switch (mode)
272 {
273 case Mode.Read:
274 flags = MAP_SHARED;
275 prot = PROT_READ;
276 oflag = O_RDONLY;
277 fmode = 0;
278 break;
279
280 case Mode.ReadWriteNew:
281 assert(size != 0);
282 flags = MAP_SHARED;
283 prot = PROT_READ | PROT_WRITE;
284 oflag = O_CREAT | O_RDWR | O_TRUNC;
285 fmode = 0660;
286 break;
287
288 case Mode.ReadWrite:
289 flags = MAP_SHARED;
290 prot = PROT_READ | PROT_WRITE;
291 oflag = O_CREAT | O_RDWR;
292 fmode = 0660;
293 break;
294
295 case Mode.ReadCopyOnWrite:
296 flags = MAP_PRIVATE;
297 prot = PROT_READ | PROT_WRITE;
298 oflag = O_RDWR;
299 fmode = 0;
300 break;
301
302 default:
303 assert(0);
304 }
305
306 if (filename.length)
307 {
308 struct_stat statbuf;
309
310 fd = unix.open(namez, oflag, fmode);
311 if (fd == -1)
312 {
313 // printf("\topen error, errno = %d\n",getErrno());
314 errNo();
315 }
316
317 if (unix.fstat(fd, &statbuf))
318 {
319 //printf("\tfstat error, errno = %d\n",getErrno());
320 unix.close(fd);
321 errNo();
322 }
323
324 if (prot & PROT_WRITE && size > statbuf.st_size)
325 {
326 // Need to make the file size bytes big
327 unix.lseek(fd, cast(off_t)(size - 1), SEEK_SET);
328 char c = 0;
329 unix.write(fd, &c, 1);
330 }
331 else if (prot & PROT_READ && size == 0)
332 size = statbuf.st_size;
333 }
334 else
335 {
336 fd = -1;
337 flags |= MAP_ANONYMOUS;
338 }
339 this.size = size;
340 size_t initial_map = (window && 2*window<size)? 2*window : cast(size_t)size;
341 p = mmap(address, initial_map, prot, flags, fd, 0);
342 if (p == MAP_FAILED) {
343 if (fd != -1)
344 unix.close(fd);
345 errNo();
346 }
347
348 data = p[0 .. initial_map];
349 }
350 else static if (! Have_MMFile)
351 {
352 throw new FileException("This system does support memory mapped files");
353 }
354 else
355 {
356 static assert(0);
357 }
358 }
359
360 /**
361 * Flushes pending output and closes the memory mapped file.
362 */
363 ~this()
364 {
365 debug (MMFILE) printf("MmFile.~this()\n");
366 unmap();
367 version (Win32)
368 {
369 if (hFileMap != null && CloseHandle(hFileMap) != TRUE)
370 errNo();
371 hFileMap = null;
372
373 if (hFile != INVALID_HANDLE_VALUE && CloseHandle(hFile) != TRUE)
374 errNo();
375 hFile = INVALID_HANDLE_VALUE;
376 }
377 else version (unix_mm)
378 {
379 if (fd != -1 && unix.close(fd) == -1)
380 errNo();
381 fd = -1;
382 }
383 else static if (! Have_MMFile)
384 {
385 }
386 else
387 {
388 static assert(0);
389 }
390 data = null;
391 }
392
393 /* Flush any pending output.
394 */
395 void flush()
396 {
397 debug (MMFILE) printf("MmFile.flush()\n");
398 version (Win32)
399 {
400 FlushViewOfFile(data.ptr, data.length);
401 }
402 else version (unix_mm)
403 {
404 int i;
405
406 i = msync(cast(void*)data, data.length, MS_SYNC); // sys/mman.h
407 if (i != 0)
408 errNo();
409 }
410 else static if (! Have_MMFile)
411 {
412 }
413 else
414 {
415 static assert(0);
416 }
417 }
418
419 /**
420 * Gives size in bytes of the memory mapped file.
421 */
422 ulong length()
423 {
424 debug (MMFILE) printf("MmFile.length()\n");
425 return size;
426 }
427
428 /**
429 * Read-only property returning the file mode.
430 */
431 Mode mode()
432 {
433 debug (MMFILE) printf("MmFile.mode()\n");
434 return mMode;
435 }
436
437 /**
438 * Returns entire file contents as an array.
439 */
440 void[] opSlice()
441 {
442 debug (MMFILE) printf("MmFile.opSlice()\n");
443 return opSlice(0,size);
444 }
445
446 /**
447 * Returns slice of file contents as an array.
448 */
449 void[] opSlice(ulong i1, ulong i2)
450 {
451 debug (MMFILE) printf("MmFile.opSlice(%lld, %lld)\n", i1, i2);
452 ensureMapped(i1,i2);
453 size_t off1 = cast(size_t)(i1-start);
454 size_t off2 = cast(size_t)(i2-start);
455 return data[off1 .. off2];
456 }
457
458 /**
459 * Returns byte at index i in file.
460 */
461 ubyte opIndex(ulong i)
462 {
463 debug (MMFILE) printf("MmFile.opIndex(%lld)\n", i);
464 ensureMapped(i);
465 size_t off = cast(size_t)(i-start);
466 return (cast(ubyte[])data)[off];
467 }
468
469 /**
470 * Sets and returns byte at index i in file to value.
471 */
472 ubyte opIndexAssign(ubyte value, ulong i)
473 {
474 debug (MMFILE) printf("MmFile.opIndex(%lld, %d)\n", i, value);
475 ensureMapped(i);
476 size_t off = cast(size_t)(i-start);
477 return (cast(ubyte[])data)[off] = value;
478 }
479
480
481 // return true if the given position is currently mapped
482 private int mapped(ulong i)
483 {
484 debug (MMFILE) printf("MmFile.mapped(%lld, %lld, %d)\n", i,start,
485 data.length);
486 return i >= start && i < start+data.length;
487 }
488
489 // unmap the current range
490 private void unmap()
491 {
492 debug (MMFILE) printf("MmFile.unmap()\n");
493 version(Windows) {
494 /* Note that under Windows 95, UnmapViewOfFile() seems to return
495 * random values, not TRUE or FALSE.
496 */
497 if (data && UnmapViewOfFile(data.ptr) == FALSE &&
498 (dwVersion & 0x80000000) == 0)
499 errNo();
500 } else version (unix_mm) {
501 if (data && munmap(cast(void*)data, data.length) != 0)
502 errNo();
503 }
504 data = null;
505 }
506
507 // map range
508 private void map(ulong start, size_t len)
509 {
510 debug (MMFILE) printf("MmFile.map(%lld, %d)\n", start, len);
511 void* p;
512 if (start+len > size)
513 len = cast(size_t)(size-start);
514 version(Windows) {
515 uint hi = cast(uint)(start>>32);
516 p = MapViewOfFileEx(hFileMap, dwDesiredAccess, hi, cast(uint)start, len, address);
517 if (!p) errNo();
518 } else version (unix_mm) {
519 p = mmap(address, len, prot, flags, fd, cast(off_t)start);
520 if (p == MAP_FAILED) errNo();
521 }
522 data = p[0 .. len];
523 this.start = start;
524 }
525
526 // ensure a given position is mapped
527 private void ensureMapped(ulong i)
528 {
529 debug (MMFILE) printf("MmFile.ensureMapped(%lld)\n", i);
530 if (!mapped(i)) {
531 unmap();
532 if (window == 0) {
533 map(0,cast(size_t)size);
534 } else {
535 ulong block = i/window;
536 if (block == 0)
537 map(0,2*window);
538 else
539 map(window*(block-1),3*window);
540 }
541 }
542 }
543
544 // ensure a given range is mapped
545 private void ensureMapped(ulong i, ulong j)
546 {
547 debug (MMFILE) printf("MmFile.ensureMapped(%lld, %lld)\n", i, j);
548 if (!mapped(i) || !mapped(j-1)) {
549 unmap();
550 if (window == 0) {
551 map(0,cast(size_t)size);
552 } else {
553 ulong iblock = i/window;
554 ulong jblock = (j-1)/window;
555 if (iblock == 0) {
556 map(0,cast(size_t)(window*(jblock+2)));
557 } else {
558 map(window*(iblock-1),cast(size_t)(window*(jblock-iblock+3)));
559 }
560 }
561 }
562 }
563
564 private:
565 char[] filename;
566 void[] data;
567 ulong start;
568 size_t window;
569 ulong size;
570 Mode mMode;
571 void* address;
572
573 version (Win32)
574 {
575 HANDLE hFile = INVALID_HANDLE_VALUE;
576 HANDLE hFileMap = null;
577 uint dwDesiredAccess;
578 }
579 else version (unix_mm)
580 {
581 int fd;
582 int prot;
583 int flags;
584 int fmode;
585 }
586 else static if (! Have_MMFile)
587 {
588 }
589 else
590 {
591 static assert(0);
592 }
593
594 // Report error, where errno gives the error number
595 void errNo()
596 {
597 version (Win32)
598 {
599 throw new FileException(filename, GetLastError());
600 }
601 else version (Unix)
602 {
603 throw new FileException(filename, getErrno());
604 }
605 else static if (! Have_MMFile)
606 {
607 throw new FileException(filename, "MMFile unsupported");
608 }
609 else
610 {
611 static assert(0);
612 }
613 }
614 }
615
616 unittest {
617 static if (Have_MMFile)
618 {
619 const size_t K = 1024;
620 size_t win = 64*K; // assume the page size is 64K
621 version(Win32) {
622 /+ these aren't defined in std.c.windows.windows so let's use the default
623 SYSTEM_INFO sysinfo;
624 GetSystemInfo(&sysinfo);
625 win = sysinfo.dwAllocationGranularity;
626 +/
627 } else version (Unix) {
628 // getpagesize() is not defined in the unix D headers so use the guess
629 }
630 MmFile mf = new MmFile("testing.txt",MmFile.Mode.ReadWriteNew,100*K,null,win);
631 ubyte[] str = cast(ubyte[])"1234567890";
632 ubyte[] data = cast(ubyte[])mf[0 .. 10];
633 data[] = str[];
634 assert( mf[0 .. 10] == str );
635 data = cast(ubyte[])mf[50 .. 60];
636 data[] = str[];
637 assert( mf[50 .. 60] == str );
638 ubyte[] data2 = cast(ubyte[])mf[20*K .. 60*K];
639 assert( data2.length == 40*K );
640 assert( data2[length-1] == 0 );
641 mf[100*K-1] = cast(ubyte)'b';
642 data2 = cast(ubyte[])mf[21*K .. 100*K];
643 assert( data2.length == 79*K );
644 assert( data2[length-1] == 'b' );
645 delete mf;
646 std.file.remove("testing.txt");
647 }
648 }