Mercurial > projects > ldc

comparison tango/example/conduits/FileBucket.d @ 132:1700239cab2e trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[svn r136] MAJOR UNSTABLE UPDATE!!! Initial commit after moving to Tango instead of Phobos. Lots of bugfixes... This build is not suitable for most things.
author lindquist
date Fri, 11 Jan 2008 17:57:40 +0100
parents
children
comparison
equal deleted inserted replaced
131:5825d48b27d1 132:1700239cab2e
1 module FileBucket;
2
3 private import tango.io.FilePath,
4 tango.io.FileConduit;
5
6 private import tango.core.Exception;
7
8 /******************************************************************************
9
10 FileBucket implements a simple mechanism to store and recover a
11 large quantity of data for the duration of the hosting process.
12 It is intended to act as a local-cache for a remote data-source,
13 or as a spillover area for large in-memory cache instances.
14
15 Note that any and all stored data is rendered invalid the moment
16 a FileBucket object is garbage-collected.
17
18 The implementation follows a fixed-capacity record scheme, where
19 content can be rewritten in-place until said capacity is reached.
20 At such time, the altered content is moved to a larger capacity
21 record at end-of-file, and a hole remains at the prior location.
22 These holes are not collected, since the lifespan of a FileBucket
23 is limited to that of the host process.
24
25 All index keys must be unique. Writing to the FileBucket with an
26 existing key will overwrite any previous content. What follows
27 is a contrived example:
28
29 ---
30 char[] text = "this is a test";
31
32 auto bucket = new FileBucket (new FilePath("bucket.bin"), FileBucket.HalfK);
33
34 // insert some data, and retrieve it again
35 bucket.put ("a key", text);
36 char[] b = cast(char[]) bucket.get ("a key");
37
38 assert (b == text);
39 bucket.close;
40 ---
41
42 ******************************************************************************/
43
44 class FileBucket
45 {
46 /**********************************************************************
47
48 Define the capacity (block-size) of each record
49
50 **********************************************************************/
51
52 struct BlockSize
53 {
54 int capacity;
55 }
56
57 // basic capacity for each record
58 private FilePath path;
59
60 // basic capacity for each record
61 private BlockSize block;
62
63 // where content is stored
64 private FileConduit file;
65
66 // pointers to file records
67 private Record[char[]] map;
68
69 // current file size
70 private long fileSize;
71
72 // current file usage
73 private long waterLine;
74
75 // supported block sizes
76 public static const BlockSize EighthK = {128-1},
77 HalfK = {512-1},
78 OneK = {1024*1-1},
79 TwoK = {1024*2-1},
80 FourK = {1024*4-1},
81 EightK = {1024*8-1},
82 SixteenK = {1024*16-1},
83 ThirtyTwoK = {1024*32-1},
84 SixtyFourK = {1024*64-1};
85
86
87 /**********************************************************************
88
89 Construct a FileBucket with the provided path and record-
90 size. Selecting a record size that roughly matches the
91 serialized content will limit 'thrashing'.
92
93 **********************************************************************/
94
95 this (char[] path, BlockSize block)
96 {
97 this (new FilePath(path), block);
98 }
99
100 /**********************************************************************
101
102 Construct a FileBucket with the provided path, record-size,
103 and inital record count. The latter causes records to be
104 pre-allocated, saving a certain amount of growth activity.
105 Selecting a record size that roughly matches the serialized
106 content will limit 'thrashing'.
107
108 **********************************************************************/
109
110 this (FilePath path, BlockSize block, uint initialRecords = 100)
111 {
112 this.path = path;
113 this.block = block;
114
115 // open a storage file
116 file = new FileConduit (path, FileConduit.ReadWriteCreate);
117
118 // set initial file size (can be zero)
119 fileSize = initialRecords * block.capacity;
120 file.seek (fileSize);
121 file.truncate ();
122 }
123
124 /**********************************************************************
125
126 Return the block-size in use for this FileBucket
127
128 **********************************************************************/
129
130 int getBufferSize ()
131 {
132 return block.capacity+1;
133 }
134
135 /**********************************************************************
136
137 Return where the FileBucket is located
138
139 **********************************************************************/
140
141 FilePath getFilePath ()
142 {
143 return path;
144 }
145
146 /**********************************************************************
147
148 Return the currently populated size of this FileBucket
149
150 **********************************************************************/
151
152 synchronized long length ()
153 {
154 return waterLine;
155 }
156
157 /**********************************************************************
158
159 Return the serialized data for the provided key. Returns
160 null if the key was not found.
161
162 **********************************************************************/
163
164 synchronized void[] get (char[] key)
165 {
166 Record r = null;
167
168 if (key in map)
169 {
170 r = map [key];
171 return r.read (this);
172 }
173 return null;
174 }
175
176 /**********************************************************************
177
178 Remove the provided key from this FileBucket.
179
180 **********************************************************************/
181
182 synchronized void remove (char[] key)
183 {
184 map.remove(key);
185 }
186
187 /**********************************************************************
188
189 Write a serialized block of data, and associate it with
190 the provided key. All keys must be unique, and it is the
191 responsibility of the programmer to ensure this. Reusing
192 an existing key will overwrite previous data.
193
194 Note that data is allowed to grow within the occupied
195 bucket until it becomes larger than the allocated space.
196 When this happens, the data is moved to a larger bucket
197 at the file tail.
198
199 **********************************************************************/
200
201 synchronized void put (char[] key, void[] data)
202 {
203 Record* r = key in map;
204
205 if (r is null)
206 {
207 auto rr = new Record;
208 map [key] = rr;
209 r = &rr;
210 }
211 r.write (this, data, block);
212 }
213
214 /**********************************************************************
215
216 Close this FileBucket -- all content is lost.
217
218 **********************************************************************/
219
220 synchronized void close ()
221 {
222 if (file)
223 {
224 file.detach;
225 file = null;
226 map = null;
227 }
228 }
229
230 /**********************************************************************
231
232 Each Record takes up a number of 'pages' within the file.
233 The size of these pages is determined by the BlockSize
234 provided during FileBucket construction. Additional space
235 at the end of each block is potentially wasted, but enables
236 content to grow in size without creating a myriad of holes.
237
238 **********************************************************************/
239
240 private static class Record
241 {
242 private long offset;
243 private int length,
244 capacity = -1;
245
246 /**************************************************************
247
248 **************************************************************/
249
250 private static void eof (FileBucket bucket)
251 {
252 throw new IOException ("Unexpected EOF in FileBucket '"~bucket.path.toString()~"'");
253 }
254
255 /**************************************************************
256
257 This should be protected from thread-contention at
258 a higher level.
259
260 **************************************************************/
261
262 void[] read (FileBucket bucket)
263 {
264 void[] data = new ubyte [length];
265
266 bucket.file.seek (offset);
267 if (bucket.file.read (data) != length)
268 eof (bucket);
269
270 return data;
271 }
272
273 /**************************************************************
274
275 This should be protected from thread-contention at
276 a higher level.
277
278 **************************************************************/
279
280 void write (FileBucket bucket, void[] data, BlockSize block)
281 {
282 length = data.length;
283
284 // create new slot if we exceed capacity
285 if (length > capacity)
286 createBucket (bucket, length, block);
287
288 // locate to start of content
289 bucket.file.seek (offset);
290
291 // write content
292 if (bucket.file.write (data) != length)
293 eof (bucket);
294 }
295
296 /**************************************************************
297
298 **************************************************************/
299
300 void createBucket (FileBucket bucket, int bytes, BlockSize block)
301 {
302 offset = bucket.waterLine;
303 capacity = (bytes + block.capacity) & ~block.capacity;
304
305 bucket.waterLine += capacity;
306 if (bucket.waterLine > bucket.fileSize)
307 {
308 // grow the filesize
309 bucket.fileSize = bucket.waterLine * 2;
310
311 // expand the physical file size
312 bucket.file.seek (bucket.fileSize);
313 bucket.file.truncate ();
314 }
315 }
316 }
317 }
318
319