Mercurial > projects > mde
changeset 14:0047b364b6d9
Changed much of the mergetag structure and some functionality. First tests on windows.
Changes to mergetag Reader methods. New functionality allowing a dataSecCreator to cause sections to be skipped.
Moved several of the mergetag modules and some of their contents around. Moved all interfaces to separate modules in iface/ .
IReader & IWriter interfaces exist; MTTReader, MTBReader, MTTWriter, MTBWriter & DualWriter all now exist and implement IReader/IWriter (although the MTB variants are dummy classes); makeReader & makeWriter should both be fully functional.
Tested building on windows with partial success (works but window won't open).
Included a temporary hack from windows to get supported resolutions information.
committer: Diggory Hardy <diggory.hardy@gmail.com>
| author | Diggory Hardy <diggory.hardy@gmail.com> |
|---|---|
| date | Fri, 07 Mar 2008 17:51:02 +0000 |
| parents | 914fed025adb |
| children | 4608be19ebe2 |
| files | conf/input.mtt conf/options.mtt doc/jobs dsss.conf mde/i18n.d mde/init.d mde/input/config.d mde/input/input.d mde/mergetag/DataSet.d mde/mergetag/DefaultData.d mde/mergetag/Reader.d mde/mergetag/Writer.d mde/mergetag/datasection.d mde/mergetag/dataset.d mde/mergetag/defaultdata.d mde/mergetag/doc/file-format-binary.txt mde/mergetag/exception.d mde/mergetag/iface/IDataSection.d mde/mergetag/iface/IReader.d mde/mergetag/iface/IWriter.d mde/mergetag/internal.d mde/mergetag/mtunittest.d mde/mergetag/read.d mde/mergetag/write.d mde/options.d mde/scheduler.d test/mdeTest.d |
| diffstat | 27 files changed, 1285 insertions(+), 1052 deletions(-) [+] |
line wrap: on
line diff
--- a/conf/input.mtt Mon Feb 25 11:11:30 2008 +0000 +++ b/conf/input.mtt Fri Mar 07 17:51:02 2008 +0000 @@ -1,6 +1,5 @@ {MT01} -<char[][]|Configs=["Std"]> -<char[]|Test="1"> +!<char[][]|Configs=["Default"]> {Default} <uint[][uint]|B=[ 0x8800001B : [[0x1000, 0x0]] ]> {UnitTest}
--- a/conf/options.mtt Mon Feb 25 11:11:30 2008 +0000 +++ b/conf/options.mtt Fri Mar 07 17:51:02 2008 +0000 @@ -1,5 +1,4 @@ {MT01} {Default} -<char[]|greeting="Hi − options now work!"> <char[]|L10n="en-GB">
--- a/doc/jobs Mon Feb 25 11:11:30 2008 +0000 +++ b/doc/jobs Fri Mar 07 17:51:02 2008 +0000 @@ -1,8 +1,11 @@ In progress: To do: -* Submit new version of parseTo -* Why doesn't input.config filtering via headers "Configs" work? +* Control logging level/output via options. +* Read config from system and user paths. +* Windows building/compatibility (currently partial) +* gdc building/compatibility (wait for tango 0.99.5 release?) +* Config loaded from correct places * OutOfMemoryException is not currently checked for − it should be at least in critical places (use high-level catching of all errors?). * Sensitivity adjustments. From es_a_out: /+ FIXME: revise. @@ -33,6 +36,8 @@ +/ Done (for git log message): -* Implemented i18n.I18nTranslation class to load strings and descriptions from files (with unittest). -* MTUnknownTypeException removed: its pointless since it's always ignored without even any message. -* A few fixes to mde.mergetag.read.Reader regarding partial reading. +Changes to mergetag Reader methods. New functionality allowing a dataSecCreator to cause sections to be skipped. +Moved several of the mergetag modules and some of their contents around. Moved all interfaces to separate modules in iface/ . +Mergetag: IReader & IWriter interfaces exist; MTTReader, MTBReader, MTTWriter, MTBWriter & DualWriter all now exist and implement IReader/IWriter (although the MTB variants are dummy classes); makeReader & makeWriter should both be fully functional. +Tested building on windows with partial success (works but window won't open). +Included a temporary hack from windows to get supported resolutions information.
--- a/dsss.conf Mon Feb 25 11:11:30 2008 +0000 +++ b/dsss.conf Fri Mar 07 17:51:02 2008 +0000 @@ -1,21 +1,24 @@ -defaulttargets = mde/mde.d +version (Posix) { + defaulttargets = mde/mde.d +} else version (Win32) { + defaulttargets = mde\mde.d +} [*] version (Posix) { buildflags=-L-ldl -} else { - warn Only posix builds have been tested; elsewhere other libraries will probably need to be linked. } [mde/mde.d] target=bin/mde +#[mde\mde.d] +#target=bin\mde + [test/mdeTest.d] buildflags=-debug -debug=mdeUnitTest -unittest target=bin/mdeTest noinstall version (Posix) { buildflags+=-L-ldl -} else { - warn Only posix builds have been tested; elsewhere other libraries will probably need to be linked. }
--- a/mde/i18n.d Mon Feb 25 11:11:30 2008 +0000 +++ b/mde/i18n.d Fri Mar 07 17:51:02 2008 +0000 @@ -26,8 +26,8 @@ import mde.options; import mde.exception; -import mde.mergetag.dataset; -import mde.mergetag.read; +import mde.mergetag.DataSet; +import mde.mergetag.Reader; import mde.mergetag.exception; import tango.util.log.Log : Log, Logger; @@ -39,7 +39,7 @@ * * Encoding used is UTF-8. */ -class I18nTranslation : DataSection +class I18nTranslation : IDataSection { final char[] name; /// The module/package/... which the instance is for final char[] L10n; /// The localization loaded (e.g. en-GB) @@ -84,18 +84,19 @@ /* Find the file for name and load it. */ FilePath filePath = new FilePath ("conf/L10n/"~name~".mtt"); - // If it's not a file or an empty file stop. - if (!filePath.exists || filePath.fileSize == 0u) { + // If it doesn't exist, stop. + // Don't bother checking it's not a folder, because it could still be a block or something. + if (!filePath.exists) { throw new L10nLoadException ("No database file conf/L10n/"~name~".mtt exists!"); } - Reader reader; + IReader reader; try { - reader = new Reader(filePath); + reader = new MTTReader(filePath); /* Note: we don't want to load every translation section depended on to its own class * instance, since we want to merge them. So make every mergetag section use the same * instance. */ - reader.dataSecCreator = delegate DataSection(ID) { + reader.dataSecCreator = delegate IDataSection(ID) { return transl; };
--- a/mde/init.d Mon Feb 25 11:11:30 2008 +0000 +++ b/mde/init.d Fri Mar 07 17:51:02 2008 +0000 @@ -18,7 +18,7 @@ import tango.core.Exception; import tango.util.log.Log : Log, Logger; import tango.util.log.ConsoleAppender : ConsoleAppender; -import tango.stdc.stringz : fromUtf8z; +import tango.stdc.stringz : fromStringz; import derelict.sdl.sdl; import derelict.util.exception; @@ -35,7 +35,8 @@ else { // For now, just log to the console: Logger root = Log.getRootLogger(); - root.setLevel(root.Level.Trace); + debug root.setLevel(root.Level.Trace); + else root.setLevel(root.Level.Info); root.addAppender(new ConsoleAppender); } } @@ -92,21 +93,53 @@ setFailure (); // abort return; } - logger.info ("Derelict: loaded SDL"); + logger.trace ("Derelict: loaded SDL"); if (SDL_Init (SDL_INIT_VIDEO | SDL_INIT_JOYSTICK /+| SDL_INIT_EVENTTHREAD+/)) { logger.fatal ("SDL initialisation failed:"); char* msg = SDL_GetError (); - logger.fatal (msg ? fromUtf8z(msg) : "no reason available"); + logger.fatal (msg ? fromStringz(msg) : "no reason available"); setFailure (); // abort return; } - SDL_SetVideoMode (800, 600, 0, 0); + addCleanupFct (&cleanupSDL); + logger.trace ("SDL initialised"); + + // Print a load of info: + logger.info ("Available video modes:"); + char[128] tmp; + SDL_Rect** modes = SDL_ListModes (null, SDL_FULLSCREEN); + if (modes is null) logger.info ("None!"); + else if (modes is cast(SDL_Rect**) -1) logger.info ("All modes are available"); + else { + for (uint i = 0; modes[i] !is null; ++i) { + logger.info (logger.format (tmp, "\t{}x{}", modes[i].w, modes[i].h)); + } + } - addCleanupFct (&cleanupSDL); - logger.info ("SDL initialised"); + SDL_VideoInfo* vi = SDL_GetVideoInfo (); + if (vi !is null) { + logger.info ("Video info:"); + logger.info ("Hardware surface support: "~ (vi.flags & SDL_HWSURFACE ? "yes" : "no")); + logger.info (logger.format (tmp, "Video memory: {}", vi.video_mem)); + + if (vi.vfmt !is null) { + logger.info ("Best video mode:"); + logger.info (logger.format (tmp, "Bits per pixel: {}", vi.vfmt.BitsPerPixel)); + } + } + + // FIXME: make this non-fatal and provide a way to re-set video mode + if (SDL_SetVideoMode (800, 600, 0, 0) is null) { // Can't open in windows!! + logger.fatal ("Unable to set video mode:"); + char* msg = SDL_GetError (); + logger.fatal (msg ? fromStringz(msg) : "no reason available"); + + setFailure (); + return; + } openJoysticks (); // after SDL init addCleanupFct (&closeJoysticks); @@ -163,7 +196,7 @@ foreach (t; tg) { try { t.join (true); - /+ Will only catch thread exceptions; but even so something still went badly wrong so we want the same functionality. + /+ We might as well catch and report any exception, rather than just ThreadExceptions: } catch (ThreadException e) { // Any threading exception +/ } catch (Exception e) { // Any other exception, i.e. caught from thread. @@ -194,9 +227,9 @@ ~this() { if (!initFailure) { - logger.info ("Cleaning up..."); + logger.trace ("Cleaning up..."); runCleanupFcts(); // if threading, note not all functions can be called simultaeneously - logger.info ("Done!"); + logger.trace ("Done!"); } }
--- a/mde/input/config.d Mon Feb 25 11:11:30 2008 +0000 +++ b/mde/input/config.d Fri Mar 07 17:51:02 2008 +0000 @@ -5,7 +5,7 @@ import mde.input.exception; -import MT = mde.mergetag.read; +import MT = mde.mergetag.Reader; import tango.scrapple.text.convert.parseTo : parseTo; import tango.util.log.Log : Log, Logger; @@ -16,7 +16,7 @@ * * Class extends DataSection so that it can be loaded by mergetag easily. */ -class Config : MT.DataSection +class Config : MT.IDataSection { alias uint[] outQueue; // This is the type for the out queue config data. /** Button event type bit-codes @@ -117,23 +117,25 @@ if (loadedFiles.contains (filename)) return; // forget it; already done that loadedFiles.add (filename); - MT.Reader file; + MT.IReader file; try { - file = new MT.Reader(filename, null, true); // open and read header + file = new MT.MTTReader(filename, null, true); // open and read header // TODO: also load user-config file file.dataSecCreator = - delegate MT.DataSection (MT.ID) { return new Config; }; + delegate MT.IDataSection (MT.ID) { return new Config; }; // D2.0: enum MT.ID CONFIGS = "Configs"; const MT.ID CONFIGS = cast(MT.ID)"Configs"; - MT.ID[] file_configs; // active config sections (may not exist) - MT.ID[]* file_configs_p = cast(MT.ID[]*) (CONFIGS in file.dataset.header._charAA); - debug foreach (i,d; file.dataset.header._charAA) logger.trace ("ID: "~cast(char[])i); - debug foreach (i,d; file.dataset.header._charA) logger.trace ("ID: "~cast(char[])i); - - if (file_configs_p) file.read(*file_configs_p); // restrict to this set IF a restriction was given + // Restrict config sections if this tag exists: + auto file_configs_p = CONFIGS in file.dataset.header._charAA; + MT.ID[] file_configs = null; + if (file_configs_p) { + file_configs = cast(MT.ID[]) *file_configs_p; + } + + if (file_configs) file.read(file_configs); // restrict to this set IF a restriction was given else file.read(); // otherwise read all } catch (MT.MTException) {
--- a/mde/input/input.d Mon Feb 25 11:11:30 2008 +0000 +++ b/mde/input/input.d Fri Mar 07 17:51:02 2008 +0000 @@ -122,12 +122,39 @@ /** Feed an SDL_Event struct (only uses if it's a key, mouse or joystick event). * - * Other types of event functions may be added. Returns true if the event was used, false if not. + * Other types of event functions may be added. Returns true if the event was used, false if not + * or no config was available. Hmm... doesn't seem very useful, but has practically no cost. * * May throw InputClassExceptions (on configuration errors). Catching the exception and continuing should * be fine. */ bool opCall (ref SDL_Event event) { + /* Non-config events. + * + * Mouse events don't need config for the GUI. Handle them first so that if no config exists + * some functionality at least is retained. + */ + switch (event.type) { + case SDL_MOUSEBUTTONDOWN: + case SDL_MOUSEBUTTONUP: + foreach (dg; mouseClickCallbacks) + dg (event.button.x, event.button.y, event.button.button, event.button.state == SDL_PRESSED); + break; + + case SDL_MOUSEMOTION: + mouse_x = event.motion.x; + mouse_y = event.motion.y; + break; + + default: + } + + /* No config available, so don't try to access it and segfault. + * Don't log a message because this function is called per-event (i.e. frequently). + * A message should already have been logged by loadConfig anyway. + */ + if (!config) return false; + switch (event.type) { // Keyboard events: case SDL_KEYDOWN: @@ -141,10 +168,6 @@ // Mouse events: case SDL_MOUSEBUTTONDOWN: case SDL_MOUSEBUTTONUP: - // Mouse clicks: - foreach (dg; mouseClickCallbacks) - dg (event.button.x, event.button.y, event.button.button, event.button.state == SDL_PRESSED); - // Button events: outQueue[]* p = (Config.B.MOUSE | event.button.button) in config.button; if (p) foreach (outQueue q; *p) { @@ -153,10 +176,6 @@ break; case SDL_MOUSEMOTION: - // Screen coordinates (set here since relMotion is also used for joystick balls): - mouse_x = event.motion.x; - mouse_y = event.motion.y; - // Relative motion: outQueue[]* p = (Config.M.WMMOUSE) in config.relMotion; if (p) foreach (outQueue q; *p) { @@ -267,7 +286,7 @@ config = *c_p; return false; } - debug logger.warn ("Config \""~profile~"\" not found."); + logger.warn ("Config profile \""~profile~"\" not found: input won't work unless a valid profile is loaded!"); return true; }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mde/mergetag/DataSet.d Fri Mar 07 17:51:02 2008 +0000 @@ -0,0 +1,56 @@ +/** This module contains the mergetag DataSet class, used for all reading and writing operations. + */ +module mde.mergetag.DataSet; + +// package imports +public import mde.mergetag.iface.IDataSection; +import mde.mergetag.DefaultData; +import mde.mergetag.exception; + + +/************************************************************************************************** + * Data class; contains a DataSection class instance for each loaded section of a file. + * + * Stored data is available for direct access via header and sec; all functions are just helper + * functions. + * + * Any class implementing IDataSection may be used to store data; by default a DefaultData class is + * used when reading a file. Another class may be used by creating the sections before reading the + * file or passing the reader a function to create the sections (see Reader.dataSecCreator). + * + * Could be a struct, except that structs are value types (not reference types). + */ +class DataSet +{ + DefaultData header; /// Header section. + IDataSection[ID] sec; /// Dynamic array of sections + + /// Template to return all sections of a child-class type. + T[ID] getSections (T : IDataSection) () { + T[ID] ret; + foreach (ID id, IDataSection s; sec) { + T x = cast(T) s; + if (x) ret[id] = x; // if non-null + } + return ret; + } +} + +debug (mdeUnitTest) { + import tango.util.log.Log : Log, Logger; + + private Logger logger; + static this() { + logger = Log.getLogger ("mde.mergetag.DataSet"); + } + + unittest { // Only covers DataSet really. + DataSet ds = new DataSet; + ds.sec[cast(ID)"test"] = new DefaultData; + assert (ds.getSections!(DefaultData)().length == 1); + ds.sec[cast(ID)"test"].addTag ("int",cast(ID)"T"," -543 "); + assert (ds.getSections!(DefaultData)()[cast(ID)"test"]._int[cast(ID)"T"] == -543); + + logger.info ("Unittest complete."); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mde/mergetag/DefaultData.d Fri Mar 07 17:51:02 2008 +0000 @@ -0,0 +1,163 @@ +/** This module contains the DefaultData class, and some notes possibly useful for implementing +* other types of DataSection. +*/ +module mde.mergetag.DefaultData; + +public import mde.mergetag.iface.IDataSection; +import mde.mergetag.exception; + +import tango.scrapple.text.convert.parseTo : parseTo; +import tango.scrapple.text.convert.parseFrom : parseFrom; + + +/** +* Default DataSection class. +* +* Supports most of the basic types supported by D (excluding cent/ucent and imaginary/complex +* types) and array versions of each of these types, plus arrays of strings. +* +* Extending the class to support more types, even custom types, shouldn't be particularly difficult +* provided mde.text.parseTo and mde.text.parseFrom are extended to support the new types. +*/ +/* The implementation now uses a fair bit of generic programming. Adjusting the types supported +* should be as simple as adjusting the list dataTypes, and possibly implemting new conversions in +* parseFrom and parseTo if you add new types (e.g. for cent or imaginary/complex types, or user types). +* +* There shouldn't really be any need to adjust the implementation, except perhaps to add new +* functions to the class (such as another type of output where the delegate used in writeAll isn't +* enough). +*/ +class DefaultData : IDataSection +{ + //BEGIN META + /* These functions are used to generate code. Compile-time functions rather than templates are + * used because they are easier to write and understand. Mixins are used to compile the resultant + * code. Must be declared before used since forward references aren't supported for compile-time + * functions. */ + + // Generate the correct name for each variable type. + static char[] varName (char[] type) { + char[] append = ""; + while (type.length >= 2 && type[$-2..$] == "[]") { + type = type[0..$-2]; + append ~= "A"; + } + return "_" ~ type ~ append; + } + + // Int-to-string converter, which may not be efficient but will run at compile time. + static char[] int2str (uint i) { + char[] ret; + const digits = "0123456789"; + if (i == 0) ret = "0"; + else for (; i > 0; i /= 10) ret = digits[i%10] ~ ret; + return ret; + } + + // Generate the code for variable declarations. + static char[] declerations (char[][] types) { + char[] ret = ""; + foreach (char[] type; types) ret ~= type ~ "[ID]\t" ~ varName(type) ~ ";\n"; + return ret; + } + + // Purely to add indentation. Could just return "" without affecting functionality. + static char[] indent (uint i) { + char[] ret; + for (; i > 0; --i) ret ~= " "; + // This is not executable at compile time: + //ret.length = i * 4; // number of characters for each indentation + //ret[] = ' '; // character to indent with + return ret; + } + + /* Generates a binary search algorithm. + * + * Currently this is tailored to it's particular use (addTag). */ + static char[] binarySearch (char[] var, char[][] consts, int indents = 0) { + if (consts.length > 3) { + return indent(indents) ~ "if (" ~ var ~ " <= \"" ~ consts[$/2 - 1] ~ "\") {\n" ~ + binarySearch (var, consts[0 .. $/2], indents + 1) ~ + indent(indents) ~ "} else {\n" ~ + binarySearch (var, consts[$/2 .. $], indents + 1) ~ + indent(indents) ~ "}\n"; + } else { + char[] ret; + ret ~= indent(indents); + foreach (c; consts) { + ret ~= "if (" ~ var ~ " == \"" ~ c ~ "\") {\n" ~ + indent(indents+1) ~ varName(c) ~ "[id] = parseTo!(" ~ c ~ ") (dt);\n" ~ + indent(indents) ~ "} else "; + } + ret = ret[0..$-6] ~ '\n'; // remove last else + return ret; + } + } + + // Generates the code to write data members (writeAll). + static char[] writeVars () { + char[] code = ""; + foreach (i,type; dataTypes) { + code ~= "foreach (id, dt; " ~ varName(type) ~ ") itemdlg (dataTypes[" ~ int2str(i) ~ "], id, parseFrom!(" ~ type ~ ")(dt));\n"; + } + return code; + } + //END META + + /** Data Members + * + * These types are all stored directly, as below, are available for direct access. The variable + * names are created dynamically at compile-time based on the dataTypes list. + * ------------------ + * int[ID] _int; // name is type prefixed by _ + * char[][ID] _charA; // [] is replaced by A + * ------------------ + * + * An alternative access method is to use the provided templates: + * -------------------- + * template Arg(T) { + * alias Name Arg; + * } + * + * type y = Arg!(type).Arg; // example of use + * -------------------- + * Note: trying to use Arg!(type) to implicitly refer to Arg!(type).Arg causes compiler errors + * due to the "alias Name Arg;" statement actually being a mixin. + */ + const char[][] dataTypes = ["bool","bool[]", + "byte","byte[]", + "char","char[]","char[][]", + "double","double[]", + "float","float[]", + "int","int[]", + "long","long[]", + "real","real[]", + "short","short[]", + "ubyte","ubyte[]", + "uint","uint[]", + "ulong","ulong[]", + "ushort","ushort[]"]; + + mixin (declerations (dataTypes)); // Declare all the variables. + + void addTag (char[] type, ID id, char[] dt) { /// Supports all types listed in dataTypes. + mixin (binarySearch ("type", dataTypes)); + } + + void writeAll (ItemDelg itemdlg) { /// Supports all types listed in dataTypes. + mixin (writeVars ()); + } + + /* These make no attempt to check Arg is valid. + * But if the symbol doesn't exist the complier will throw an error anyway, e.g.: + * Error: identifier '_boolAA' is not defined + */ + template Arg(T : T[]) { + const ArgString = Arg!(T).ArgString ~ `A`; + mixin(`alias `~ArgString~` Arg;`); + } + template Arg(T) { + const ArgString = `_` ~ T.stringof; + mixin(`alias `~ArgString~` Arg;`); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mde/mergetag/Reader.d Fri Mar 07 17:51:02 2008 +0000 @@ -0,0 +1,511 @@ +/************************************************************************************************** + * This module contains all reading functions, for both binary and text MergeTag files. + *************************************************************************************************/ + +module mde.mergetag.Reader; + +// package imports +public import mde.mergetag.iface.IReader; +import mde.mergetag.DataSet; +import mde.mergetag.DefaultData; +import mde.mergetag.exception; +import mde.mergetag.internal; + +import tango.core.Exception; + +// tango imports +import tango.io.UnicodeFile; +import Util = tango.text.Util; +import ConvInt = tango.text.convert.Integer; +import tango.util.collection.model.View : View; +import tango.util.collection.HashSet : HashSet; +import tango.util.log.Log : Log, Logger; + +private Logger logger; +static this() { + logger = Log.getLogger ("mde.mergetag.Reader"); +} + +// TODO: allow compressing with zlib for both binary and text? (.mtz, .mtt, .mtb extensions) + +/** Make an IReader class. +* +* If no extension is given, search for a file using each extension (.mtt and .mtb) appended to +* path, and set path to the most recent file name. +* +* When an extension is available (either after the above or when supplied), use the appropriate +* reader (MTT or MTB). +* +* Throws: +* $(TABLE +* $(TR $(TH Exception) $(TH Thrown when)) +* $(TR $(TD MTFileFormatException) $(TD Unable to determine format (only analysing file name))) +* $(TR $(TD MTFileIOException) $(TD When no extension is given, neither appending .mtt nor +* appending .mtb resolves a valid file)) +* ) +* +*/ +IReader makeReader (char[] path, DataSet ds = null, bool rdHeader = false) { + return makeReader (new FilePath(path), ds, rdHeader); +} +IReader makeReader (PathView path, DataSet ds = null, bool rdHeader = false) { + if (path.ext.length == 0) { + PathView tPath = new FilePath (path.toString ~ ".mtt"); + PathView bPath = new FilePath (path.toString ~ ".mtb"); + + bool bPathExists = bPath.exists; + + if (tPath.exists) { + if (bPathExists) { + // take the latest version (roughly speaking...) + path = tPath.modified > bPath.modified ? tPath : bPath; + } else path = tPath; + } else { + if (bPathExists) path = bPath; + else { + logger.error ("No file exists: "~path.toString~"[.mtt|.mtb]"); + throw new MTFileIOException; + } + } + } + + if (path.ext == "mtb") return new MTBReader (path, ds, rdHeader); + else if (path.ext == "mtt") return new MTTReader (path, ds, rdHeader); + else throw new MTFileFormatException; +} + +/** + * Class for reading a file. + * + * Use as: + * ----------------------- + * IReader foo; + * try { + * foo = new MTTReader("foo.mtt"); + * foo.read(); + * } + * catch (MTException) {} + * // get your data from foo.dataset. + * ----------------------- + * + * Throws: + * $(TABLE + * $(TR $(TH Exception) $(TH Thrown when)) + * $(TR $(TD MTFileIOException) $(TD An error occurs while opening the file)) + * $(TR $(TD MTFileFormatException) $(TD The file doesn't start with a recognised header/version)) + * $(TR $(TD MTSyntaxException) $(TD A file syntax error occurs)) + * $(TR $(TD MTException) $(TD An unexpected error occurs)) + * ) + * Note that all exceptions extend MTException and when any exception is thrown the class is + * rendered unusable: any subsequent calls to read will be ignored. + * + * Threading: Separate instances of Reader should be thread-safe provided access to the same + * dataset is synchronized; i.e. no two readers refering to the same dataset should run + * simultaneously. (The Reader class could be made thread-safe w.r.t. datasets, but + * performance-wise I doubt it would be worth it.) + * Do not run a single instance of Reader in multiple threads simultaneously. + */ +class MTTReader : IReader +{ +//BEGIN DATA + /** Get or set the DataSet + * + * A container for all read data. + * + * This may be accessed from here; however it may be preferable to use an external reference + * (passed to the class on initialisation). + */ + DataSet dataset () { return _dataset; } + void dataset (DataSet ds) /// ditto + { _dataset = ds; } + + /** A delegate for creating new DataSections within the dataset. + * + * Allows a user-made class to be used in the DataSet instead of DefaultData (used if no + * dataSecCreator exists). Also allows an existing class instance to be used instead of a new + * one. + * + * This works by supplying a function which returns a reference to an instance of a class + * implementing IDataSection. The function is passed the ID of the new section and may use this + * to use different IDataSection classes for different sections. + * + * The function may also return null, in which case the section will be skipped. In the version + * of read taking a set of sections to read, the section will not be marked as read and may + * still be read later (assuming dataSecCreator returns non-null). However, in the version of + * read not taking the set argument, all sections are set as read regardless, and the section + * cannot be read later. + */ + void dataSecCreator (IDataSection delegate (ID) dSC) { + _dataSecCreator = dSC; + } + +private: + static Logger logger; + + // Non-static symbols: + final char[] ErrFile; // added after ErrInFile to do the same without the "in " bit. + final char[] ErrInFile; // something like "in \"path/file.mtt\"" + + final char[] fbuf; // file is read into this + MTFormatVersion.VERS fileVer = MTFormatVersion.VERS.INVALID; // Remains INVALID until set otherwise by CTOR. + + IDataSection delegate (ID) _dataSecCreator = null; // see property setter above + + uint endOfHeader; + bool allRead = false; // true if endOfHeader == fbuf.length or read([]) has run + bool fatal = false; // a fatal file error occured; don't try to recover + /* If the file is scanned for sections, the starting position of all sections are stored + * in secTable. If this is empty, either no sections exist (and endOfHeader == fbuf.length) + * or a section scan has not been run (read() with no section names doesn't need to do so). + */ + struct SecMD { // sec meta data + static SecMD opCall (uint _pos, bool _read) { + SecMD ret; + ret.pos = _pos; + ret.read = _read; + return ret; + } + uint pos; // position to start reading + bool read; // true if already read + } + SecMD [ID] secTable; + + DataSet _dataset; +//END DATA + +//BEGIN METHODS: CTOR / DTOR + static this () { + logger = Log.getLogger ("mde.mergetag.read.Reader"); + } + + /** Tries to open file path and read it into a buffer. + * + * Params: + * path = The name or FilePath of the file to open. + * Standard extensions are .mtt and .mtb for text and binary files respectively. + * ds = If null create a new DataSet, else use existing DataSet ds and merge read + * data into it. + * rdHeader = If true, read the header like a standard section. Doesn't read the header by + * default since if it's not requested it's likely not wanted. + * + * Memory: + * This currently works by loading the whole file into memory at once. This should be fine most + * of the time, but could potentially be a problem. Changing this would mean significantly + * changes to the way the code works. + */ + /* Ideas for implementing a partial-loading memory model: + * Use a conduit directly. + * Use a fiber to do the parsing; let it switch back when it runs out of memory. + * Redesign the code so it never needs to look backwards in the buffer? + * + * Major problem: reading only some sections and keeping references to other sections + * would no longer be possible. + */ + public this (char[] path, DataSet ds = null, bool rdHeader = false) { + this (new FilePath (path), ds, rdHeader); + } + /** ditto */ + public this (PathView path, DataSet ds = null, bool rdHeader = false) { + // Create a dataset or use an existing one + if (ds !is null) _dataset = ds; + else _dataset = new DataSet(); + + // Open & read the file + try { // Supports unicode files with a BOM; defaults to UTF8 when there isn't a BOM: + scope file = new UnicodeFile!(char) (path, Encoding.Unknown); + fbuf = cast(char[]) file.read(); + } catch (Exception e) { + throwMTErr ("Error reading file: " ~ e.msg, new MTFileIOException); + } + // Remember the file name so that we can report errors (somewhat) informatively: + ErrFile = path.path ~ path.file; + ErrInFile = " in \"" ~ ErrFile ~ '"'; + + // Version checking & matching header section tag: + if (fbuf.length < 6 || fbuf[0] != '{' || fbuf[1] != 'M' || fbuf[2] != 'T' || fbuf[5] != '}') + throwMTErr("Not a valid MergeTag text file" ~ ErrInFile, new MTFileFormatException); + fileVer = MTFormatVersion.parseString (fbuf[3..5]); + if (fileVer == MTFormatVersion.VERS.INVALID) + throwMTErr("Unrecognised MergeTag version: MT" ~ fbuf[3..5] ~ ErrInFile, new MTFileFormatException); + + // Header reading/skipping: + if (rdHeader) { // only bother actually reading it if it was requested + // If already existing, merge; else create a new DefaultData. + if (!_dataset.header) _dataset.header = new DefaultData; + endOfHeader = parseSection (6, cast(IDataSection) _dataset.header); + } + else endOfHeader = parseSection (6,null); + } +//END METHODS: CTOR / DTOR + +//BEGIN METHODS: PUBLIC + /** Scans for sections if not already done and returns a list of IDs. + * + * Won't work (will return an empty array) if all sections have already been read without + * scanning for sections. + */ + public uint[] getSectionNames () { + if (fatal) return []; + if (!secTable.length) read([]); // scan for sections + return cast(uint[]) secTable.keys; + } + + /** Reads (some) sections of the file into data. Note that sections will never be _read twice. + * + * To be more accurate, the file is copied into a buffer by this(). read() then parses the + * contents of this buffer, and stores the contents in dataset. + * + * Each section read is stored in a DataSection class. By default this is an instance of + * DefaultData; this can be customised (see dataSecCreator). + * + * If secSet is provided, reading is restricted to sections given in secSet, otherwise all + * sections are read. Sections given in secSet but not found in the file are not reported as an + * error. Suggested: supply a HashSet!(uint) as the View!(ID). An ArrayBag!(ID) as used is not a + * good choice, except that in this case it's empty. + * + * Merging: + * Where a section already exists in the DataSet (when either the section is given more than + * once in the file, or it was read from a different file by another reader) it is merged. + * Entries already in the DataSet take priority. + * + * Performance: + * Note that loading only desired sections like this still parses the sections not + * read (although it does not try to understand the type or data fields), so there is only a + * small performance advantage to this where other sections do exist in the file. There is also + * some overhead in only partially reading the file to keep track of where other sections are so + * that the entire file need not be re-read if further (or all remaining) sections are read + * later. + */ + public void read () { + if (secTable.length) { + foreach (ID id, ref SecMD smd; secTable) { + if (!smd.read) { + IDataSection ds = getOrCreateSec (id); + parseSection (smd.pos, ds); + // allRead is set true so there's no point setting smd.read = true + } + } + } else { // this time we don't need to use secTable + for (uint pos = endOfHeader; pos < fbuf.length;) { + ID id = fbufReadSecMarker (pos); + IDataSection ds = getOrCreateSec (id); + pos = parseSection (pos, ds); + } + } + + allRead = true; + } + /** ditto */ + public void read (ID[] secSet) { + HashSet!(ID) hs = new HashSet!(ID); + foreach (id; secSet) hs.add(id); + read (hs); + } + /** ditto */ + public void read (View!(ID) secSet) { + if (allRead || fatal) return; // never do anything in either case + + if (secTable.length) { + foreach (ID id; secSet) { + SecMD* psmd = id in secTable; + if (psmd && !psmd.read) { // may not exist + IDataSection ds = getOrCreateSec (id); + parseSection (psmd.pos, ds); + if (ds !is null) psmd.read = true; // getOrCreateSec may return null + } + } + } else { + for (uint pos = endOfHeader; pos < fbuf.length;) { + ID id = fbufReadSecMarker (pos); + secTable[id] = SecMD(pos,false); // add to table + if (secSet.contains(id)) { + IDataSection ds = getOrCreateSec (id); + pos = parseSection (pos, ds); + if (ds !is null) secTable[id].read = true; + } else { + pos = parseSection (pos, null); // skip section + } + } + } + } +//END METHODS: PUBLIC + +//BEGIN METHODS: PRIVATE + /* Utility function for read + * Look for a section; return it if it exists otherwise create a new section: + * use _dataSecCreator if it exists or just create a DefaultData if not. + * However if _dataSecCreator returns null don't add it to the dataset. + */ + private IDataSection getOrCreateSec (ID id) { + IDataSection* i = id in _dataset.sec; + if (i) return *i; + else { + IDataSection s; + if (_dataSecCreator !is null) s = _dataSecCreator(id); + else s = new DefaultData; + if (s !is null) _dataset.sec[id] = s; + return s; + } + } + + /* Reads a section, starting from index pos, finishing at the next section marker (returning + the position of the start of the marker). pos should start after the section marker. + + After analysing tags, the function passes the type, ID and data to addTag. + + NOTE: from performance tests on indexing char[]'s and dereferencing char*'s, the char*'s are + slightly faster, but a tiny difference isn't worth the extra effort/risk of using char*'s. + */ + private uint parseSection (uint pos, IDataSection dsec) { + /* Searches fbuf starting from start to find one of <=>| and stops at its index. + + If quotable then be quote-aware for single and double quotes. + Note: there's no length restriction for the content of the quote since it could be a single + non-ascii UTF-8 char which would look like several chars. + */ + void fbufLocateDataTagChar (inout uint pos, bool quotable) { + for (; pos < fbuf.length; ++pos) { + if ((fbuf[pos] >= '<' && fbuf[pos] <= '>') || fbuf[pos] == '|') return; + else if (quotable) { + char c = fbuf[pos]; + if (c == '\'' || c == '"') { + ++pos; + while (fbuf[pos] != c) { + if (fbuf[pos] == '\\') ++pos; // escape seq. + fbufIncrement(pos); + } + } + } + } + } + + // Used to ignore a tag (if it starts !< or !{ or should otherwise be ignored): + bool comment = false; + for (; pos < fbuf.length; ++pos) { + if (Util.isSpace(fbuf[pos])) continue; // whitespace + else if (fbuf[pos] == '<') { // data tag + char[] ErrDTAG = "Bad data tag format: not <type|id=data>" ~ ErrInFile; + + fbufIncrement (pos); + + // Type section of tag: + uint pos_s = pos; + fbufLocateDataTagChar (pos, false); // find end of type section + if (fbuf[pos] != '|') throwMTErr (ErrDTAG, new MTSyntaxException); + char[] type = fbuf[pos_s..pos]; + + fbufIncrement (pos); + + // ID section of tag: + pos_s = pos; + fbufLocateDataTagChar (pos, false); // find end of type section + if (fbuf[pos] != '=') throwMTErr (ErrDTAG, new MTSyntaxException); + ID tagID = cast(ID) fbuf[pos_s..pos]; + + fbufIncrement (pos); + + // Data section of tag: + pos_s = pos; + fbufLocateDataTagChar (pos, true); // find end of data section + if (fbuf[pos] != '>') throwMTErr (ErrDTAG, new MTSyntaxException); + char[] data = fbuf[pos_s..pos]; + + if (!comment && dsec !is null) { + type = Util.trim(type); + try { + dsec.addTag (type, tagID, data); + } + catch (TextException e) { + logger.warn ("TextException while reading " ~ ErrFile ~ ":"); // following a parse error + logger.warn (e.msg); + } + catch (Exception e) { + logger.error ("Unknown error occured" ~ ErrInFile ~ ':'); + logger.error (e.msg); + throwMTErr (e.msg); // Fatal to Reader + } + } else comment = false; // cancel comment status now + } + else if (fbuf[pos] == '{') { + if (comment) { // simple block comment + uint depth = 0; // depth of embedded comment blocks + while (true) { + fbufIncrement (pos); + if (fbuf[pos] == '}') { + if (depth == 0) break; + else --depth; + } else if (fbuf[pos] == '{') + ++depth; + } + comment = false; // end of this comment + } else { + return pos; // next section coming up; we are done + } + } + else if (fbuf[pos] == '!') { // possibly a comment; check next char + comment = true; // starting a comment (or an error) + // variable is reset at end of comment + } else // must be an error + throwMTErr ("Invalid character (or sequence starting \"!\") outside of tag" ~ ErrInFile, new MTSyntaxException); + } + // if code execution reaches here, we're at EOF + // possible error: last character was ! (but don't bother checking since it's inconsequential) + return pos; + } + + /* Parses fbuf for a section marker. Already knows fbuf[pos] == '{'. + */ + private ID fbufReadSecMarker (inout uint pos) { + // at this point pos is whatever a parseSection run returned + // since we haven't hit EOF, fbuf[pos] MUST be '{' so no need to check + fbufIncrement(pos); + + uint start = pos; + for (; pos < fbuf.length; ++pos) + if (fbuf[pos] == '}' || fbuf[pos] == '{') break; + + if (pos >= fbuf.length || fbuf[pos] != '}') + throwMTErr ("Bad section tag format: not {id}" ~ ErrInFile, new MTSyntaxException); + + ID id = cast(ID) fbuf[start..pos]; + fbufIncrement(pos); + return id; + } + + /* Increments pos and checks it hasn't hit fbuf.length . */ + private void fbufIncrement(inout uint pos) { + ++pos; + if (pos >= fbuf.length) throwMTErr("Unexpected EOF" ~ ErrInFile, new MTSyntaxException); + } + + private void throwMTErr (char[] msg, MTException exc = new MTException) { + fatal = true; // if anyone catches the error and tries to do anything --- we're dead now + logger.error (msg); // report the error + throw exc; // and signal our error + } +//END METHODS: PRIVATE +} + +class MTBReader : IReader +{ + public this (char[] path, DataSet ds = null, bool rdHeader = false) { + this (new FilePath (path), ds, rdHeader); + } + public this (PathView path, DataSet ds = null, bool rdHeader = false) { + throw new MTNotImplementedException; + } + + DataSet dataset () { /// Get the DataSet + return null; + } + void dataset (DataSet) {} /// Set the DataSet + + void dataSecCreator (IDataSection delegate (ID)) {} /// Set the dataSecCreator + + uint[] getSectionNames () { /// Get identifiers for all sections + return []; + } + void read () {} /// Commence reading + void read (ID[] secSet) {} /// ditto + void read (View!(ID) secSet) {} /// ditto +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mde/mergetag/Writer.d Fri Mar 07 17:51:02 2008 +0000 @@ -0,0 +1,269 @@ +/************************************************************************************************** + * This module contains all writing functions, for both binary and text MergeTag files. + * + * Files can be written in a text or binary form; binary is faster and smaller while text allows + * editing with an ordinary text editor. TextWriter and BinaryWriter are the main classes, both of + * which implement the interface IWriter. DualWriter is another class implementing IWriter, which + * contains a private instance of a TextWriter and a BinaryWriter and implements all methods in the + * interface simply by chaining the appropriate method from each of these classes, thus performing + * two writes at once. + * + * Any of these three classes may be used directly, or makeWriter may be invoked to create an + * instance of the appropriate class. + *************************************************************************************************/ + module mde.mergetag.Writer; + +// package imports +public import mde.mergetag.iface.IWriter; +import mde.mergetag.DataSet; +import mde.mergetag.internal; +import mde.mergetag.exception; + +// tango imports +import tango.core.Exception; +import tango.io.FileConduit; +import tango.io.Buffer : Buffer, IBuffer; +import tango.io.Print : Print; +import convInt = tango.text.convert.Integer; +import tango.util.log.Log : Log, Logger; + +private Logger logger; +static this () { + logger = Log.getLogger ("mde.mergetag.Writer"); +} + + +/** Method to create and return either a MTTWriter or a MTBWriter. + * + * Has two modes of operation: if method is FromExtension, examines the existing extension and + * creates a MTT/MTB writer if the extension is mtt or mtb (throwing if not). + * + * Otherwise, writing format is determined directly by method, and appropriate extensions are + * added to the file name without checking for an existing extension. + * + * Params: + * path = File path + * dataset = Dataset passed to Writer to write from (if null, must be set before write() is called) + * method = $(TABLE + * $(TR $(TH Value) $(TH Writer returned) $(TH Suffix added)) + * $(TR $(TD FromExtension) $(TD MTBWriter or MTTWriter)$(TD $(I none))) + * $(TR $(TD Binary) $(TD MTBWriter) $(TD .mtb)) + * $(TR $(TD Text) $(TD MTTWriter) $(TD .mtt)) + * $(TR $(TD Both) $(TD DualWriter) $(TD .mtb / .mtt)) + * ) + * + * Throws: + * MTFileFormatException if neither test can deduce the writing method, the supplied writing + * method is invalid or the determined/supplied method is not yet implemented. + * + * Use as: + * ----------------------- + * DataSet dataset; // contains data to write + * IWriter foo; + * try { + * foo = makeWriter(...); + * foo.write(); + * } + * catch (MTException) {} + * ----------------------- + * Where the makeWriter line has one of the following forms: + * ----------------------- + * foo = makeWriter("foo.mtt", dataset); + * foo = makeWriter("foo", dataset, WriterMethod.Text); + * ----------------------- + * + * Throws: + * MTFileFormatException if unable to determine writing format or use requested format. + */ +IWriter makeWriter (char[] path, DataSet dataset = null, WriterMethod method = WriterMethod.FromExtension) { + if (method == WriterMethod.FromExtension) { + PathView fpath = new FilePath (path); + + if (fpath.ext == "mtt") return new MTTWriter (fpath, dataset); + else if (fpath.ext == "mtb") return new MTBWriter (fpath, dataset); + else { + logger.error ("Unable to determine writing format: text or binary"); + throw new MTFileFormatException; + } + } + else { + if (method == WriterMethod.Binary) return new MTBWriter (path~".mtb", dataset); + else if (method == WriterMethod.Text) return new MTTWriter (path~".mtt", dataset); + else if (method == WriterMethod.Both) return new DualWriter (path, dataset); + else throw new MTFileFormatException; + } +} + + +/** + * Class to write a dataset to a file. + * + * Files are only actually open for writing while the write() method is running. + * + * Throws: + * $(TABLE + * $(TR $(TH Exception) $(TH Thrown when)) + * $(TR $(TD MTNoDataSetException) $(TD No dataset is available to write from)) + * $(TR $(TD MTFileIOException) $(TD An error occurs while attemting to write the file)) + * $(TR $(TD MTException) $(TD An unexpected error occurs)) + * ) + * Note that all exceptions extend MTException; unlike Reader exceptions don't block further calls. + */ +class MTTWriter : IWriter +{ +//BEGIN DATA + /// Get or set the DataSet (i.e. the container from which all data is written). + DataSet dataset () { return _dataset; } + void dataset (DataSet ds) /// ditto + { _dataset = ds; } + + +private: + // taken from tango.io.Console, mostly to make sure notepad can read our files: + version (Win32) + const char[] Eol = "\r\n"; + else + const char[] Eol = "\n"; + + /* The container where data is written from. */ + DataSet _dataset; + + PathView _path; +//END DATA + +//BEGIN CTOR / DTOR + /** Prepares to open file path for writing. + * + * The call doesn't actually execute any code so cannot fail (unless out of memory). + * + * Params: + * path = The name or FilePath of the file to open. + * Standard extensions are .mtt and .mtb for text and binary files respectively. + * dataset_ = If null create a new DataSet, else use existing DataSet *dataset_ and merge read + * data into it. + */ + public this (char[] path, DataSet ds = null) { + this (new FilePath (path), ds); + } + /** ditto */ + public this (PathView path, DataSet ds = null) { + _path = path; + _dataset = ds; + } +//END CTOR / DTOR + + /** Writes the header and all DataSections. + * + * Firstly writes the header unless it has already been written. Then writes all DataSections + * to the file. Thus if write is called more than once with or without changing the DataSet the + * header should be written only once. This behaviour could, for instance, be used to write + * multiple DataSets into one file without firstly merging them. Note that this behaviour may + * be changed when binary support is added. + */ + public void write () + { + if (!_dataset) throwMTErr ("write(): no Dataset available to write from!", new MTNoDataSetException ()); + + try { + FileConduit conduit; // actual conduit; don't use directly when there's content in the buffer + IBuffer buffer; // write strings directly to this (use opCall(void[]) ) + + // Open a conduit on the file: + conduit = new FileConduit (_path, FileConduit.WriteCreate); + scope(exit) conduit.close(); + + buffer = new Buffer(conduit); // And a buffer + scope(exit) buffer.flush(); + + // Write the header: + buffer ("{MT" ~ MTFormatVersion.CurrentString ~ "}" ~ Eol); + if (_dataset.header !is null) writeSection (buffer, _dataset.header); + + // Write the rest: + foreach (ID id, IDataSection sec; _dataset.sec) { + writeSectionIdentifier (buffer, id); + writeSection (buffer, sec); + } + + buffer.flush(); + + } + catch (IOException e) { + throwMTErr ("Error writing to file: " ~ e.msg, new MTFileIOException); + } + catch (Exception e) { + throwMTErr ("Unexpected exception when writing file: " ~ e.msg); + } + } + + private void writeSectionIdentifier (IBuffer buffer, ID id) { + buffer ("{" ~ cast(char[])id ~ "}" ~ Eol); + } + + private void writeSection (IBuffer buffer, IDataSection sec) { + void writeItem (char[] tp, ID id, char[] dt) { // actually writes an item + buffer ("<" ~ tp ~ "|" ~ cast(char[])id ~"=" ~ dt ~ ">" ~ Eol); + } + sec.writeAll (&writeItem); + + buffer (Eol); // blank line at end of each section + } + + private void throwMTErr (char[] msg, Exception exc = new MTException) { + logger.error (msg); // report the error + throw exc; // and signal our error + } +} + +/* +* Implement MTBWriter (and move both writers to own modules?). +*/ +class MTBWriter : IWriter { + public this (char[] path, DataSet ds = null) { + this (new FilePath (path), ds); + } + public this (PathView path, DataSet ds = null) { + throw new MTNotImplementedException; + + /+_path = path; + _dataset = ds;+/ + } + + DataSet dataset () { + return null; + } + void dataset (DataSet) {} + + void write () {} +} + +/* Basic implementation for mtt only. +* +*Implement std CTORs which add extensions to each filename and extra CTORs which take two filenames. +*/ +class DualWriter : IWriter { + /** The individual writers. + * + * Potentially could be used directly, but I'd suggest not. */ + MTTWriter mtt; + //MTBWriter mtb; /** ditto */ + + public this (char[] path, DataSet ds = null) { + mtt = new MTTWriter (path~".mtt", ds); + } + + DataSet dataset () { + return mtt.dataset; + } + void dataset (DataSet ds) { + mtt.dataset = ds; + } + + /** Write. + * + * Write text then binary, so the mtb file will be the most recent. + */ + void write () { + mtt.write(); + } +}
--- a/mde/mergetag/datasection.d Mon Feb 25 11:11:30 2008 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,51 +0,0 @@ -/** This module contains the interface DataSection used by DataSet. -* -* It has been moved from the dataset module to avoid cyclic dependancies, since defaultdata depends -* on DataSection and dataset depends on defaultdata. -* -* Also some base mergetag symbols have been moved here. -*/ -module mde.mergetag.datasection; - -/** Typedef for data & section indexes (can be changed to ulong if necessary.) */ -typedef char[] ID; - -/** - * Interface for data storage classes, which contain all data-tags loaded from a single section of a - * file. - * - * A class implementing this may implement the addTag function to do whatever it likes with the - * data passed. DefaultData is one implementation which separates this data out into supported - * types and stores it appropriately (allowing merging with existing entries by keeping whichever - * tag was last loaded), while ignoring unsupported types. A different - * implementation could filter out the tags desired and use them directly, and ignore the rest. - * - * The tango.scrapple.text.convert.parseTo module provides a useful set of templated functions to - * convert the data accordingly. It is advised to keep the type definitions as defined in the file- - * format except for user-defined types, although this isn't necessary for library operation - * (addTag and writeAll are solely responsible for using and setting the type, ID and data fields). - * - * Another idea for a DataSection class: - * Use a void*[ID] variable to store all data (may also need a type var for each item). - * addTag should call a templated function which calls parse then casts to a void* and stores the data. - * Use a templated get(T)(ID) method which checks the type and casts to T. - */ -interface DataSection -{ - /** Delegate passed to writeAll. */ - typedef void delegate (char[],ID,char[]) ItemDelg; - - /** Handles parsing of data items for all recognised types. - * - * Should ignore unsupported types/unwanted tags. - * - * TextExceptions (thrown by parseTo/parseFrom) are caught and a warning logged; execution - * then continues (so the offending tag gets dropped). */ - void addTag (char[],ID,char[]); - - /** Responsible for getting all data tags saved. - * - * writeAll should call the ItemDelg once for each tag to be saved with parameters in the same - * form as received by addTag (char[] type, ID id, char[] data). */ - void writeAll (ItemDelg); -}
--- a/mde/mergetag/dataset.d Mon Feb 25 11:11:30 2008 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,116 +0,0 @@ -/** This module contains the mergetag DataSet class together with an interface for DataSections. - * - * It basically forms the "glue" upon which most of mergetag works, hence gets publically imported - * by most mergetag modules. - */ -module mde.mergetag.dataset; - -// package imports -import mde.mergetag.exception; -import mde.mergetag.defaultdata; // used in DataSet so it should be publically imported - -import tango.util.log.Log : Log, Logger; - -private Logger logger; -static this() { - logger = Log.getLogger ("mde.mergetag.dataset"); -} - -/** Typedef for data & section indexes (can be changed to ulong if necessary.) */ -typedef char[] ID; - -package struct MTFormatVersion { - enum VERS : ubyte { // convenient list of all known file format versions - INVALID = 0x00, - MT01 = 0x01, // not yet final - } - /// The current MergeTag version - static const VERS Current = VERS.MT01; - static const char[2] CurrentString = "01"; - - static VERS parseString (char[] str) - in { - assert (str.length == 2); - } body { - if (str[0] == '0' && str[1] == '1') return VERS.MT01; - else return VERS.INVALID; - } -} - -/************************************************************************************************** - * Data class; contains a DataSection class instance for each loaded section of a file. - * - * Stored data is available for direct access via header and sec; all functions are just helper - * functions. - * - * Any class implementing DataSection may be used to store data; by default a DefaultData class is - * used when reading a file. Another class may be used by creating the sections before reading the - * file or passing the reader a function to create the sections (see Reader.dataSecCreator). - * - * Could be a struct, except that structs are value types (not reference types). - */ -class DataSet -{ - DefaultData header; /// Header section. - DataSection[ID] sec; /// Dynamic array of sections - - /// Template to return all sections of a child-class type. - T[ID] getSections (T : DataSection) () { - T[ID] ret; - foreach (ID id, DataSection s; sec) { - T x = cast(T) s; - if (x) ret[id] = x; // if non-null - } - return ret; - } -} - -/** - * Interface for data storage classes, which contain all data-tags loaded from a single section of a - * file. - * - * A class implementing this may implement the addTag function to do whatever it likes with the - * data passed. DefaultData is one implementation which separates this data out into supported - * types and stores it appropriately (allowing merging with existing entries by keeping whichever - * tag was last loaded), while ignoring unsupported types. A different - * implementation could filter out the tags desired and use them directly, and ignore the rest. - * - * The tango.scrapple.text.convert.parseTo module provides a useful set of templated functions to - * convert the data accordingly. It is advised to keep the type definitions as defined in the file- - * format except for user-defined types, although this isn't necessary for library operation - * (addTag and writeAll are solely responsible for using and setting the type, ID and data fields). - * - * Another idea for a DataSection class: - * Use a void*[ID] variable to store all data (may also need a type var for each item). - * addTag should call a templated function which calls parse then casts to a void* and stores the data. - * Use a templated get(T)(ID) method which checks the type and casts to T. - */ -interface DataSection -{ - /** Delegate passed to writeAll. */ - typedef void delegate (char[],ID,char[]) ItemDelg; - - /** Handles parsing of data items for all recognised types. - * - * Should ignore unsupported types/unwanted tags. - * - * TextExceptions (thrown by parseTo/parseFrom) are caught and a warning logged; execution - * then continues (so the offending tag gets dropped). */ - void addTag (char[],ID,char[]); - - /** Responsible for getting all data tags saved. - * - * writeAll should call the ItemDelg once for each tag to be saved with parameters in the same - * form as received by addTag (char[] type, ID id, char[] data). */ - void writeAll (ItemDelg); -} - -debug (mdeUnitTest) unittest { // Only covers DataSet really. - DataSet ds = new DataSet; - ds.sec[cast(ID)"test"] = new DefaultData; - assert (ds.getSections!(DefaultData)().length == 1); - ds.sec[cast(ID)"test"].addTag ("int",cast(ID)"T"," -543 "); - assert (ds.getSections!(DefaultData)()[cast(ID)"test"]._int[cast(ID)"T"] == -543); - - logger.info ("Unittest complete."); -}
--- a/mde/mergetag/defaultdata.d Mon Feb 25 11:11:30 2008 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,160 +0,0 @@ -/// This module contains the DefaultData class, and some notes possibly useful for implementing -/// other types of DataSection. -module mde.mergetag.defaultdata; - -public import mde.mergetag.dataset; -import mde.mergetag.exception; - -import tango.scrapple.text.convert.parseTo : parseTo; -import tango.scrapple.text.convert.parseFrom : parseFrom; - -/** -* Default DataSection class. -* -* Supports most of the basic types supported by D (excluding cent/ucent and imaginary/complex -* types) and array versions of each of these types, plus arrays of strings. -* -* Extending the class to support more types, even custom types, shouldn't be particularly difficult -* provided mde.text.parseTo and mde.text.parseFrom are extended to support the new types. -*/ -/* The implementation now uses a fair bit of generic programming. Adjusting the types supported -* should be as simple as adjusting the list dataTypes, and possibly implemting new conversions in -* parseFrom and parseTo if you add new types (e.g. for cent or imaginary/complex types, or user types). -* -* There shouldn't really be any need to adjust the implementation, except perhaps to add new -* functions to the class (such as another type of output where the delegate used in writeAll isn't -* enough). -*/ -class DefaultData : DataSection -{ - //BEGIN META - /* These functions are used to generate code. Compile-time functions rather than templates are - * used because they are easier to write and understand. Mixins are used to compile the resultant - * code. Must be declared before used since forward references aren't supported for compile-time - * functions. */ - - // Generate the correct name for each variable type. - static char[] varName (char[] type) { - char[] append = ""; - while (type.length >= 2 && type[$-2..$] == "[]") { - type = type[0..$-2]; - append ~= "A"; - } - return "_" ~ type ~ append; - } - - // Int-to-string converter, which may not be efficient but will run at compile time. - static char[] int2str (uint i) { - char[] ret; - const digits = "0123456789"; - if (i == 0) ret = "0"; - else for (; i > 0; i /= 10) ret = digits[i%10] ~ ret; - return ret; - } - - // Generate the code for variable declarations. - static char[] declerations (char[][] types) { - char[] ret = ""; - foreach (char[] type; types) ret ~= type ~ "[ID]\t" ~ varName(type) ~ ";\n"; - return ret; - } - - // Purely to add indentation. Could just return "" without affecting functionality. - static char[] indent (uint i) { - char[] ret; - for (; i > 0; --i) ret ~= " "; - // This is not executable at compile time: - //ret.length = i * 4; // number of characters for each indentation - //ret[] = ' '; // character to indent with - return ret; - } - - /* Generates a binary search algorithm. - * - * Currently this is tailored to it's particular use (addTag). */ - static char[] binarySearch (char[] var, char[][] consts, int indents = 0) { - if (consts.length > 3) { - return indent(indents) ~ "if (" ~ var ~ " <= \"" ~ consts[$/2 - 1] ~ "\") {\n" ~ - binarySearch (var, consts[0 .. $/2], indents + 1) ~ - indent(indents) ~ "} else {\n" ~ - binarySearch (var, consts[$/2 .. $], indents + 1) ~ - indent(indents) ~ "}\n"; - } else { - char[] ret; - foreach (c; consts) { - ret ~= indent(indents) ~ "if (" ~ var ~ " == \"" ~ c ~ "\") {\n" ~ - indent(indents+1) ~ varName(c) ~ "[id] = parseTo!(" ~ c ~ ") (dt);\n" ~ - indent(indents) ~ "} else "; - } - ret = ret[0..$-6]; // remove last else - return ret; - } - } - - // Generates the code to write data members (writeAll). - static char[] writeVars () { - char[] code = ""; - foreach (i,type; dataTypes) { - code ~= "foreach (id, dt; " ~ varName(type) ~ ") itemdlg (dataTypes[" ~ int2str(i) ~ "], id, parseFrom!(" ~ type ~ ")(dt));\n"; - } - return code; - } - //END META - - /** Data Members - * - * These types are all stored directly, as below, are available for direct access. The variable - * names are created dynamically at compile-time based on the dataTypes list. - * ------------------ - * int[ID] _int; // name is type prefixed by _ - * char[][ID] _charA; // [] is replaced by A - * ------------------ - * - * An alternative access method is to use the provided templates: - * -------------------- - * template Arg(T) { - * alias Name Arg; - * } - * - * type y = Arg!(type).Arg; // example of use - * -------------------- - * Note: trying to use Arg!(type) to implicitly refer to Arg!(type).Arg causes compiler errors - * due to the "alias Name Arg;" statement actually being a mixin. - */ - const char[][] dataTypes = ["bool","bool[]", - "byte","byte[]", - "char","char[]","char[][]", - "double","double[]", - "float","float[]", - "int","int[]", - "long","long[]", - "real","real[]", - "short","short[]", - "ubyte","ubyte[]", - "uint","uint[]", - "ulong","ulong[]", - "ushort","ushort[]"]; - - mixin (declerations (dataTypes)); // Declare all the variables. - - void addTag (char[] type, ID id, char[] dt) { /// Supports all types listed in dataTypes. - mixin (binarySearch ("type", dataTypes)); - } - - void writeAll (ItemDelg itemdlg) { /// Supports all types listed in dataTypes. - mixin (writeVars ()); - } - - /* These make no attempt to check Arg is valid. - * But if the symbol doesn't exist the complier will throw an error anyway, e.g.: - * Error: identifier '_boolAA' is not defined - */ - template Arg(T : T[]) { - const ArgString = Arg!(T).ArgString ~ `A`; - mixin(`alias `~ArgString~` Arg;`); - } - template Arg(T) { - const ArgString = `_` ~ T.stringof; - mixin(`alias `~ArgString~` Arg;`); - } -}
--- a/mde/mergetag/doc/file-format-binary.txt Mon Feb 25 11:11:30 2008 +0000 +++ b/mde/mergetag/doc/file-format-binary.txt Fri Mar 07 17:51:02 2008 +0000 @@ -3,3 +3,9 @@ BOM --- a Byte Order Mark should be used to determin endianness (MT01 (or other version) in bytes, but converted to two ushorts to detect endianness?) + +File should then consist of sections: + +Header data including an address for the header section data if included. + +Sections list. Include a list of sections with identifiers and addresses, sorted by identifier and in a suitible format to easily be converted to a D hash-map. Addresses for each section should consist of both a start and an end address; the end address should be checked upon reading the section. In addition the start address must be checked against the end of file to avoid security vulnerabilities with reading other memory blocks.
--- a/mde/mergetag/exception.d Mon Feb 25 11:11:30 2008 +0000 +++ b/mde/mergetag/exception.d Fri Mar 07 17:51:02 2008 +0000 @@ -44,7 +44,7 @@ } } -/** Thrown by addTag (in classes implementing dataset.DataSection) when a data parsing error occurs +/** Thrown by addTag (in classes implementing IDataSection) when a data parsing error occurs * (really just to make whoever called addTag to log a warning saying where the error occured). */ class MTaddTagParseException : MTException { this () { @@ -65,3 +65,11 @@ super ("No dataset"); } } + +/// Thrown when attempting to use an unimplemented part of the package +/// Really, just until MTB stuff is implemented +class MTNotImplementedException : MTException { + this () { + super ("Functionality not implemented!"); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mde/mergetag/iface/IDataSection.d Fri Mar 07 17:51:02 2008 +0000 @@ -0,0 +1,51 @@ +/** This module contains the interface IDataSection used by DataSet. +* +* It has been given its own module to avoid cyclic dependancies and separate out the functionality +* of mergetag. +* +* Also some base mergetag symbols have been moved here. +*/ +module mde.mergetag.iface.IDataSection; + +/** Typedef for data & section indexes (can be changed to ulong if necessary.) */ +typedef char[] ID; + +/** + * Interface for data storage classes, generally called DataSections, which contain all data-tags + * loaded from a single section of a file. + * + * A class implementing this may implement the addTag function to do whatever it likes with the + * data passed. DefaultData is one implementation which separates this data out into supported + * types and stores it appropriately (allowing merging with existing entries by keeping whichever + * tag was last loaded), while ignoring unsupported types. A different + * implementation could filter out the tags desired and use them directly, and ignore the rest. + * + * The tango.scrapple.text.convert.parseTo module provides a useful set of templated functions to + * convert the data accordingly. It is advised to keep the type definitions as defined in the file- + * format except for user-defined types, although this isn't necessary for library operation + * (addTag and writeAll are solely responsible for using and setting the type, ID and data fields). + * + * Another idea for a DataSection class: + * Use a void*[ID] variable to store all data (may also need a type var for each item). + * addTag should call a templated function which calls parse then casts to a void* and stores the data. + * Use a templated get(T)(ID) method which checks the type and casts to T. + */ +interface IDataSection +{ + /** Delegate passed to writeAll. */ + typedef void delegate (char[],ID,char[]) ItemDelg; + + /** Handles parsing of data items for all recognised types. + * + * Should ignore unsupported types/unwanted tags. + * + * TextExceptions (thrown by parseTo/parseFrom) are caught and a warning logged; execution + * then continues (so the offending tag gets dropped). */ + void addTag (char[],ID,char[]); + + /** Responsible for getting all data tags saved. + * + * writeAll should call the ItemDelg once for each tag to be saved with parameters in the same + * form as received by addTag (char[] type, ID id, char[] data). */ + void writeAll (ItemDelg); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mde/mergetag/iface/IReader.d Fri Mar 07 17:51:02 2008 +0000 @@ -0,0 +1,22 @@ +/** +* Interface for readers. +*/ +module mde.mergetag.iface.IReader; + +import mde.mergetag.DataSet; + +import tango.util.collection.model.View : View; + +/** Interface for all mergetag readers (MTTReader etc.). +*/ +interface IReader { + DataSet dataset (); /// Get the DataSet + void dataset (DataSet); /// Set the DataSet + + void dataSecCreator (IDataSection delegate (ID)); /// Set the dataSecCreator + + uint[] getSectionNames (); /// Get identifiers for all sections + void read (); /// Commence reading + void read (ID[] secSet); /// ditto + void read (View!(ID) secSet); /// ditto +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mde/mergetag/iface/IWriter.d Fri Mar 07 17:51:02 2008 +0000 @@ -0,0 +1,33 @@ +/** +* Interface for writers. +*/ +module mde.mergetag.iface.IWriter; + +import mde.mergetag.DataSet; + + +/** Interface for all mergetag writers (MTTWriter etc.). +*/ +interface IWriter { + DataSet dataset (); /// Get the DataSet + void dataset (DataSet); /// Set the DataSet + + void write (); /// Commence writing +} + +/** +* Enumeration for specifying the writing method ("Params" section shows possible values). +* +* Params: +* FromExtension = Determine writing format from file name extension (must be one of .mtb or .mtt). +* Binary = Use binary mode (adds extension .mtb without checking for an existing extension). +* Text = Use text mode (adds extension .mtt without checking for an existing extension). +* Both = Write simultaneously in binary and text modes (with appropriate extensions added to each +* file name. +*/ +enum WriterMethod : byte { + FromExtension = -1, + Binary = 1, + Text = 2, + Both = 3 +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mde/mergetag/internal.d Fri Mar 07 17:51:02 2008 +0000 @@ -0,0 +1,20 @@ +/// Contains functions/data structures used internally by mergetag. +module mde.mergetag.internal; + +package abstract class MTFormatVersion { + enum VERS : ubyte { // convenient list of all known file format versions + INVALID = 0x00, + MT01 = 0x01, // not yet final + } + /// The current MergeTag version + static const VERS Current = VERS.MT01; + static const char[2] CurrentString = "01"; + + static VERS parseString (char[] str) + in { + assert (str.length == 2); + } body { + if (str[0] == '0' && str[1] == '1') return VERS.MT01; + else return VERS.INVALID; + } +}
--- a/mde/mergetag/mtunittest.d Mon Feb 25 11:11:30 2008 +0000 +++ b/mde/mergetag/mtunittest.d Fri Mar 07 17:51:02 2008 +0000 @@ -1,9 +1,10 @@ /// This module provides a unittest for mergetag. module mde.mergetag.mtunittest; -import mde.mergetag.read; -import mde.mergetag.write; -import mde.mergetag.defaultdata; +import mde.mergetag.Reader; +import mde.mergetag.Writer; +import mde.mergetag.DataSet; +import mde.mergetag.DefaultData; import tango.scrapple.text.convert.parseTo : parseTo; import tango.scrapple.text.convert.parseFrom : parseFrom; @@ -13,14 +14,14 @@ debug (mdeUnitTest) { private Logger logger; static this() { - logger = Log.getLogger ("mde.mergetag.unittest"); + logger = Log.getLogger ("mde.mergetag.mtunittest"); } unittest { /* This does a basic write-out and read-in test for each type with its default value. * Thus it provides some basic testing for the whole mergetag package. */ - const file = "unittest.mtt"; + const file = "unittest"; const ID UT_ID = cast (ID) "mdeUT"; const headInfo = "mde Unit Test"; @@ -41,11 +42,11 @@ } mixin (genUTCode()); // Add an entry to dd for each type - // FIXME: if/when binary writing is available, write in both formats - IWriter w = makeWriter (file, dsW); + IWriter w = makeWriter (file, dsW, WriterMethod.Both); w.write(); - Reader r = new Reader (file, null, true); + // FIXME: when binary writing is supported, read both formats and check + IReader r = makeReader (file~".mtt", null, true); r.read(); DataSet dsR = r.dataset; @@ -56,7 +57,7 @@ assert (p); assert (*p == headInfo); - DataSection* sec_p = (UT_ID in dsR.sec); + IDataSection* sec_p = (UT_ID in dsR.sec); assert (sec_p); DefaultData secR = cast(DefaultData) *sec_p; assert (secR !is null);
--- a/mde/mergetag/read.d Mon Feb 25 11:11:30 2008 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,414 +0,0 @@ -/************************************************************************************************** - * This module contains all reading functions, for both binary and text MergeTag files. - * - * It publically imports mde.mergetag.dataset. - *************************************************************************************************/ - -module mde.mergetag.read; - -// package imports -public import mde.mergetag.dataset; -import mde.mergetag.defaultdata; -import mde.mergetag.exception; - -import tango.core.Exception; - -// tango imports -import tango.io.UnicodeFile; -import Util = tango.text.Util; -import ConvInt = tango.text.convert.Integer; -import tango.util.collection.model.View : View; -import tango.util.collection.ArrayBag : ArrayBag; -import tango.util.collection.HashSet : HashSet; -import tango.util.log.Log : Log, Logger; - -// TODO: allow compressing with zlib for both binary and text? (.mtz, .mtt, .mtb extensions) - -/** - * Class for reading a file. - * - * Use as: - * ----------------------- - * Reader foo; - * try { - * foo = new Reader("foo.mtt"); - * foo.read(); - * } - * catch (MTException) {} - * // get your data from foo.dataset. - * ----------------------- - * - * Throws: - * $(TABLE - * $(TR $(TH Exception) $(TH Thrown when)) - * $(TR $(TD MTFileIOException) $(TD An error occurs while opening the file)) - * $(TR $(TD MTFileFormatException) $(TD The file doesn't start with a recognised header/version)) - * $(TR $(TD MTSyntaxException) $(TD A file syntax error occurs)) - * $(TR $(TD MTException) $(TD An unexpected error occurs)) - * ) - * Note that all exceptions extend MTException and when any exception is thrown the class is - * rendered unusable: any subsequent calls to read will be ignored. - * - * Threading: Separate instances of Reader should be thread-safe provided access to the same - * dataset is synchronized; i.e. no two readers refering to the same dataset should run - * simultaneously. (The Reader class could be made thread-safe w.r.t. datasets, but - * performance-wise I doubt it would be worth it.) - * Do not run a single instance of Reader in multiple threads simultaneously. - */ -class Reader -{ -//BEGIN DATA - /** - A container for all read data. - - This may be accessed from here; however it may be preferable to use an external reference - (passed to the class on initialisation). - */ - DataSet dataset; - - /** A delegate for creating new DataSections within the dataset. - * - * Allows a user-made class to be used in the DataSet instead of DefaultData. Also allows an - * existing class instance to be used instead of a new one. - * - * This works by supplying a function which returns a reference to an instance of a class - * implementing DataSection. The function is passed the ID of the new section and may use this - * to use different DataSection classes for different sections. - */ - DataSection delegate (ID) dataSecCreator = null; - -private: - static Logger logger; - - // Non-static symbols: - final char[] ErrFile; // added after ErrInFile to do the same without the "in " bit. - final char[] ErrInFile; // something like "in \"path/file.mtt\"" - - final char[] fbuf; // file is read into this - MTFormatVersion.VERS fileVer = MTFormatVersion.VERS.INVALID; // Remains INVALID until set otherwise by CTOR. - - uint endOfHeader; - bool allRead = false; // true if endOfHeader == fbuf.length or read([]) has run - bool fatal = false; // a fatal file error occured; don't try to recover - /* If the file is scanned for sections, the starting position of all sections are stored - * in secTable. If this is empty, either no sections exist (and endOfHeader == fbuf.length) - * or a section scan has not been run (read() with no section names doesn't need to do so). - */ - struct SecMD { // sec meta data - static SecMD opCall (uint _pos, bool _read) { - SecMD ret; - ret.pos = _pos; - ret.read = _read; - return ret; - } - uint pos; // position to start reading - bool read; // true if already read - } - SecMD [ID] secTable; -//END DATA - -//BEGIN METHODS: CTOR / DTOR - static this () { - logger = Log.getLogger ("mde.mergetag.read.Reader"); - } - - /** Tries to open file path and read it into a buffer. - * - * Params: - * path = The name or FilePath of the file to open. - * Standard extensions are .mtt and .mtb for text and binary files respectively. - * dataset_ = If null create a new DataSet, else use existing DataSet *dataset_ and merge read - * data into it. - * rdHeader = If true, read the header like a standard section. Doesn't read the header by - * default since if it's not requested it's likely not wanted. - * - * Memory: - * This currently works by loading the whole file into memory at once. This should be fine most - * of the time, but could potentially be a problem. Changing this would mean significantly - * changes to the way the code works. - */ - /* Ideas for implementing a partial-loading memory model: - * Use a conduit directly. - * Use a fiber to do the parsing; let it switch back when it runs out of memory. - * Redesign the code so it never needs to look backwards in the buffer? - * - * Major problem: reading only some sections and keeping references to other sections - * would no longer be possible. - */ - public this (char[] path, DataSet* dataset_ = null, bool rdHeader = false) { - this (new FilePath (path), dataset_, rdHeader); - } - /** ditto */ - public this (PathView path, DataSet* dataset_ = null, bool rdHeader = false) { - // Create a dataset or use an existing one - if (dataset_) dataset = *dataset_; - else dataset = new DataSet(); - - // Open & read the file - try { // Supports unicode files with a BOM; defaults to UTF8 when there isn't a BOM: - scope file = new UnicodeFile!(char) (path, Encoding.Unknown); - fbuf = cast(char[]) file.read(); - } catch (Exception e) { - throwMTErr ("Error reading file: " ~ e.msg, new MTFileIOException); - } - // Remember the file name so that we can report errors (somewhat) informatively: - ErrFile = path.path ~ path.file; - ErrInFile = " in \"" ~ ErrFile ~ '"'; - - // Version checking & matching header section tag: - if (fbuf.length < 6 || fbuf[0] != '{' || fbuf[1] != 'M' || fbuf[2] != 'T' || fbuf[5] != '}') - throwMTErr("Not a valid MergeTag text file" ~ ErrInFile, new MTFileFormatException); - fileVer = MTFormatVersion.parseString (fbuf[3..5]); - if (fileVer == MTFormatVersion.VERS.INVALID) - throwMTErr("Unrecognised MergeTag version: MT" ~ fbuf[3..5] ~ ErrInFile, new MTFileFormatException); - - // Header reading/skipping: - if (rdHeader) { // only bother actually reading it if it was requested - // If already existing, merge. - if (!dataset.header) dataset.header = new DefaultData; - endOfHeader = parseSection (6, cast(DataSection*) &dataset.header); - } - else endOfHeader = parseSection (6,null); - } -//END METHODS: CTOR / DTOR - -//BEGIN METHODS: PUBLIC - /// Scans for sections if not already done and returns a list of IDs. - public uint[] getSectionNames () { - if (fatal) return []; - if (!secTable.length) - for (uint pos = endOfHeader; pos < fbuf.length;) { - ID id = fbufReadSecMarker (pos); - secTable[id] = SecMD(pos,false); // add to table - pos = parseSection (pos, null); - } - return cast(uint[]) secTable.keys; - } - - /** Reads (some) sections of the file into data. Note that sections will never be _read twice. - * - * To be more accurate, the file is copied into a buffer by this(). read() then parses the - * contents of this buffer, and stores the contents in dataset. - * - * Each section read is stored in a DataSection class. By default this is an instance of - * DefaultData; this can be customised (see setDataSectionCreator). - * - * If secSet is non-empty, reading is restricted to sections given in secSet, otherwise all - * sections are read. Sections given in secSet but not found in the file are not reported as an - * error. Suggested: supply a HashSet!(uint) as the View!(ID). An ArrayBag!(ID) as used is not a - * good choice, except that in this case it's empty. - * - * Merging: - * Where a section already exists in the DataSet (when either the section is given more than - * once in the file, or it was read from a different file by another reader) it is merged. - * Entries already in the DataSet take priority. - * - * Performance: - * Note that loading only desired sections like this still parses the sections not - * read (although it does not try to understand the type or data fields), so there is only a - * small performance advantage to this where other sections do exist in the file. There is also - * some overhead in only partially reading the file to keep track of where other sections are so - * that the entire file need not be re-read if further (or all remaining) sections are read - * later. - */ - public void read (ID[] secSet) { - HashSet!(ID) hs = new HashSet!(ID); - foreach (id; secSet) hs.add(id); - read (hs); - } - public void read (View!(ID) secSet = new ArrayBag!(ID)) { /** ditto */ - /* Look for a section; return it if it exists otherwise create a new section: - * use dataSecCreator if it exists or just create a DefaultData if not. - */ - DataSection getOrCreateSec (ID id) { - DataSection* i = id in dataset.sec; - if (i) return *i; - return (dataset.sec[id] = (dataSecCreator !is null) ? dataSecCreator(id) : new DefaultData); - } - - if (allRead || fatal) return; // never do anything in either case - if (secSet.size) { - if (secTable.length) { - foreach (ID id; secSet) { - SecMD* psmd = id in secTable; - if (psmd && !psmd.read) { // may not exist - DataSection ds = getOrCreateSec (id); - parseSection (psmd.pos, &ds); - psmd.read = true; - } - } - } else { - for (uint pos = endOfHeader; pos < fbuf.length;) { - ID id = fbufReadSecMarker (pos); - secTable[id] = SecMD(pos,false); // add to table - if (secSet.contains(id)) { - DataSection ds = getOrCreateSec (id); - pos = parseSection (pos, &ds); - secTable[id].read = true; - } else - pos = parseSection (pos, null); // skip section - } - } - } else { - if (secTable.length) { - foreach (ID id, ref SecMD smd; secTable) { - if (!smd.read) { - DataSection ds = getOrCreateSec (id); - parseSection (smd.pos, &ds); - smd.read = true; - } - } - } else { // this time we don't need to use secTable - for (uint pos = endOfHeader; pos < fbuf.length;) { - ID id = fbufReadSecMarker (pos); - DataSection ds = getOrCreateSec (id); - pos = parseSection (pos, &ds); - } - } - allRead = true; - } - } -//END METHODS: PUBLIC - -//BEGIN METHODS: PRIVATE - /* Reads a section, starting from index pos, finishing at the next section marker (returning - the position of the start of the marker). pos should start after the section marker. - - After analysing tags, the function passes the type, ID and data to addTag. - - NOTE: from performance tests on indexing char[]'s and dereferencing char*'s, the char*'s are - slightly faster, but a tiny difference isn't worth the extra effort/risk of using char*'s. - */ - private uint parseSection (uint pos, DataSection* dsec) { - /* Searches fbuf starting from start to find one of <=>| and stops at its index. - - If quotable then be quote-aware for single and double quotes. - Note: there's no length restriction for the content of the quote since it could be a single - non-ascii UTF-8 char which would look like several chars. - */ - void fbufLocateDataTagChar (inout uint pos, bool quotable) { - for (; pos < fbuf.length; ++pos) { - if ((fbuf[pos] >= '<' && fbuf[pos] <= '>') || fbuf[pos] == '|') return; - else if (quotable) { - char c = fbuf[pos]; - if (c == '\'' || c == '"') { - ++pos; - while (fbuf[pos] != c) { - if (fbuf[pos] == '\\') ++pos; // escape seq. - fbufIncrement(pos); - } - } - } - } - } - - bool comment = false; // preceding char was ! - for (; pos < fbuf.length; ++pos) { - if (Util.isSpace(fbuf[pos])) continue; // whitespace - else if (fbuf[pos] == '<') { // data tag - char[] ErrDTAG = "Bad data tag format: not <type|id=data>" ~ ErrInFile; - - fbufIncrement (pos); - - // Type section of tag: - uint pos_s = pos; - fbufLocateDataTagChar (pos, false); // find end of type section - if (fbuf[pos] != '|') throwMTErr (ErrDTAG, new MTSyntaxException); - char[] type = fbuf[pos_s..pos]; - - fbufIncrement (pos); - - // ID section of tag: - pos_s = pos; - fbufLocateDataTagChar (pos, false); // find end of type section - if (fbuf[pos] != '=') throwMTErr (ErrDTAG, new MTSyntaxException); - ID tagID = cast(ID) fbuf[pos_s..pos]; - - fbufIncrement (pos); - - // Data section of tag: - pos_s = pos; - fbufLocateDataTagChar (pos, true); // find end of data section - if (fbuf[pos] != '>') throwMTErr (ErrDTAG, new MTSyntaxException); - char[] data = fbuf[pos_s..pos]; - - if (!comment && dsec != null) { - type = Util.trim(type); - try { - dsec.addTag (type, tagID, data); - } - catch (TextException e) { - logger.warn ("TextException while reading " ~ ErrFile ~ ":"); // following a parse error - logger.warn (e.msg); - } - catch (Exception e) { - logger.error ("Unknown error occured" ~ ErrInFile ~ ':'); - logger.error (e.msg); - throwMTErr (e.msg); // Fatal to Reader - } - } else comment = false; // cancel comment status now - } - else if (fbuf[pos] == '{') { - if (comment) { // simple block comment - uint depth = 0; // depth of embedded comment blocks - while (true) { - fbufIncrement (pos); - if (fbuf[pos] == '}') { - if (depth == 0) break; - else --depth; - } else if (fbuf[pos] == '{') - ++depth; - } - comment = false; // end of this comment - } else { - return pos; // next section coming up; we are done - } - } - else if (fbuf[pos] == '!') { // possibly a comment; check next char - comment = true; // starting a comment (or an error) - // variable is reset at end of comment - } else // must be an error - throwMTErr ("Invalid character (or sequence starting \"!\") outside of tag" ~ ErrInFile, new MTSyntaxException); - } - // if code execution reaches here, we're at EOF - // possible error: last character was ! (but don't bother checking since it's inconsequential) - return pos; - } - - /* Parses fbuf for a section marker. Already knows fbuf[pos] == '{'. - */ - private ID fbufReadSecMarker (inout uint pos) { - // at this point pos is whatever a parseSection run returned - // since we haven't hit EOF, fbuf[pos] MUST be '{' so no need to check - fbufIncrement(pos); - - uint start = pos; - for (; pos < fbuf.length; ++pos) - if (fbuf[pos] == '}' || fbuf[pos] == '{') break; - - if (pos >= fbuf.length || fbuf[pos] != '}') - throwMTErr ("Bad section tag format: not {id}" ~ ErrInFile, new MTSyntaxException); - - ID id = cast(ID) fbuf[start..pos]; - fbufIncrement(pos); - return id; - } - - /* Increments pos and checks it hasn't hit fbuf.length . */ - private void fbufIncrement(inout uint pos) { - ++pos; - if (pos >= fbuf.length) throwMTErr("Unexpected EOF" ~ ErrInFile, new MTSyntaxException); - } - - private void throwMTErr (char[] msg, MTException exc = new MTException) { - fatal = true; // if anyone catches the error and tries to do anything --- we're dead now - logger.error (msg); // report the error - throw exc; // and signal our error - } -//END METHODS: PRIVATE - - /+ A unittest here is really not practical since a file must be read from. - + A unittest is included in defaultdata.d . - unittest {} - +/ -}
--- a/mde/mergetag/write.d Mon Feb 25 11:11:30 2008 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,232 +0,0 @@ -/************************************************************************************************** - * This module contains all writing functions, for both binary and text MergeTag files. - * - * Files can be written in a text or binary form; binary is faster and smaller while text allows - * editing with an ordinary text editor. TextWriter and BinaryWriter are the main classes, both of - * which implement the interface IWriter. DualWriter is another class implementing IWriter, which - * contains a private instance of a TextWriter and a BinaryWriter and implements all methods in the - * interface simply by chaining the appropriate method from each of these classes, thus performing - * two writes at once. - * - * Any of these three classes may be used directly, or makeWriter may be invoked to create an - * instance of the appropriate class. - *************************************************************************************************/ -module mde.mergetag.write; - -// package imports -public import mde.mergetag.dataset; -import mde.mergetag.exception; - -// tango imports -import tango.core.Exception; -import tango.io.FileConduit; -import tango.io.Buffer : Buffer, IBuffer; -import tango.io.Print : Print; -import convInt = tango.text.convert.Integer; -import tango.util.log.Log : Log, Logger; - -private Logger logger; -static this () { - logger = Log.getLogger ("mde.mergetag.write"); -} - -/** - * Enumeration for specifying the writing method ("Params" section shows possible values). - * - * Params: - * Unspecified = If the filename ends with one of .mtb or .mtt, the file is written in - * that format. Otherwise a binary mode is assumed. - * Binary = Use binary mode (default extension: .mtb or no extension). - * Text = Use text mode (default extension: .mtt; as with the above it is not automatically added). - * Both = -*/ -enum WriterMethod : byte { - Unspecified = -1, - Binary = 1, - Text = 2, - Both = 3 -} - -/** Method to create and return either a TextWriter or a BinaryWriter, depending primalily on the - * method parameter and using the filename extension as a fallback. - * - * An exception is thrown if neither test can deduce the writing method. - * - * Use as: - * ----------------------- - * DataSet dataset; // contains data to write - * IWriter foo; - * try { - * foo = makeWriter("foo.mtt", dataset); - * foo.write(); - * } - * catch (MTException) {} - * ----------------------- - * - * Throws: - * MTFileFormatException if unable to determine writing format or use requested format. - */ -IWriter makeWriter (char[] path, DataSet dataset = null, WriterMethod method = WriterMethod.Unspecified) { - return makeWriter (new FilePath (path), dataset, method); -} -/** ditto */ -IWriter makeWriter (PathView path, DataSet dataset = null, WriterMethod method = WriterMethod.Unspecified) -{ - void throwMTErr (char[] msg, Exception exc = new MTException) { - logger.error (msg); - throw exc; - } - - if (method == WriterMethod.Unspecified) { - if (path.ext == "mtt") method = WriterMethod.Text; - else if (path.ext == "mtb") method = WriterMethod.Binary; - else throwMTErr ("Unable to determine writing format: text or binary", new MTFileFormatException); - } - if (method == WriterMethod.Binary) throwMTErr ("Binary writing not supported yet!", new MTFileFormatException); - else if (method == WriterMethod.Text) return new TextWriter (path, dataset); - else if (method == WriterMethod.Both) throwMTErr ("Dual writing not supported yet!", new MTFileFormatException); - else debug throwMTErr ("Bad value of method", new MTFileFormatException); -} - -/// Interface for methods and data necessarily available in TextWriter and/or BinaryWriter. -// FIXME: rename MTWriter or so. -interface IWriter { - /// Get or set the DataSet. - DataSet dataset (); - void dataset (DataSet); /// ditto - - void write (); /// Writing method. -} - -/+ -scope class BinaryWriter : IWriter -{ -} -+/ - -/** - * Class to write a dataset to a file. - * - * Files are only actually open for writing while the write() method is running. - * - * Throws: - * $(TABLE - * $(TR $(TH Exception) $(TH Thrown when)) - * $(TR $(TD MTNoDataSetException) $(TD No dataset is available to write from)) - * $(TR $(TD MTFileIOException) $(TD An error occurs while attemting to write the file)) - * $(TR $(TD MTException) $(TD An unexpected error occurs)) - * ) - * Note that all exceptions extend MTException; unlike Reader exceptions don't block further calls. - */ -class TextWriter : IWriter -{ -//BEGIN DATA - /// Get or set the DataSet. - DataSet dataset () { return dataset; } - void dataset (DataSet ds) /// ditto - { dataset = ds; } - - -private: - // taken from tango.io.Console, mostly to make sure notepad can read our files: - version (Win32) - const char[] Eol = "\r\n"; - else - const char[] Eol = "\n"; - - /* The container where data is written from. */ - DataSet _dataset; - - PathView _path; -//END DATA - -//BEGIN CTOR / DTOR - /** Prepares to open file path for writing. - * - * The call doesn't actually execute any code so cannot fail (unless out of memory). - * - * Params: - * path = The name or FilePath of the file to open. - * Standard extensions are .mtt and .mtb for text and binary files respectively. - * dataset_ = If null create a new DataSet, else use existing DataSet *dataset_ and merge read - * data into it. - */ - public this (char[] path, DataSet ds = null) { - this (new FilePath (path), ds); - } - /** ditto */ - public this (PathView path, DataSet ds = null) { - _path = path; - _dataset = ds; - } -//END CTOR / DTOR - - /** Writes the header and all DataSections. - * - * Firstly writes the header unless it has already been written. Then writes all DataSections - * to the file. Thus if write is called more than once with or without changing the DataSet the - * header should be written only once. This behaviour could, for instance, be used to write - * multiple DataSets into one file without firstly merging them. Note that this behaviour may - * be changed when binary support is added. - */ - public void write () - { - if (!_dataset) throwMTErr ("write(): no Dataset available to write from!", new MTNoDataSetException ()); - - try { - FileConduit conduit; // actual conduit; don't use directly when there's content in the buffer - IBuffer buffer; // write strings directly to this (use opCall(void[]) ) - - // Open a conduit on the file: - conduit = new FileConduit (_path, FileConduit.WriteCreate); - scope(exit) conduit.close(); - - buffer = new Buffer(conduit); // And a buffer - scope(exit) buffer.flush(); - - // Write the header: - buffer ("{MT" ~ MTFormatVersion.CurrentString ~ "}" ~ Eol); - if (_dataset.header !is null) writeSection (buffer, _dataset.header); - - // Write the rest: - foreach (ID id, DataSection sec; _dataset.sec) { - writeSectionIdentifier (buffer, id); - writeSection (buffer, sec); - } - - buffer.flush(); - - } - catch (IOException e) { - throwMTErr ("Error writing to file: " ~ e.msg, new MTFileIOException); - } - catch (Exception e) { - throwMTErr ("Unexpected exception when writing file: " ~ e.msg); - } - } - - private void writeSectionIdentifier (IBuffer buffer, ID id) { - buffer ("{" ~ cast(char[])id ~ "}" ~ Eol); - } - - private void writeSection (IBuffer buffer, DataSection sec) { - void writeItem (char[] tp, ID id, char[] dt) { // actually writes an item - buffer ("<" ~ tp ~ "|" ~ cast(char[])id ~"=" ~ dt ~ ">" ~ Eol); - } - sec.writeAll (&writeItem); - - buffer (Eol); // blank line at end of each section - } - - private void throwMTErr (char[] msg, Exception exc = new MTException) { - logger.error (msg); // report the error - throw exc; // and signal our error - } -} - -/+ -Implement std CTORs which add extensions to each filename and extra CTORs which take two filenames. -scope class DualWriter : IWriter -{ -} -+/
--- a/mde/options.d Mon Feb 25 11:11:30 2008 +0000 +++ b/mde/options.d Fri Mar 07 17:51:02 2008 +0000 @@ -7,9 +7,9 @@ import mde.exception; -import mde.mergetag.read; -import mde.mergetag.write; -import mde.mergetag.dataset; +import mde.mergetag.Reader; +import mde.mergetag.Writer; +import mde.mergetag.DataSet; import mde.mergetag.exception; import tango.scrapple.text.convert.parseTo : parseTo; @@ -30,7 +30,7 @@ * * All options and handling should be non-static to allow possibility of profiles later. */ -class Options : DataSection +class Options : IDataSection { /* The options. * @@ -38,21 +38,23 @@ * * They can be read and set directly by other code (so not thread-safe for writing currently). */ - char[] greeting; // just a testing message char[] L10n; // locale, e.g. en-GB + /* Have any options been changed? Only bother writing if true. + * + * Maybe update this in the future to save if the last file edit was not made by mde. */ + bool changed = false; + /* The code to load/save the values. * * Uses mergetag. */ void addTag (char[] tp, ID id, char[] dt) { if (tp == "char[]") { - if (id == cast(ID)"greeting") greeting = parseTo!(char[]) (dt); - else if (id == cast(ID)"L10n") L10n = parseTo!(char[]) (dt); + if (id == cast(ID)"L10n") L10n = parseTo!(char[]) (dt); } } void writeAll (ItemDelg dlg) { - dlg ("char[]", cast(ID)"greeting", parseFrom!(char[]) (greeting)); dlg ("char[]", cast(ID)"L10n", parseFrom!(char[]) (L10n)); } @@ -65,14 +67,15 @@ private static const secName = cast(ID)"Default"; static void load () { FilePath filePath = new FilePath (fileName); - // If it's not a file or an empty file stop. It should still be created on exit (assuming - // it's not a folder). - if (!filePath.exists || filePath.fileSize == 0u) return; + + // Check it exists (if not it should still be created on exit). + // Don't bother checking it's not a folder, because it could still be a block or something. + if (!filePath.exists) return; - Reader reader; + IReader reader; try { - reader = new Reader(filePath); - reader.dataSecCreator = delegate DataSection(ID) { + reader = new MTTReader(filePath); + reader.dataSecCreator = delegate IDataSection(ID) { return new Options; }; reader.read; @@ -82,7 +85,7 @@ throw new optionsLoadException ("Loading aborted: mergetag exception"); } - DataSection* secP = secName in reader.dataset.sec; + IDataSection* secP = secName in reader.dataset.sec; options = cast(Options) *secP; if (options is null) { throw new optionsLoadException ("Loading failed: expected section not found"); @@ -90,6 +93,8 @@ // else loading was succesful. } static void save () { + if (!options.changed) return; // skip + DataSet ds = new DataSet(); ds.sec[secName] = options;
--- a/mde/scheduler.d Mon Feb 25 11:11:30 2008 +0000 +++ b/mde/scheduler.d Fri Mar 07 17:51:02 2008 +0000 @@ -8,7 +8,7 @@ // in an associative array, returning the ID [on adding fct pointer]. // FIXME: support delegates or not? /// This class can run scheduled functions per frame or every t seconds (sim-time). -class Scheduler +abstract class Scheduler { /** The type of function pointer to be passed to the scheduler. *
--- a/test/mdeTest.d Mon Feb 25 11:11:30 2008 +0000 +++ b/test/mdeTest.d Fri Mar 07 17:51:02 2008 +0000 @@ -4,7 +4,7 @@ // This module should import all mde modules containing unittests: import mde.input.input; -import mde.mergetag.dataset; +import mde.mergetag.DataSet; import mde.mergetag.mtunittest; import mde.exception; import mde.init;