--- a/mde/setup/Init.d	Sun Aug 31 15:59:17 2008 +0100
+++ b/mde/setup/Init.d	Thu Sep 11 11:33:51 2008 +0100
@@ -14,30 +14,50 @@
 along with this program.  If not, see <http://www.gnu.org/licenses/>. */
 
 /**************************************************************************************************
- * Initialisation setup and exit cleanup module.
+ * Initialisation and cleanup (shutdown) module.
  *
- * This module provides an infrastructure for handling much of the initialisation and
- * deinitialisation of the program. It does not, however, provide much of the (de)initialisation
- * code; with the exception of that for the logger.
+ * Program startup follows this sequence: static this() functions, pre-init, init.
+ * Shutdown consists of: cleanup, post-cleanup, static ~this() functions.
+ * 
+ * static this and ~this functions should not use any external resources, such as dynamically
+ * loaded libraries or files, and should not interact with other modules. They should be almost
+ * guaranteed not to fail. Preferably, they shouldn't involve large amounts of memory or
+ * processing, but this is not a requirement.
+ * 
+ * Pre-init: init code written in this module. Generally only prerequisets of most other stages
+ * go here.
+ * 
+ * Init: This is where init code from external modules gets hooked in. Each stage consists of an
+ * initialization function, a corresponding cleanup function, a state, and any dependencies (other
+ * init functions which must be run before this one). Init functions are run according to these
+ * dependencies, potentially threaded simultaeneously with other init functions.
+ * 
+ * Cleanup: Cleanup happens similarly to init for all stages requiring shutdown (according to their
+ * state). The init dependencies are applied in reverse order (so if X depended on Y, Y's cleanup
+ * will not run until X's cleanup has completed), and again the functions may be threaded.
+ * 
+ * Post-cleanup: like pre-init, this is code written in Init.
  *************************************************************************************************/
 module mde.setup.Init;
 
-import mde.setup.init2;     // This module is responsible for setting up some init functions.
-import mde.setup.initFunctions;
+import mde.setup.InitStage;     // Controls external delegates run by init
 import mde.setup.exception;
 
 import mde.lookup.Options;
 import paths = mde.setup.paths;
-import mde.exception;
+import mde.exception;           // optionsLoadException
 
 // tango imports
 import tango.core.Thread;
+import tango.core.sync.Condition;
 import tango.core.Exception;
-import tango.stdc.stringz : fromStringz;
+import tango.util.container.LinkedList;
+
+//import tango.stdc.stringz : fromStringz;
 import tango.io.Console;	// for printing command-line usage
 import TimeStamp = tango.text.convert.TimeStamp, tango.time.WallClock;	// output date in log file
 import tango.util.Arguments;
-import tango.util.log.Log : Log, Logger, Level;
+import tango.util.log.Log;
 import tango.util.log.AppendConsole;
 import tango.util.log.AppendFiles;
 
@@ -48,60 +68,32 @@
 import derelict.freetype.ft;
 import derelict.util.exception;
 
-/**
- * Static CTOR
- *
- * This should handle a minimal amount of functionality where useful. For instance, configuring the
- * logger here and not in Init allows unittests to use the logger.
- */
-static this()
-{
-    Logger root = Log.root;
-    // Set the level here, but set it again once options have been loaded:
-    debug root.level(Logger.Trace);
-    else  root.level(Logger.Info);
-    // Temporarily log to the console (until we've found paths and loaded options):
-    root.add(new AppendConsole);
-}
-static ~this()
-{
-}
 
-/**
+/**************************************************************************************************
  * Init class
  *
  * A scope class created at beginning of the program and destroyed at the end; thus the CTOR
  * handles program initialisation and the DTOR handles program cleanup.
- */
+ *************************************************************************************************/
 scope class Init
 {
-    //private bool failure = false;       // set true on failure during init, so that clean
-    private static Logger logger;
     static this() {
+        // Set up the logger temporarily (until pre-init):
+        Logger root = Log.root;
+        debug root.level(Logger.Trace);
+        else  root.level(Logger.Info);
+        root.add(new AppendConsole);
+        
         logger = Log.getLogger ("mde.setup.Init");
     }
     
-    /** this() − initialisation
-    *
-    * Runs general initialisation code, in a threaded manner where this isn't difficult.
-    * If any init fails, cleanup is still handled by ~this().
-    *
-    * Init order: 1. Pre-init (loads components needed by most init functions). 2. Dynamic library
-    * loading (load any dynamic libraries first, so that if loading fails nothing else need be
-    * done). 3. Init functions (threaded functions handling the rest of initialisation).
-    */
-    /* In a single-threaded function this could be done with:
-    * scope(failure) cleanup;
-    * This won't work with a threaded init function since any threads completing succesfully will
-    * not clean-up, and a fixed list of clean-up functions cannot be used since clean-up functions
-    * must not run where the initialisation functions have failed.
-    * Hence a list of clean-up functions is built similarly to scope(failure) --- see addCleanupFct.
-    */
+    /** this() − pre-init and init */
     this(char[][] cmdArgs)
     {
-        debug logger.trace ("Init: starting");
-        
-        //BEGIN Pre-init (stage init0)
+        /******************************************************************************************
+         * Pre-init - init code written in this module.
+         *****************************************************************************************/
+        debug logger.trace ("Init: starting pre-init");
         //FIXME: warn on invalid arguments, including base-path on non-Windows
         // But Arguments doesn't support this (in tango 0.99.6 and in r3563).
         Arguments args;
@@ -210,23 +202,21 @@
         }
         debug logger.trace ("Init: dynamic libraries loaded");
         //END Load dynamic libraries
-        //END Pre-init
         
         
-        //BEGIN Init (stages init2, init4)
-        /* Call init functions.
-        *
-        * Current method is to try using threads, and on failure assume no threads were actually
-        * created and run functions in a non-threaded manner. */
+        /******************************************************************************************
+         * Init − where init code from external modules gets hooked in.
+         *****************************************************************************************/
+        debug logger.trace ("Init: done pre-init, starting init stages");
         
-        try {
-            if (runStageThreaded (init)) runStageForward (init);
-        }
-        catch (InitStageException) {    // This init stage failed.
-            // FIXME: check DTOR still runs
-            throw new InitException ("An init function failed (see above message(s))");
-        }
-        //END Init
+        // Calculate reverse dependencies of stages:
+        foreach (key,stage_p; stages)
+            foreach (name; stage_p.depends)
+                stages[name].rdepends ~= key;
+        if (miscOpts.numThreads < 1 || miscOpts.numThreads > 64)        // limit to a sensible number of threads
+            miscOpts.set!(int)("numThreads", 4);        // FIXME enforce limit in Options
+        
+        runStages!(true);       // startup delegates
         
         debug logger.trace ("Init: done");
     }
@@ -236,113 +226,153 @@
     {
         debug logger.trace ("Cleanup: starting");
         
-        Options.save(); // save options... do so here for now
+        runStages!(false);      // cleanup delegates
         
-        // General cleanup:
-        try {
-            if (runStageThreaded (cleanup)) runStageReverse (cleanup);
-        }
-        catch (InitStageException) {
-            // Nothing else to do but report:
-            logger.error ("One or more cleanup functions failed!");
-        }
+        Options.save(); // save options... do so here for now
         
         debug logger.trace ("Cleanup: done");
     }
     
-    
-    //BEGIN runStage...
-    private static {
-        /* The following three functions, runStage*, each run all functions in a stage in some order,
-        * catching any exceptions thrown by the functions (although this isn't guaranteed for threads),
-        * and throw an InitStageException on initFailure. */
-    
-        const LOG_IF_MSG = "Init function ";
-        const LOG_CF_MSG = "Cleanup function ";
-        const LOG_F_START = " - running";
-        const LOG_F_END = " - completed";
-        const LOG_F_BAD = " - failed";
-        const LOG_F_FAIL = " - exception: ";
-        /* Runs all functions consecutively, first-to-last.
-        * If any function fails, halts immediately. */
-        void runStageForward (InitStage s) {
-            foreach (func; s.funcs) {
-                if (initFailure) break;
-                try {
-                    debug logger.trace (LOG_IF_MSG ~ func.name ~ LOG_F_START);
-                    func.func();
-                    debug logger.trace (LOG_IF_MSG ~ func.name ~ (initFailure ? LOG_F_BAD : LOG_F_END));
-                } catch (Exception e) {
-                    logger.fatal (LOG_IF_MSG ~ func.name ~ LOG_F_FAIL ~
-                            ((e.msg is null || e.msg == "") ? "(no failure message)" : e.msg) );
-                
-                    setInitFailure();
+    // run init stages or cleanup if startup is false
+    private static void runStages(bool startup) () {
+        LinkedList!(InitStage*) toRun = new LinkedList!(InitStage*);
+        foreach (v; stages) {
+            // Filter only stages with the relevant delegate. It is not checked later that the
+            // delegate exists!
+            // If no init/cleanup function exists, implicit activation/deactivation can occur so
+            // that dependents/dependencies may activate/deactivate.
+            static if (startup) {
+                if (v.state == StageState.INACTIVE) {
+                    if ((*v).init)
+                        toRun.append (v);
+                    else
+                        v.state = StageState.ACTIVE;
                 }
-            }
-            
-            if (initFailure) throw new InitStageException;    // Problem running; abort and cleanup from here.
-        }
-        /* Runs all functions consecutively, last-to-first.
-        * If any function fails, continue until all have been run. */
-        void runStageReverse (InitStage s) {
-            foreach_reverse (func; s.funcs) {
-                try {
-                    debug logger.trace (LOG_CF_MSG ~ func.name ~ LOG_F_START);
-                    func.func();
-                    debug logger.trace (LOG_CF_MSG ~ func.name ~ (initFailure ? LOG_F_BAD : LOG_F_END));
-                } catch (Exception e) {
-                    logger.fatal (LOG_CF_MSG ~ func.name ~ LOG_F_FAIL ~
-                            ((e.msg is null || e.msg == "") ? "(no failure message)" : e.msg) );
-                
-                    setInitFailure();
+            } else {
+                if (v.state == StageState.ACTIVE) {
+                    if (v.cleanup)
+                        toRun.append (v);
+                    else
+                        v.state = StageState.INACTIVE;
                 }
             }
-            if (initFailure) throw new InitStageException;    // Problem running; abort and cleanup from here.
         }
-        /* Tries running functions in a threaded way. Returns false if successful, true if not but
-        * functions should be run without threads. */
-        bool runStageThreaded (InitStage s) {
-            if (!miscOpts.useThreads) return true;  // Use unthreaded route instead
+        debug logger.trace ("Added {} init/cleanup stages", toRun.size);
+        int numWorking = miscOpts.numThreads;
+        bool abortInit = false;
+        Mutex toRunM = new Mutex;       // synchronization on toRun, numWorking
+        Condition toRunC = new Condition(toRunM);       // used by threads waiting for remaining stages' dependencies to be met
         
-            ThreadGroup tg;
-            try {                           // creating/starting threads could fail
-                tg = new ThreadGroup;
-                foreach (func; s.funcs) {   // Start all threads
-                    debug logger.trace (LOG_IF_MSG ~ func.name ~ LOG_F_START);
-                    tg.create(func.func);
-                    debug logger.trace (LOG_IF_MSG ~ func.name ~ (initFailure ? LOG_F_BAD : LOG_F_END));
-                }
-            } catch (ThreadException e) {   // Problem with threading; try without threads
-                logger.error ("Caught ThreadException while trying to create threads:");
-                logger.error (e.msg);
-                logger.info ("Will disable threads and continue, assuming no threads were created.");
+        /* This is a threadable member function to run init stages.
+         * Operation follows:
+         * 1 Look for a stage to run:
+         *   if found:
+         *      notify waiting threads work may be available
+         *      init stage
+         *      goto 1
+         *   if not found:
+         *      if any other threads are working, wait (dependencies may not be met yet)
+         *      if no other threads are working, notify waiting threads and terminate (finished)
+         * When notified, threads start at 1. */
+        void initThreadFct () {
+            debug logger.trace ("initThreadFct: starting");
+            try {       // created as a thread - must not throw exceptions
+            InitStage* stage;
             
-                miscOpts.set!(bool)("useThreads", false);   // Disable threads entirely
-                return true;                // Try again without threads
-            }
-        
-            /* Wait for all threads to complete.
-            *
-            * If something went wrong, we still need to do this before cleaning up.
-            */
-            foreach (t; tg) {
-                try {
-                    t.join (true);
-                } catch (Exception e) {
-                    // Relying on catching exceptions thrown by other threads is a bad idea.
-                    // Hence all threads should catch their own exceptions and return safely.
+            threadLoop: while (true) {      // thread loops until a problem occurs or nothing else can be done
+            // Look for a job:
+            synchronized (toRunM) {
+                --numWorking;           // stopped working: looking/waiting for a job
+                if (abortInit) break threadLoop;  // something went wrong in another thread
+                static if (startup)
+                    int num_rdepends = (stage is null) ? 0 : stage.rdepends.length;
+                else
+                    int num_rdepends = (stage is null) ? 0 : stage.depends.length;
                 
-                    logger.fatal ("Unhandled exception from Init function:");
-                    logger.fatal (e.msg);
-                
-                    setInitFailure ();        // abort (but join other threads first)
+                getStage: while (true) {
+                    auto it = toRun.iterator;   // asserts if toRun is empty
+                    itStages: while (it.next (stage)) {   // get next element of toRun
+                        static if (startup) {
+                            foreach (d; stage.depends)
+                                if (stages[d].state != StageState.ACTIVE)
+                                    continue itStages;  // dependency isn't met (yet)
+                        } else {
+                            foreach (d; stage.rdepends)
+                                if (stages[d].state != StageState.INACTIVE)
+                                    continue itStages;  // reverse dependency isn't unmet (yet)
+                        }
+                        
+                        // All dependencies met
+                        it.remove;
+                        break getStage;
+                    }
+                    
+                    // No stage remaining with all dependencies met
+                    if (!toRun.isEmpty && numWorking)     // still some working so more dependencies may be met later
+                        toRunC.wait;    // wait until another thread finishes
+                    else                // no thread is working, so none of what's left is doable, or nothing's left
+                        break threadLoop;
                 }
+                ++numWorking;           // got a job!
+                if (num_rdepends > 2)   // how many stages depended on the last one run?
+                    toRunC.notifyAll(); // tell all waiting threads there may be work now
+                else if (num_rdepends == 2)
+                    toRunC.notify();    // there's potentially work for this thread and one other
+                // else there won't be additional work so don't notify
             }
             
-            if (initFailure) throw new InitStageException;    // Problem running; abort and cleanup from here.
-            return false;                   // Done successfully
+            // Do a job:
+            try {
+                // FIXME - old stage start&finish trace messages - we don't have a name!
+                debug logger.trace ("InitStage {}: starting", stage);
+                static if (startup)
+                    stage.state = (*stage).init();  // init is a property of a pointer (oh no!)
+                else
+                    stage.state = stage.cleanup();
+                debug logger.trace ("InitStage {}: completed; state: {}", stage, stage.state);
+            } catch (InitStageException e) {
+                debug logger.trace ("InitStage {}: failed", stage);
+                stage.state = e.state;
+                abortInit = true;
+                break threadLoop;
+            } catch (Exception e) {
+                debug logger.trace ("InitStage {}: failed", stage);
+                abortInit = true;
+                break threadLoop;
+            }
+            }
+            } catch (Exception e) {
+                logger.fatal ("Exception in initThreadFct: "~e.msg);
+                abortInit = true;
+            }
+            toRunC.notifyAll(); // Most likely if we're exiting, we should make sure others aren't waiting.
+            debug logger.trace ("initThreadFct: returning");
+            return;
         }
-    //END runStage...
+        
+        // Start miscOpts.NumThreads - 1 threads:
+        try {
+            ThreadGroup g = new ThreadGroup;
+            for (int i = miscOpts.numThreads; i > 1; --i)
+                g.create (&initThreadFct);
+            initThreadFct();    // also run in current thread
+            g.joinAll (false);  // don't rethrow exceptions - there SHOULD NOT be any
+        } catch (ThreadException e) {
+            logger.error ("Exception while using threads: "~e.msg);
+            logger.error ("Disabling threads and attempting to continue.");
+            miscOpts.set!(int)("NumThreads", 1);        // count includes current thread
+            initThreadFct();                            // try with just this thread
+        }       // any other exception will be caught in main() and abort program
+        
+        if (abortInit)
+            throw new InitException ("An init/cleanup function failed (see above message(s))");
+        
+        foreach (stage; toRun)
+            logger.warn ("InitStage {}: was not run due to unmet dependencies", stage);
+    }
+    
+    private static {
+        Logger logger;
         
         void printUsage (char[] progName) {
             Cout ("mde [no version]").newline;
@@ -366,32 +396,90 @@
     }
     
     debug (mdeUnitTest) unittest {
-        /* Fake init and cleanup. Use unittest-specific init and cleanup InitStages to avoid
-        * messing other init/cleanup up. */
-        static InitStage initUT, cleanupUT;
+        logger.trace ("Starting unittest");
+        
+        auto realInit = stages;         // backup the real init stages
+        stages = new typeof(stages);    // an empty test-bed
+        
+        bool init1, init2, init3 = true;
+        bool failed = false;    // set if anything goes wrong
+        StageState s1InitReturns = StageState.ACTIVE;
+        addInitStage ("stg1", delegate StageState() {
+            init1 = true;
+            return s1InitReturns;
+        }, delegate StageState() {
+            init1 = false;
+            return StageState.INACTIVE;
+        });
+        addInitStage ("stg2", delegate StageState() {
+            if (!init1) failed = true;
+            init2 = true;
+            return StageState.ACTIVE;
+        }, delegate StageState() {
+            if (!init1) failed = true;
+            init2 = false;
+            return StageState.INACTIVE;
+        }, ["stg1"]);
+        InitStage s3;
+        s3.init = delegate StageState() {
+            throw new InitStageException (cast(StageState)7);   // not a normal state, but good for a test
+            return StageState.ERROR;
+        };
+        s3.cleanup = delegate StageState() {
+            if (!init1) failed = true;
+            init3 = false;
+            return StageState.INACTIVE;
+        };
+        s3.depends = [ toStageName("stg1") ];
+        s3.state = StageState.ACTIVE;   // already active, so s3.init should not run (first time)
+        addInitStage ("stg3", &s3);
         
-        static bool initialised = false;
-        static void cleanupFunc1 () {
-            initialised = false;
-        }
-        static void cleanupFunc2 () {
-            assert (initialised == true);
-        }
-                
-        static void initFunc () {
-            initialised = true;
-            cleanupUT.addFunc (&cleanupFunc1, "UT cleanup 1");
-            cleanupUT.addFunc (&cleanupFunc2, "UT cleanup 2");
-        }
+        // Stuff normally done in Init.this():
+        // Calculate reverse dependencies of stages:
+        foreach (key,stage_p; stages)
+            foreach (name; stage_p.depends)
+                stages[name].rdepends ~= key;
+        if (miscOpts.numThreads < 1 || miscOpts.numThreads > 64)        // limit to a sensible number of threads
+            miscOpts.set!(int)("numThreads", 4);        // FIXME enforce limit in Options
+        
+        
+        // Run the above.
+        runStages!(true);
+        assert (init1);
+        assert (init2);
+        assert (!failed);
+        foreach (s; stages)
+            assert (s.state == StageState.ACTIVE);
         
-        initUT.addFunc (&initFunc, "UT init");
+        runStages!(false);
+        assert (!init1);
+        assert (!init2);
+        assert (!init3);
+        assert (!failed);
+        foreach (s; stages)
+            assert (s.state == StageState.INACTIVE);
         
-        runStageForward (initUT);
-        assert (initialised);
+        s1InitReturns = StageState.ERROR;
+        // Run again. S2/S3 shouldn't run, S1 won't shut down
+        runStages!(true);
+        assert (init1);
+        assert (!init2);
+        assert (!init3);
+        assert (!failed);
+        runStages!(false);
+        assert (init1); // S1 cleanup won't run
         
-        runStageReverse (cleanupUT);
-        assert (!initialised);
-
+        stages[toStageName("stg1")].state = StageState.INACTIVE;     // hack it back so we can still test
+        s1InitReturns = StageState.ACTIVE;
+        init1 = false;
+        try {
+            runStages!(true);
+            assert (false);             // runStages should throw because s3.init runs now
+        } catch (Exception) {}
+        assert (init1); // s1.init should run first
+        assert (stages[toStageName("stg3")].state == cast(StageState)7);        // set by the exception
+        
+        stages = realInit;      // restore the real init stages
         logger.info ("Unittest complete.");
     }
 }