Mercurial > projects > aid

changeset 3:314d68bafeff

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Backprop and backprop_test added (no testing).
author revcompgeek
date Fri, 11 Apr 2008 18:12:55 -0600
parents 9655c8362b25
children 73beed484455
files trunk/aid/misc.d trunk/aid/nn/multilayer/backprop.d trunk/aid/nn/outputFunctions.d trunk/aid/nn/perceptron.d trunk/backprop_test.d trunk/dsss.conf
diffstat 6 files changed, 286 insertions(+), 34 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/trunk/aid/misc.d	Fri Apr 11 18:12:55 2008 -0600
@@ -0,0 +1,13 @@
+module aid.misc;
+
+import std.random;
+
+class InputException : Exception {
+	this(char[] message){
+		super(message);
+	}
+}
+
+double rnd(){ // The function that should be included in every math library!
+	return (cast(double)rand())/uint.max;
+}
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/trunk/aid/nn/multilayer/backprop.d	Fri Apr 11 18:12:55 2008 -0600
@@ -0,0 +1,170 @@
+module aid.nn.multilevel.backprop;
+
+import aid.nn.outputFunctions;
+import aid.misc;
+import std.random;
+import std.stream;
+
+class Backprop {
+	private uint       numInputs;
+	private float[][][] units;     // Includes the output units. units[layer][unit][inputWeight]
+	private OutputFunctionPtr[] functions;
+	public float learningRate;
+	
+	///Constructor
+	public this(uint numInputs,uint[] numUnits,OutputFunctionPtr[] functions,float value=0.05,bool randomize=true){
+		if(numUnits.length == 0) throw new InputException("numUnits must be greater than 0");
+		if(numUnits.length != functions.length) throw new InputException("numUnits and functions must be the same length");
+		this.numInputs = numInputs;
+		this.functions = functions;
+		initUnitLayer(0,numUnits[0],numInputs,value,randomize);
+		for(int i=1; i<numUnits.length; i++){
+			initUnitLayer(i,numUnits[i],numUnits[i-1],value,randomize);
+		}
+	}
+	
+	// Helper function to initialize a certain layer.
+	private void initUnitLayer(uint layer,uint num,uint numPrev,float value,bool randomize){
+		units[layer].length = num;
+		for(int i=0; i<num; i++){
+			units[layer][i].length = numPrev+1; // include the bias weight
+			for(int j=0; j<numPrev+1; j++){
+				if(randomize) units[layer][i][j] = rnd() * value * 2 - value; // between -value and value
+				else units[layer][i][j] = value;
+			}
+		}
+	}
+	
+	////////////////////////////////////////////////////// Evaluation //////////////////////////////////////////////////////
+	/// Evaluates the neural network.
+	public float[] evaluate(float[] inputs){
+		return evaluateFull(inputs)[$]; // the last item (outputs) of the return value
+	}
+	
+	/// Evaluates the neural network and returns the output from all units.
+	public float[][] evaluateFull(float[] inputs){
+		if(inputs.length != numInputs) throw new InputException("Wrong length of inputs.");
+		float[][] outputs;
+		outputs.length = units.length;
+		outputs[0] = evaluateLayer(0,inputs);
+		for(int i=0; i<units.length; i++){
+			outputs[i] = this.evaluateLayer(i,outputs[i-1]);
+		}
+		return outputs;
+	}
+	
+	// Helper function to evaluate the outputs of a single layer.
+	private float[] evaluateLayer(uint layer,float[] layerInputs){
+		float[] output;
+		output.length = layerInputs.length;
+		for(int i=0; i<layerInputs.length; i++){
+			output[i] = evaluateUnit(layer,i,layerInputs);
+		}
+		return output;
+	}
+	
+	// Helper function to evaluate the output of a single unit.
+	private float evaluateUnit(uint layer, uint unit, float[] layerInputs){
+		float total = units[layer][unit][0]; //bias
+		for(int i=1; i<layerInputs.length; i++){
+			total += layerInputs[i-1] * units[layer][unit][i]; // wi * xi
+		}
+		if(functions[layer] != null) return functions[layer](total); // apply the function (if there is one)
+		else return total; // just return the result instead
+	}
+	
+	
+	////////////////////////////////////////////////////// Training //////////////////////////////////////////////////////
+	/// Trains the neural network.
+	/// TODO:
+	///   Pull error calculation into a separate function.
+	public void train(float[][] allInputs, float[][] allOutputs){
+		if(allInputs.length != allOutputs.length) throw new InputException("allInputs and allOutputs must be the same size");
+		float[][][] weightUpdate;
+		float[][] outputsError;
+		float[][] outputs;
+		float total; //temp variable
+		
+		// Initialize the weightUpdate and outputsError variables
+		weightUpdate.length = units.length;
+		outputsError.length = units.length;
+		for(int i=0; i<weightUpdate.length; i++){
+			weightUpdate[i].length = units[i].length;
+			outputsError[i].length = units[i].length;
+			for(int j=0; j<weightUpdate[i].length; i++){
+				weightUpdate[i][j].length = units[i][j].length;
+			}
+		}
+		
+		
+		// Loop through each of the training examples
+		for(int example=0; example < allInputs.length; example++){
+			outputs = evaluateFull(allInputs[example]);
+			
+			// Computing error of output layer
+			for(int i=0; i<outputs[$].length; i++)
+				outputsError[$][i] = outputs[$][i] * (1 - outputs[$][i]) * (allOutputs[example][i] - outputs[$][i]); // o(1-o)(t-o)
+			
+			// Loop through each of the hidden layers (backwards - BACKpropagation!)
+			for(int i=units.length-2; i >= 0; i--){ // -2 to skip the output layer
+				// loop through the units in each hidden layer
+				for(int j=0; j<units[i].length; j++){
+					total=0;
+					// total up w * e for the units the output of this unit goes into
+					for(int k=0; k<units[i+1].length; k++){
+						total += units[i+1][k][j+1] * outputsError[i+1][k];
+					}
+					// multiply total by o(1-o), store in outputsError
+					outputsError[i][j] = outputs[i][j] * (1 - outputs[i][j]) * total;
+				}
+			}
+			
+			// special case for the units that receive the input values
+			for(int j=0; j<units[0].length; j++){ // unit
+				weightUpdate[0][j][0] += outputsError[0][j]; //bias
+				for(int k=1; k<units[0][j].length; k++){ // input
+					weightUpdate[0][j][k] += outputsError[0][j] * allInputs[example][k-1];
+				}
+			}
+			
+			// Update the weightUpdate array
+			for(int i=1; i<units.length; i++){ // layer
+				for(int j=0; j<units[i].length; j++){ // unit
+					weightUpdate[i][j][0] += outputsError[i][j]; //bias
+					for(int k=1; k<units[i][j].length; k++){ // input
+						weightUpdate[i][j][k] += outputsError[i][j] * outputs[i-1][k-1]; // previous layer, account for bias
+					}
+				}
+			}
+		}
+		
+		// Apply the weightUpdate array to the weights
+		for(int i=0; i<units.length; i++){ // layer
+			for(int j=0; j<units[i].length; j++){ // unit
+				for(int k=0; k<units[i][j].length; k++){ // input
+					units[i][j][k] += this.learningRate * weightUpdate[i][j][k];
+				}
+			}
+		}
+	}
+	
+	/// Calculate the output error
+	float calculateError(float[][] allInputs, float[][] allOutputs){
+		if(allInputs.length != allOutputs.length) throw new InputException("allInputs and allOutputs must be the same size");
+		float[] outputs;
+		float total,temp;
+		for(int i=0; i<allInputs.length; i++){
+			outputs = evaluate(allInputs[i]);
+			if(outputs.length != allOutputs[i].length) throw new InputException("Wrong output length");
+			for(int j=0; j<outputs.length; j++){
+				temp = allOutputs[i][j] - outputs[j];
+				total += temp * temp;
+			}
+		}
+		return 0.5 * total;
+	}
+}
+
+
+
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/trunk/aid/nn/outputFunctions.d	Fri Apr 11 18:12:55 2008 -0600
@@ -0,0 +1,23 @@
+module aid.nn.outputFunctions;
+
+import aid.nn.outputFunctions;
+import std.math;
+
+alias double function(double) OutputFunctionPtr;
+
+enum OutputFunction {
+	Linear=0, Sign, Sigmoid, Tanh
+}
+
+double sign(double y){
+	if(y>0) return 1;
+	return -1;
+}
+
+double sigmoid(double x){
+	return 1/(1+exp(-x));
+}
+
+double tanh(double x){
+	return cast(double)tanh(cast(real)x);
+}
\ No newline at end of file
--- a/trunk/aid/nn/perceptron.d	Sat Apr 05 23:41:30 2008 -0600
+++ b/trunk/aid/nn/perceptron.d	Fri Apr 11 18:12:55 2008 -0600
@@ -1,38 +1,10 @@
 module aid.nn.perceptron;
 
-import std.random;
+import aid.nn.outputFunctions;
+import aid.misc;
 import std.math;
 import std.string;
 
-double rnd(){ // The function that should be included in every math library!
-	return (cast(double)rand())/uint.max;
-}
-
-class InputException : Exception {
-	this(char[] message){
-		super(message);
-	}
-}
-
-// The output functions
-
-alias double function(double) OutputFunction;
-
-double sign(double y){
-	if(y>0) return 1;
-	return -1;
-}
-
-double sigmoid(double x){
-	return 1/(1+exp(-x));
-}
-
-double tanh(double x){
-	return cast(double)tanh(cast(real)x);
-}
-
-// End output functions
-
 
 class perceptron {
 	private int numInputs;
@@ -42,7 +14,7 @@
 	
 	/**
 	 * This is the constructor for loading the neural network from a string.
-	 * 
+	 *
 	 * Params:
 	 *  savedString = The string that was output from the save function.
 	 *
@@ -74,10 +46,10 @@
 	
 	/**
 	 * Evaluates the neural network.
-	 * 
+	 *
 	 * Params:
 	 *  inputs = The set of inputs to evaluate.
-	 * 
+	 *
 	 * Returns: 1 to indicate true, -1 for false
 	 */
 
@@ -117,7 +89,7 @@
 
 	/**
 	 * Calculates the error based on the sum squared error function.
-	 * 
+	 *
 	 * Params:
 	 *  inputs = An array of arrays of all testing inputs.
 	 *  outputs = An array of all the outputs that the cooresponding inputs should have.
@@ -140,3 +112,4 @@
 	}
 }
 
+// TODO: Impliment loading and saving of files
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/trunk/backprop_test.d	Fri Apr 11 18:12:55 2008 -0600
@@ -0,0 +1,71 @@
+module backprop_test;
+
+import aid.nn.multilayer.backprop;
+import aid.nn.outputFunctions;
+import std.stdio;
+
+/+float[][] trainingInputs = [
+	[0,0,0],
+	[0,0,1],
+	[0,1,0],
+	[0,1,1],
+	[1,0,0],
+	[1,0,1],
+	[1,1,0],
+	[1,1,1]];
+
+float[][] trainingOutputs = [
+	[0.1],
+	[0.9],
+	[0.9],
+	[0.1],
+	[0.9],
+	[0.1],
+	[0.1],
+	[0.9]];+/
+
+float[][] trainingInputs = [
+	[0,0],
+	[0,1],
+	[1,0],
+	[1,1]];
+
+float[][] trainingOutputs = [
+	[0.1],
+	[0.9],
+	[0.9],
+	[0.1]];
+
+void main(){
+	Backprop nn = new Backprop(2,[4,1],[&sigmoid,&sigmoid]);
+	
+	float error = 10.0;
+	float[] output;
+	int iter = 0;
+	while(error >= 0.5){
+		error = nn.calculateError(trainingInputs,trainingOutputs);
+		if(iter % 100 == 0){
+			writefln("Iter: %d",iter);
+			for(int i=0; i<trainingInputs.length; i++){
+				output = nn.evaluate(trainingInputs[i]);
+				writef("  %d:", i); printArray(output);
+			}
+			writefln("  Error: %f", error);
+		}
+		nn.train(trainingInputs,trainingOutputs);
+	}
+	writefln("Total Iters: %d",iter);
+	for(int i=0; i<trainingInputs.length; i++){
+		output = nn.evaluate(trainingInputs[i]);
+		writef("  %d:", i); printArray(output);
+	}
+	writefln("  Error: %f", error);
+}
+
+void printArray(float[] array){
+	writef("[");
+	for(int i=0; i<array.length-1; i++){
+		writef("%f, ",array[i]);
+	}
+	writefln("%f]",array[$]);
+}
\ No newline at end of file
--- a/trunk/dsss.conf	Sat Apr 05 23:41:30 2008 -0600
+++ b/trunk/dsss.conf	Fri Apr 11 18:12:55 2008 -0600
@@ -1,7 +1,9 @@
 [aid]
+type=sourcelibrary
 exclude=aid/maze/
 exclude+=aid/containers/
 [ga_code.d]
 [ga_maze.d]
 [mazegen.d]
 [perceptron_test.d]
+[backprop_test.d]