# HG changeset patch
# User revcompgeek
# Date 1208058937 21600
# Node ID 73beed4844556df9e7657e5dab75bb8abc78dbc9
# Parent  314d68bafefff1624509a3f422cddc4890affd90
Backprop working correctly.

diff -r 314d68bafeff -r 73beed484455 trunk/aid/nn/multilayer/backprop.d
--- a/trunk/aid/nn/multilayer/backprop.d	Fri Apr 11 18:12:55 2008 -0600
+++ b/trunk/aid/nn/multilayer/backprop.d	Sat Apr 12 21:55:37 2008 -0600
@@ -4,19 +4,22 @@
 import aid.misc;
 import std.random;
 import std.stream;
+import std.stdio;
 
 class Backprop {
 	private uint       numInputs;
-	private float[][][] units;     // Includes the output units. units[layer][unit][inputWeight]
+	private double[][][] units;     // Includes the output units. units[layer][unit][inputWeight]
 	private OutputFunctionPtr[] functions;
-	public float learningRate;
+	public double learningRate;
 	
 	///Constructor
-	public this(uint numInputs,uint[] numUnits,OutputFunctionPtr[] functions,float value=0.05,bool randomize=true){
+	public this(uint numInputs,uint[] numUnits,OutputFunctionPtr[] functions,double learningRate=0.03,double value=0.1,bool randomize=true){
 		if(numUnits.length == 0) throw new InputException("numUnits must be greater than 0");
 		if(numUnits.length != functions.length) throw new InputException("numUnits and functions must be the same length");
 		this.numInputs = numInputs;
 		this.functions = functions;
+		this.learningRate = learningRate;
+		units.length = numUnits.length;
 		initUnitLayer(0,numUnits[0],numInputs,value,randomize);
 		for(int i=1; i<numUnits.length; i++){
 			initUnitLayer(i,numUnits[i],numUnits[i-1],value,randomize);
@@ -24,7 +27,7 @@
 	}
 	
 	// Helper function to initialize a certain layer.
-	private void initUnitLayer(uint layer,uint num,uint numPrev,float value,bool randomize){
+	private void initUnitLayer(uint layer,uint num,uint numPrev,double value,bool randomize){
 		units[layer].length = num;
 		for(int i=0; i<num; i++){
 			units[layer][i].length = numPrev+1; // include the bias weight
@@ -37,40 +40,46 @@
 	
 	////////////////////////////////////////////////////// Evaluation //////////////////////////////////////////////////////
 	/// Evaluates the neural network.
-	public float[] evaluate(float[] inputs){
-		return evaluateFull(inputs)[$]; // the last item (outputs) of the return value
+	public double[] evaluate(double[] inputs){
+		return evaluateFull(inputs)[$-1]; // the last item (outputs) of the return value
 	}
 	
 	/// Evaluates the neural network and returns the output from all units.
-	public float[][] evaluateFull(float[] inputs){
+	public double[][] evaluateFull(double[] inputs){
 		if(inputs.length != numInputs) throw new InputException("Wrong length of inputs.");
-		float[][] outputs;
+		double[][] outputs;
 		outputs.length = units.length;
 		outputs[0] = evaluateLayer(0,inputs);
-		for(int i=0; i<units.length; i++){
+		for(int i=1; i<units.length; i++){
 			outputs[i] = this.evaluateLayer(i,outputs[i-1]);
 		}
 		return outputs;
 	}
 	
 	// Helper function to evaluate the outputs of a single layer.
-	private float[] evaluateLayer(uint layer,float[] layerInputs){
-		float[] output;
-		output.length = layerInputs.length;
-		for(int i=0; i<layerInputs.length; i++){
+	private double[] evaluateLayer(uint layer,double[] layerInputs){
+		double[] output;
+		output.length = units[layer].length;
+		//printArray(layerInputs);
+		for(int i=0; i<units[layer].length; i++){
 			output[i] = evaluateUnit(layer,i,layerInputs);
 		}
 		return output;
 	}
 	
 	// Helper function to evaluate the output of a single unit.
-	private float evaluateUnit(uint layer, uint unit, float[] layerInputs){
-		float total = units[layer][unit][0]; //bias
-		for(int i=1; i<layerInputs.length; i++){
+	private double evaluateUnit(uint layer, uint unit, double[] layerInputs){
+		//writef("(%d,%d)=",layer,unit);
+		//printArray(layerInputs);
+		double total = units[layer][unit][0]; //bias
+		for(int i=1; i<layerInputs.length+1; i++){
 			total += layerInputs[i-1] * units[layer][unit][i]; // wi * xi
+			//writef("@");
 		}
+		//writefln("        ! %f",total);
 		if(functions[layer] != null) return functions[layer](total); // apply the function (if there is one)
-		else return total; // just return the result instead
+		writefln("no function");
+		return total; // just return the result instead
 	}
 	
 	
@@ -78,52 +87,69 @@
 	/// Trains the neural network.
 	/// TODO:
 	///   Pull error calculation into a separate function.
-	public void train(float[][] allInputs, float[][] allOutputs){
-		if(allInputs.length != allOutputs.length) throw new InputException("allInputs and allOutputs must be the same size");
-		float[][][] weightUpdate;
-		float[][] outputsError;
-		float[][] outputs;
-		float total; //temp variable
+	public void train(double[][] trainingInputs, double[][] trainingOutputs){
+		if(trainingInputs.length != trainingOutputs.length) throw new InputException("trainingInputs and trainingOutputs must be the same size");
+		double[][][] weightUpdate;
+		double[][] outputsError;
+		double[][] outputs;
+		double total; //temp variable
 		
 		// Initialize the weightUpdate and outputsError variables
 		weightUpdate.length = units.length;
 		outputsError.length = units.length;
-		for(int i=0; i<weightUpdate.length; i++){
+		//writefln("#%d,%d",weightUpdate.length,outputsError.length);
+		for(int i=0; i<units.length; i++){
 			weightUpdate[i].length = units[i].length;
 			outputsError[i].length = units[i].length;
-			for(int j=0; j<weightUpdate[i].length; i++){
+			//writefln("##(%d)%d,%d",i,weightUpdate[i].length,outputsError[i].length);
+			for(int j=0; j<weightUpdate[i].length; j++){
 				weightUpdate[i][j].length = units[i][j].length;
+				for(int k=0; k<weightUpdate[i][j].length; k++) weightUpdate[i][j][k] = 0.0f;
+				//writefln("###(%d)%d",j,weightUpdate[i][j].length);
 			}
 		}
 		
 		
 		// Loop through each of the training examples
-		for(int example=0; example < allInputs.length; example++){
-			outputs = evaluateFull(allInputs[example]);
+		for(int example=0; example < trainingInputs.length; example++){
+			outputs = evaluateFull(trainingInputs[example]);
 			
 			// Computing error of output layer
-			for(int i=0; i<outputs[$].length; i++)
-				outputsError[$][i] = outputs[$][i] * (1 - outputs[$][i]) * (allOutputs[example][i] - outputs[$][i]); // o(1-o)(t-o)
+			for(int i=0; i<outputs[$-1].length; i++){ // units of last layer
+				//writefln("{%d,%d,%d,%d}",example,i,outputs.length,outputsError[$-1].length);
+				outputsError[$-1][i] = outputs[$-1][i] * (1 - outputs[$-1][i]) * (trainingOutputs[example][i] - outputs[$-1][i]);
+			} // o(1-o)(t-o)
 			
+			//printArray(outputsError[$-1]);
+			//printArray(units[length-1]);
+			
+			//*
 			// Loop through each of the hidden layers (backwards - BACKpropagation!)
-			for(int i=units.length-2; i >= 0; i--){ // -2 to skip the output layer
+			for(int layer=units.length-2; layer >= 0; layer--){ // -2 to skip the output layer
+				//writef("|");
 				// loop through the units in each hidden layer
-				for(int j=0; j<units[i].length; j++){
+				for(int unit=0; unit<units[layer].length; unit++){
+					//writef("*");
 					total=0;
 					// total up w * e for the units the output of this unit goes into
-					for(int k=0; k<units[i+1].length; k++){
-						total += units[i+1][k][j+1] * outputsError[i+1][k];
+					for(int k=0; k<units[layer+1].length; k++){
+						//writef("{weight=%f,error=%f}", units[layer+1][k][unit+1/* +1 for bias*/], outputsError[layer+1][k]);
+						total += units[layer+1][k][unit+1/* +1 for bias*/] * outputsError[layer+1][k];
 					}
+					//writefln("=%f(total)",total);
 					// multiply total by o(1-o), store in outputsError
-					outputsError[i][j] = outputs[i][j] * (1 - outputs[i][j]) * total;
+					outputsError[layer][unit] = outputs[layer][unit] * (1 - outputs[layer][unit]) * total;
 				}
-			}
+			} //writefln();
+			
+			//writef("outputError="); printArray(outputsError);
 			
 			// special case for the units that receive the input values
-			for(int j=0; j<units[0].length; j++){ // unit
-				weightUpdate[0][j][0] += outputsError[0][j]; //bias
-				for(int k=1; k<units[0][j].length; k++){ // input
-					weightUpdate[0][j][k] += outputsError[0][j] * allInputs[example][k-1];
+			for(int unit=0; unit<units[0].length; unit++){ // unit
+				//writefln(":%d,%d,%d,%d",j,weightUpdate.length,weightUpdate[0].length,weightUpdate[0][j].length);
+				weightUpdate[0][unit][0] += outputsError[0][unit]; //bias
+				for(int input=1; input<units[0][unit].length; input++){ // input
+					weightUpdate[0][unit][input] += outputsError[0][unit] * trainingInputs[example][input-1]; // account for bias
 				}
 			}
 			
@@ -132,6 +158,7 @@
 				for(int j=0; j<units[i].length; j++){ // unit
 					weightUpdate[i][j][0] += outputsError[i][j]; //bias
 					for(int k=1; k<units[i][j].length; k++){ // input
+						//writefln("[%d,%d,%d]=%f; %f; %f",i,j,k,weightUpdate[i][j][k],outputsError[i][j],outputs[i-1][k-1]);
 						weightUpdate[i][j][k] += outputsError[i][j] * outputs[i-1][k-1]; // previous layer, account for bias
 					}
 				}
@@ -142,6 +169,7 @@
 		for(int i=0; i<units.length; i++){ // layer
 			for(int j=0; j<units[i].length; j++){ // unit
 				for(int k=0; k<units[i][j].length; k++){ // input
+					//writefln("[%d,%d,%d]=%f; %f",i,j,k,units[i][j][k],weightUpdate[i][j][k]);
 					units[i][j][k] += this.learningRate * weightUpdate[i][j][k];
 				}
 			}
@@ -149,22 +177,49 @@
 	}
 	
 	/// Calculate the output error
-	float calculateError(float[][] allInputs, float[][] allOutputs){
-		if(allInputs.length != allOutputs.length) throw new InputException("allInputs and allOutputs must be the same size");
-		float[] outputs;
-		float total,temp;
-		for(int i=0; i<allInputs.length; i++){
-			outputs = evaluate(allInputs[i]);
-			if(outputs.length != allOutputs[i].length) throw new InputException("Wrong output length");
+	double calculateError(double[][] trainingInputs, double[][] trainingOutputs){
+		if(trainingInputs.length != trainingOutputs.length) throw new InputException("trainingInputs and trainingOutputs must be the same size");
+		double[] outputs;
+		double total=0,temp;
+		for(int i=0; i<trainingInputs.length; i++){
+			outputs = evaluate(trainingInputs[i]);
+			if(outputs.length != trainingOutputs[i].length) throw new InputException("Wrong output length");
 			for(int j=0; j<outputs.length; j++){
-				temp = allOutputs[i][j] - outputs[j];
+				temp = trainingOutputs[i][j] - outputs[j];
+				//writefln("&%f,%f",temp*temp,total);
 				total += temp * temp;
 			}
 		}
 		return 0.5 * total;
 	}
+	
+	double[][][] getWeights(){
+		return units.dup;
+	}
+}
+
+void printArray(double[] array){
+	writef("[");
+	for(int i=0; i<array.length-1; i++){
+		writef("%f, ",array[i]);
+	}
+	writefln("%f]",array[$-1]);
+}
+
+void printArray(double[][] array){
+	writef("[");
+	for(int i=0; i<array.length; i++){
+		printArray(array[i]);
+	}
+	writefln("]");
+}
+
+void printArray(double[][][] array){
+	writef("[");
+	for(int i=0; i<array.length; i++){
+		printArray(array[i]);
+	}
+	writefln("]");
 }
 
 
-
-
diff -r 314d68bafeff -r 73beed484455 trunk/backprop_test.d
--- a/trunk/backprop_test.d	Fri Apr 11 18:12:55 2008 -0600
+++ b/trunk/backprop_test.d	Sat Apr 12 21:55:37 2008 -0600
@@ -3,8 +3,9 @@
 import aid.nn.multilayer.backprop;
 import aid.nn.outputFunctions;
 import std.stdio;
+import std.random;
 
-/+float[][] trainingInputs = [
+/+double[][] trainingInputs = [
 	[0,0,0],
 	[0,0,1],
 	[0,1,0],
@@ -14,7 +15,7 @@
 	[1,1,0],
 	[1,1,1]];
 
-float[][] trainingOutputs = [
+double[][] trainingOutputs = [
 	[0.1],
 	[0.9],
 	[0.9],
@@ -24,27 +25,39 @@
 	[0.1],
 	[0.9]];+/
 
-float[][] trainingInputs = [
+/+double[][] trainingInputs = [
 	[0,0],
+	[1,0],
 	[0,1],
-	[1,0],
 	[1,1]];
 
-float[][] trainingOutputs = [
-	[0.1],
+double[][] trainingOutputs = [
 	[0.9],
-	[0.9],
-	[0.1]];
+	[0.1],
+	[0.1],
+	[0.9]];+/
+
+double[][] trainingInputs = [
+	[0.9,0.1,0.1,0.1,0.1,0.1,0.1,0.1],
+	[0.1,0.9,0.1,0.1,0.1,0.1,0.1,0.1],
+	[0.1,0.1,0.9,0.1,0.1,0.1,0.1,0.1],
+	[0.1,0.1,0.1,0.9,0.1,0.1,0.1,0.1],
+	[0.1,0.1,0.1,0.1,0.9,0.1,0.1,0.1],
+	[0.1,0.1,0.1,0.1,0.1,0.9,0.1,0.1],
+	[0.1,0.1,0.1,0.1,0.1,0.1,0.9,0.1],
+	[0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.9]];
 
 void main(){
-	Backprop nn = new Backprop(2,[4,1],[&sigmoid,&sigmoid]);
+	//rand_seed(0,0);
+	Backprop nn = new Backprop(8,[3,8],[&sigmoid,&sigmoid],.1);
 	
-	float error = 10.0;
-	float[] output;
+	double error = nn.calculateError(trainingInputs,trainingInputs);
+	double[] output;
 	int iter = 0;
-	while(error >= 0.5){
-		error = nn.calculateError(trainingInputs,trainingOutputs);
-		if(iter % 100 == 0){
+	writef("weights="); printArray(nn.getWeights());
+	writef("outputs="); printArray(nn.evaluateFull(trainingInputs[$-1]));
+	while(error >= 0.01 && iter < 50000){
+		if(iter % 500 == 0){
 			writefln("Iter: %d",iter);
 			for(int i=0; i<trainingInputs.length; i++){
 				output = nn.evaluate(trainingInputs[i]);
@@ -52,20 +65,38 @@
 			}
 			writefln("  Error: %f", error);
 		}
-		nn.train(trainingInputs,trainingOutputs);
+		nn.train(trainingInputs,trainingInputs);
+		error = nn.calculateError(trainingInputs,trainingInputs);
+		iter++;
 	}
 	writefln("Total Iters: %d",iter);
 	for(int i=0; i<trainingInputs.length; i++){
-		output = nn.evaluate(trainingInputs[i]);
-		writef("  %d:", i); printArray(output);
+		writef("  %d:", i); printArray(nn.evaluateFull(trainingInputs[i])[0]);
 	}
 	writefln("  Error: %f", error);
+	writef("weights="); printArray(nn.getWeights());
 }
 
-void printArray(float[] array){
+void printArray(double[] array){
 	writef("[");
 	for(int i=0; i<array.length-1; i++){
 		writef("%f, ",array[i]);
 	}
-	writefln("%f]",array[$]);
+	writefln("%f]",array[$-1]);
+}
+
+void printArray(double[][] array){
+	writef("[");
+	for(int i=0; i<array.length; i++){
+		printArray(array[i]);
+	}
+	writefln("]");
+}
+
+void printArray(double[][][] array){
+	writef("[");
+	for(int i=0; i<array.length; i++){
+		printArray(array[i]);
+	}
+	writefln("]");
 }
\ No newline at end of file