Mercurial > projects > aid
changeset 4:73beed484455
Backprop working correctly.
author | revcompgeek |
---|---|
date | Sat, 12 Apr 2008 21:55:37 -0600 |
parents | 314d68bafeff |
children | 810d58835f86 |
files | trunk/aid/nn/multilayer/backprop.d trunk/backprop_test.d |
diffstat | 2 files changed, 154 insertions(+), 68 deletions(-) [+] |
line wrap: on
line diff
--- a/trunk/aid/nn/multilayer/backprop.d Fri Apr 11 18:12:55 2008 -0600 +++ b/trunk/aid/nn/multilayer/backprop.d Sat Apr 12 21:55:37 2008 -0600 @@ -4,19 +4,22 @@ import aid.misc; import std.random; import std.stream; +import std.stdio; class Backprop { private uint numInputs; - private float[][][] units; // Includes the output units. units[layer][unit][inputWeight] + private double[][][] units; // Includes the output units. units[layer][unit][inputWeight] private OutputFunctionPtr[] functions; - public float learningRate; + public double learningRate; ///Constructor - public this(uint numInputs,uint[] numUnits,OutputFunctionPtr[] functions,float value=0.05,bool randomize=true){ + public this(uint numInputs,uint[] numUnits,OutputFunctionPtr[] functions,double learningRate=0.03,double value=0.1,bool randomize=true){ if(numUnits.length == 0) throw new InputException("numUnits must be greater than 0"); if(numUnits.length != functions.length) throw new InputException("numUnits and functions must be the same length"); this.numInputs = numInputs; this.functions = functions; + this.learningRate = learningRate; + units.length = numUnits.length; initUnitLayer(0,numUnits[0],numInputs,value,randomize); for(int i=1; i<numUnits.length; i++){ initUnitLayer(i,numUnits[i],numUnits[i-1],value,randomize); @@ -24,7 +27,7 @@ } // Helper function to initialize a certain layer. - private void initUnitLayer(uint layer,uint num,uint numPrev,float value,bool randomize){ + private void initUnitLayer(uint layer,uint num,uint numPrev,double value,bool randomize){ units[layer].length = num; for(int i=0; i<num; i++){ units[layer][i].length = numPrev+1; // include the bias weight @@ -37,40 +40,46 @@ ////////////////////////////////////////////////////// Evaluation ////////////////////////////////////////////////////// /// Evaluates the neural network. - public float[] evaluate(float[] inputs){ - return evaluateFull(inputs)[$]; // the last item (outputs) of the return value + public double[] evaluate(double[] inputs){ + return evaluateFull(inputs)[$-1]; // the last item (outputs) of the return value } /// Evaluates the neural network and returns the output from all units. - public float[][] evaluateFull(float[] inputs){ + public double[][] evaluateFull(double[] inputs){ if(inputs.length != numInputs) throw new InputException("Wrong length of inputs."); - float[][] outputs; + double[][] outputs; outputs.length = units.length; outputs[0] = evaluateLayer(0,inputs); - for(int i=0; i<units.length; i++){ + for(int i=1; i<units.length; i++){ outputs[i] = this.evaluateLayer(i,outputs[i-1]); } return outputs; } // Helper function to evaluate the outputs of a single layer. - private float[] evaluateLayer(uint layer,float[] layerInputs){ - float[] output; - output.length = layerInputs.length; - for(int i=0; i<layerInputs.length; i++){ + private double[] evaluateLayer(uint layer,double[] layerInputs){ + double[] output; + output.length = units[layer].length; + //printArray(layerInputs); + for(int i=0; i<units[layer].length; i++){ output[i] = evaluateUnit(layer,i,layerInputs); } return output; } // Helper function to evaluate the output of a single unit. - private float evaluateUnit(uint layer, uint unit, float[] layerInputs){ - float total = units[layer][unit][0]; //bias - for(int i=1; i<layerInputs.length; i++){ + private double evaluateUnit(uint layer, uint unit, double[] layerInputs){ + //writef("(%d,%d)=",layer,unit); + //printArray(layerInputs); + double total = units[layer][unit][0]; //bias + for(int i=1; i<layerInputs.length+1; i++){ total += layerInputs[i-1] * units[layer][unit][i]; // wi * xi + //writef("@"); } + //writefln(" ! %f",total); if(functions[layer] != null) return functions[layer](total); // apply the function (if there is one) - else return total; // just return the result instead + writefln("no function"); + return total; // just return the result instead } @@ -78,52 +87,69 @@ /// Trains the neural network. /// TODO: /// Pull error calculation into a separate function. - public void train(float[][] allInputs, float[][] allOutputs){ - if(allInputs.length != allOutputs.length) throw new InputException("allInputs and allOutputs must be the same size"); - float[][][] weightUpdate; - float[][] outputsError; - float[][] outputs; - float total; //temp variable + public void train(double[][] trainingInputs, double[][] trainingOutputs){ + if(trainingInputs.length != trainingOutputs.length) throw new InputException("trainingInputs and trainingOutputs must be the same size"); + double[][][] weightUpdate; + double[][] outputsError; + double[][] outputs; + double total; //temp variable // Initialize the weightUpdate and outputsError variables weightUpdate.length = units.length; outputsError.length = units.length; - for(int i=0; i<weightUpdate.length; i++){ + //writefln("#%d,%d",weightUpdate.length,outputsError.length); + for(int i=0; i<units.length; i++){ weightUpdate[i].length = units[i].length; outputsError[i].length = units[i].length; - for(int j=0; j<weightUpdate[i].length; i++){ + //writefln("##(%d)%d,%d",i,weightUpdate[i].length,outputsError[i].length); + for(int j=0; j<weightUpdate[i].length; j++){ weightUpdate[i][j].length = units[i][j].length; + for(int k=0; k<weightUpdate[i][j].length; k++) weightUpdate[i][j][k] = 0.0f; + //writefln("###(%d)%d",j,weightUpdate[i][j].length); } } // Loop through each of the training examples - for(int example=0; example < allInputs.length; example++){ - outputs = evaluateFull(allInputs[example]); + for(int example=0; example < trainingInputs.length; example++){ + outputs = evaluateFull(trainingInputs[example]); // Computing error of output layer - for(int i=0; i<outputs[$].length; i++) - outputsError[$][i] = outputs[$][i] * (1 - outputs[$][i]) * (allOutputs[example][i] - outputs[$][i]); // o(1-o)(t-o) + for(int i=0; i<outputs[$-1].length; i++){ // units of last layer + //writefln("{%d,%d,%d,%d}",example,i,outputs.length,outputsError[$-1].length); + outputsError[$-1][i] = outputs[$-1][i] * (1 - outputs[$-1][i]) * (trainingOutputs[example][i] - outputs[$-1][i]); + } // o(1-o)(t-o) + //printArray(outputsError[$-1]); + //printArray(units[length-1]); + + //* // Loop through each of the hidden layers (backwards - BACKpropagation!) - for(int i=units.length-2; i >= 0; i--){ // -2 to skip the output layer + for(int layer=units.length-2; layer >= 0; layer--){ // -2 to skip the output layer + //writef("|"); // loop through the units in each hidden layer - for(int j=0; j<units[i].length; j++){ + for(int unit=0; unit<units[layer].length; unit++){ + //writef("*"); total=0; // total up w * e for the units the output of this unit goes into - for(int k=0; k<units[i+1].length; k++){ - total += units[i+1][k][j+1] * outputsError[i+1][k]; + for(int k=0; k<units[layer+1].length; k++){ + //writef("{weight=%f,error=%f}", units[layer+1][k][unit+1/* +1 for bias*/], outputsError[layer+1][k]); + total += units[layer+1][k][unit+1/* +1 for bias*/] * outputsError[layer+1][k]; } + //writefln("=%f(total)",total); // multiply total by o(1-o), store in outputsError - outputsError[i][j] = outputs[i][j] * (1 - outputs[i][j]) * total; + outputsError[layer][unit] = outputs[layer][unit] * (1 - outputs[layer][unit]) * total; } - } + } //writefln(); + + //writef("outputError="); printArray(outputsError); // special case for the units that receive the input values - for(int j=0; j<units[0].length; j++){ // unit - weightUpdate[0][j][0] += outputsError[0][j]; //bias - for(int k=1; k<units[0][j].length; k++){ // input - weightUpdate[0][j][k] += outputsError[0][j] * allInputs[example][k-1]; + for(int unit=0; unit<units[0].length; unit++){ // unit + //writefln(":%d,%d,%d,%d",j,weightUpdate.length,weightUpdate[0].length,weightUpdate[0][j].length); + weightUpdate[0][unit][0] += outputsError[0][unit]; //bias + for(int input=1; input<units[0][unit].length; input++){ // input + weightUpdate[0][unit][input] += outputsError[0][unit] * trainingInputs[example][input-1]; // account for bias } } @@ -132,6 +158,7 @@ for(int j=0; j<units[i].length; j++){ // unit weightUpdate[i][j][0] += outputsError[i][j]; //bias for(int k=1; k<units[i][j].length; k++){ // input + //writefln("[%d,%d,%d]=%f; %f; %f",i,j,k,weightUpdate[i][j][k],outputsError[i][j],outputs[i-1][k-1]); weightUpdate[i][j][k] += outputsError[i][j] * outputs[i-1][k-1]; // previous layer, account for bias } } @@ -142,6 +169,7 @@ for(int i=0; i<units.length; i++){ // layer for(int j=0; j<units[i].length; j++){ // unit for(int k=0; k<units[i][j].length; k++){ // input + //writefln("[%d,%d,%d]=%f; %f",i,j,k,units[i][j][k],weightUpdate[i][j][k]); units[i][j][k] += this.learningRate * weightUpdate[i][j][k]; } } @@ -149,22 +177,49 @@ } /// Calculate the output error - float calculateError(float[][] allInputs, float[][] allOutputs){ - if(allInputs.length != allOutputs.length) throw new InputException("allInputs and allOutputs must be the same size"); - float[] outputs; - float total,temp; - for(int i=0; i<allInputs.length; i++){ - outputs = evaluate(allInputs[i]); - if(outputs.length != allOutputs[i].length) throw new InputException("Wrong output length"); + double calculateError(double[][] trainingInputs, double[][] trainingOutputs){ + if(trainingInputs.length != trainingOutputs.length) throw new InputException("trainingInputs and trainingOutputs must be the same size"); + double[] outputs; + double total=0,temp; + for(int i=0; i<trainingInputs.length; i++){ + outputs = evaluate(trainingInputs[i]); + if(outputs.length != trainingOutputs[i].length) throw new InputException("Wrong output length"); for(int j=0; j<outputs.length; j++){ - temp = allOutputs[i][j] - outputs[j]; + temp = trainingOutputs[i][j] - outputs[j]; + //writefln("&%f,%f",temp*temp,total); total += temp * temp; } } return 0.5 * total; } + + double[][][] getWeights(){ + return units.dup; + } +} + +void printArray(double[] array){ + writef("["); + for(int i=0; i<array.length-1; i++){ + writef("%f, ",array[i]); + } + writefln("%f]",array[$-1]); +} + +void printArray(double[][] array){ + writef("["); + for(int i=0; i<array.length; i++){ + printArray(array[i]); + } + writefln("]"); +} + +void printArray(double[][][] array){ + writef("["); + for(int i=0; i<array.length; i++){ + printArray(array[i]); + } + writefln("]"); } - -
--- a/trunk/backprop_test.d Fri Apr 11 18:12:55 2008 -0600 +++ b/trunk/backprop_test.d Sat Apr 12 21:55:37 2008 -0600 @@ -3,8 +3,9 @@ import aid.nn.multilayer.backprop; import aid.nn.outputFunctions; import std.stdio; +import std.random; -/+float[][] trainingInputs = [ +/+double[][] trainingInputs = [ [0,0,0], [0,0,1], [0,1,0], @@ -14,7 +15,7 @@ [1,1,0], [1,1,1]]; -float[][] trainingOutputs = [ +double[][] trainingOutputs = [ [0.1], [0.9], [0.9], @@ -24,27 +25,39 @@ [0.1], [0.9]];+/ -float[][] trainingInputs = [ +/+double[][] trainingInputs = [ [0,0], + [1,0], [0,1], - [1,0], [1,1]]; -float[][] trainingOutputs = [ - [0.1], +double[][] trainingOutputs = [ [0.9], - [0.9], - [0.1]]; + [0.1], + [0.1], + [0.9]];+/ + +double[][] trainingInputs = [ + [0.9,0.1,0.1,0.1,0.1,0.1,0.1,0.1], + [0.1,0.9,0.1,0.1,0.1,0.1,0.1,0.1], + [0.1,0.1,0.9,0.1,0.1,0.1,0.1,0.1], + [0.1,0.1,0.1,0.9,0.1,0.1,0.1,0.1], + [0.1,0.1,0.1,0.1,0.9,0.1,0.1,0.1], + [0.1,0.1,0.1,0.1,0.1,0.9,0.1,0.1], + [0.1,0.1,0.1,0.1,0.1,0.1,0.9,0.1], + [0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.9]]; void main(){ - Backprop nn = new Backprop(2,[4,1],[&sigmoid,&sigmoid]); + //rand_seed(0,0); + Backprop nn = new Backprop(8,[3,8],[&sigmoid,&sigmoid],.1); - float error = 10.0; - float[] output; + double error = nn.calculateError(trainingInputs,trainingInputs); + double[] output; int iter = 0; - while(error >= 0.5){ - error = nn.calculateError(trainingInputs,trainingOutputs); - if(iter % 100 == 0){ + writef("weights="); printArray(nn.getWeights()); + writef("outputs="); printArray(nn.evaluateFull(trainingInputs[$-1])); + while(error >= 0.01 && iter < 50000){ + if(iter % 500 == 0){ writefln("Iter: %d",iter); for(int i=0; i<trainingInputs.length; i++){ output = nn.evaluate(trainingInputs[i]); @@ -52,20 +65,38 @@ } writefln(" Error: %f", error); } - nn.train(trainingInputs,trainingOutputs); + nn.train(trainingInputs,trainingInputs); + error = nn.calculateError(trainingInputs,trainingInputs); + iter++; } writefln("Total Iters: %d",iter); for(int i=0; i<trainingInputs.length; i++){ - output = nn.evaluate(trainingInputs[i]); - writef(" %d:", i); printArray(output); + writef(" %d:", i); printArray(nn.evaluateFull(trainingInputs[i])[0]); } writefln(" Error: %f", error); + writef("weights="); printArray(nn.getWeights()); } -void printArray(float[] array){ +void printArray(double[] array){ writef("["); for(int i=0; i<array.length-1; i++){ writef("%f, ",array[i]); } - writefln("%f]",array[$]); + writefln("%f]",array[$-1]); +} + +void printArray(double[][] array){ + writef("["); + for(int i=0; i<array.length; i++){ + printArray(array[i]); + } + writefln("]"); +} + +void printArray(double[][][] array){ + writef("["); + for(int i=0; i<array.length; i++){ + printArray(array[i]); + } + writefln("]"); } \ No newline at end of file