Mercurial > projects > aid
changeset 3:314d68bafeff
Backprop and backprop_test added (no testing).
author | revcompgeek |
---|---|
date | Fri, 11 Apr 2008 18:12:55 -0600 |
parents | 9655c8362b25 |
children | 73beed484455 |
files | trunk/aid/misc.d trunk/aid/nn/multilayer/backprop.d trunk/aid/nn/outputFunctions.d trunk/aid/nn/perceptron.d trunk/backprop_test.d trunk/dsss.conf |
diffstat | 6 files changed, 286 insertions(+), 34 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trunk/aid/misc.d Fri Apr 11 18:12:55 2008 -0600 @@ -0,0 +1,13 @@ +module aid.misc; + +import std.random; + +class InputException : Exception { + this(char[] message){ + super(message); + } +} + +double rnd(){ // The function that should be included in every math library! + return (cast(double)rand())/uint.max; +} \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trunk/aid/nn/multilayer/backprop.d Fri Apr 11 18:12:55 2008 -0600 @@ -0,0 +1,170 @@ +module aid.nn.multilevel.backprop; + +import aid.nn.outputFunctions; +import aid.misc; +import std.random; +import std.stream; + +class Backprop { + private uint numInputs; + private float[][][] units; // Includes the output units. units[layer][unit][inputWeight] + private OutputFunctionPtr[] functions; + public float learningRate; + + ///Constructor + public this(uint numInputs,uint[] numUnits,OutputFunctionPtr[] functions,float value=0.05,bool randomize=true){ + if(numUnits.length == 0) throw new InputException("numUnits must be greater than 0"); + if(numUnits.length != functions.length) throw new InputException("numUnits and functions must be the same length"); + this.numInputs = numInputs; + this.functions = functions; + initUnitLayer(0,numUnits[0],numInputs,value,randomize); + for(int i=1; i<numUnits.length; i++){ + initUnitLayer(i,numUnits[i],numUnits[i-1],value,randomize); + } + } + + // Helper function to initialize a certain layer. + private void initUnitLayer(uint layer,uint num,uint numPrev,float value,bool randomize){ + units[layer].length = num; + for(int i=0; i<num; i++){ + units[layer][i].length = numPrev+1; // include the bias weight + for(int j=0; j<numPrev+1; j++){ + if(randomize) units[layer][i][j] = rnd() * value * 2 - value; // between -value and value + else units[layer][i][j] = value; + } + } + } + + ////////////////////////////////////////////////////// Evaluation ////////////////////////////////////////////////////// + /// Evaluates the neural network. + public float[] evaluate(float[] inputs){ + return evaluateFull(inputs)[$]; // the last item (outputs) of the return value + } + + /// Evaluates the neural network and returns the output from all units. + public float[][] evaluateFull(float[] inputs){ + if(inputs.length != numInputs) throw new InputException("Wrong length of inputs."); + float[][] outputs; + outputs.length = units.length; + outputs[0] = evaluateLayer(0,inputs); + for(int i=0; i<units.length; i++){ + outputs[i] = this.evaluateLayer(i,outputs[i-1]); + } + return outputs; + } + + // Helper function to evaluate the outputs of a single layer. + private float[] evaluateLayer(uint layer,float[] layerInputs){ + float[] output; + output.length = layerInputs.length; + for(int i=0; i<layerInputs.length; i++){ + output[i] = evaluateUnit(layer,i,layerInputs); + } + return output; + } + + // Helper function to evaluate the output of a single unit. + private float evaluateUnit(uint layer, uint unit, float[] layerInputs){ + float total = units[layer][unit][0]; //bias + for(int i=1; i<layerInputs.length; i++){ + total += layerInputs[i-1] * units[layer][unit][i]; // wi * xi + } + if(functions[layer] != null) return functions[layer](total); // apply the function (if there is one) + else return total; // just return the result instead + } + + + ////////////////////////////////////////////////////// Training ////////////////////////////////////////////////////// + /// Trains the neural network. + /// TODO: + /// Pull error calculation into a separate function. + public void train(float[][] allInputs, float[][] allOutputs){ + if(allInputs.length != allOutputs.length) throw new InputException("allInputs and allOutputs must be the same size"); + float[][][] weightUpdate; + float[][] outputsError; + float[][] outputs; + float total; //temp variable + + // Initialize the weightUpdate and outputsError variables + weightUpdate.length = units.length; + outputsError.length = units.length; + for(int i=0; i<weightUpdate.length; i++){ + weightUpdate[i].length = units[i].length; + outputsError[i].length = units[i].length; + for(int j=0; j<weightUpdate[i].length; i++){ + weightUpdate[i][j].length = units[i][j].length; + } + } + + + // Loop through each of the training examples + for(int example=0; example < allInputs.length; example++){ + outputs = evaluateFull(allInputs[example]); + + // Computing error of output layer + for(int i=0; i<outputs[$].length; i++) + outputsError[$][i] = outputs[$][i] * (1 - outputs[$][i]) * (allOutputs[example][i] - outputs[$][i]); // o(1-o)(t-o) + + // Loop through each of the hidden layers (backwards - BACKpropagation!) + for(int i=units.length-2; i >= 0; i--){ // -2 to skip the output layer + // loop through the units in each hidden layer + for(int j=0; j<units[i].length; j++){ + total=0; + // total up w * e for the units the output of this unit goes into + for(int k=0; k<units[i+1].length; k++){ + total += units[i+1][k][j+1] * outputsError[i+1][k]; + } + // multiply total by o(1-o), store in outputsError + outputsError[i][j] = outputs[i][j] * (1 - outputs[i][j]) * total; + } + } + + // special case for the units that receive the input values + for(int j=0; j<units[0].length; j++){ // unit + weightUpdate[0][j][0] += outputsError[0][j]; //bias + for(int k=1; k<units[0][j].length; k++){ // input + weightUpdate[0][j][k] += outputsError[0][j] * allInputs[example][k-1]; + } + } + + // Update the weightUpdate array + for(int i=1; i<units.length; i++){ // layer + for(int j=0; j<units[i].length; j++){ // unit + weightUpdate[i][j][0] += outputsError[i][j]; //bias + for(int k=1; k<units[i][j].length; k++){ // input + weightUpdate[i][j][k] += outputsError[i][j] * outputs[i-1][k-1]; // previous layer, account for bias + } + } + } + } + + // Apply the weightUpdate array to the weights + for(int i=0; i<units.length; i++){ // layer + for(int j=0; j<units[i].length; j++){ // unit + for(int k=0; k<units[i][j].length; k++){ // input + units[i][j][k] += this.learningRate * weightUpdate[i][j][k]; + } + } + } + } + + /// Calculate the output error + float calculateError(float[][] allInputs, float[][] allOutputs){ + if(allInputs.length != allOutputs.length) throw new InputException("allInputs and allOutputs must be the same size"); + float[] outputs; + float total,temp; + for(int i=0; i<allInputs.length; i++){ + outputs = evaluate(allInputs[i]); + if(outputs.length != allOutputs[i].length) throw new InputException("Wrong output length"); + for(int j=0; j<outputs.length; j++){ + temp = allOutputs[i][j] - outputs[j]; + total += temp * temp; + } + } + return 0.5 * total; + } +} + + + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trunk/aid/nn/outputFunctions.d Fri Apr 11 18:12:55 2008 -0600 @@ -0,0 +1,23 @@ +module aid.nn.outputFunctions; + +import aid.nn.outputFunctions; +import std.math; + +alias double function(double) OutputFunctionPtr; + +enum OutputFunction { + Linear=0, Sign, Sigmoid, Tanh +} + +double sign(double y){ + if(y>0) return 1; + return -1; +} + +double sigmoid(double x){ + return 1/(1+exp(-x)); +} + +double tanh(double x){ + return cast(double)tanh(cast(real)x); +} \ No newline at end of file
--- a/trunk/aid/nn/perceptron.d Sat Apr 05 23:41:30 2008 -0600 +++ b/trunk/aid/nn/perceptron.d Fri Apr 11 18:12:55 2008 -0600 @@ -1,38 +1,10 @@ module aid.nn.perceptron; -import std.random; +import aid.nn.outputFunctions; +import aid.misc; import std.math; import std.string; -double rnd(){ // The function that should be included in every math library! - return (cast(double)rand())/uint.max; -} - -class InputException : Exception { - this(char[] message){ - super(message); - } -} - -// The output functions - -alias double function(double) OutputFunction; - -double sign(double y){ - if(y>0) return 1; - return -1; -} - -double sigmoid(double x){ - return 1/(1+exp(-x)); -} - -double tanh(double x){ - return cast(double)tanh(cast(real)x); -} - -// End output functions - class perceptron { private int numInputs; @@ -42,7 +14,7 @@ /** * This is the constructor for loading the neural network from a string. - * + * * Params: * savedString = The string that was output from the save function. * @@ -74,10 +46,10 @@ /** * Evaluates the neural network. - * + * * Params: * inputs = The set of inputs to evaluate. - * + * * Returns: 1 to indicate true, -1 for false */ @@ -117,7 +89,7 @@ /** * Calculates the error based on the sum squared error function. - * + * * Params: * inputs = An array of arrays of all testing inputs. * outputs = An array of all the outputs that the cooresponding inputs should have. @@ -140,3 +112,4 @@ } } +// TODO: Impliment loading and saving of files \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trunk/backprop_test.d Fri Apr 11 18:12:55 2008 -0600 @@ -0,0 +1,71 @@ +module backprop_test; + +import aid.nn.multilayer.backprop; +import aid.nn.outputFunctions; +import std.stdio; + +/+float[][] trainingInputs = [ + [0,0,0], + [0,0,1], + [0,1,0], + [0,1,1], + [1,0,0], + [1,0,1], + [1,1,0], + [1,1,1]]; + +float[][] trainingOutputs = [ + [0.1], + [0.9], + [0.9], + [0.1], + [0.9], + [0.1], + [0.1], + [0.9]];+/ + +float[][] trainingInputs = [ + [0,0], + [0,1], + [1,0], + [1,1]]; + +float[][] trainingOutputs = [ + [0.1], + [0.9], + [0.9], + [0.1]]; + +void main(){ + Backprop nn = new Backprop(2,[4,1],[&sigmoid,&sigmoid]); + + float error = 10.0; + float[] output; + int iter = 0; + while(error >= 0.5){ + error = nn.calculateError(trainingInputs,trainingOutputs); + if(iter % 100 == 0){ + writefln("Iter: %d",iter); + for(int i=0; i<trainingInputs.length; i++){ + output = nn.evaluate(trainingInputs[i]); + writef(" %d:", i); printArray(output); + } + writefln(" Error: %f", error); + } + nn.train(trainingInputs,trainingOutputs); + } + writefln("Total Iters: %d",iter); + for(int i=0; i<trainingInputs.length; i++){ + output = nn.evaluate(trainingInputs[i]); + writef(" %d:", i); printArray(output); + } + writefln(" Error: %f", error); +} + +void printArray(float[] array){ + writef("["); + for(int i=0; i<array.length-1; i++){ + writef("%f, ",array[i]); + } + writefln("%f]",array[$]); +} \ No newline at end of file