Mercurial > projects > aid

annotate trunk/aid/nn/multilayer/backprop.d @ 5:810d58835f86

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added momentum and stochastic training to backprop.
author revcompgeek
date Tue, 15 Apr 2008 14:39:49 -0600
parents 73beed484455
children ff92c77006c7
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
1
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
2 module aid.nn.multilevel.backprop;
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
4 import aid.nn.outputFunctions;
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
5 import aid.misc;
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
6 import std.random;
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
7 import std.stream;
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
8 import std.stdio;
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
9
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
10 class Backprop {
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
11 private uint numInputs;
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
12 private double[][][] units; // Includes the output units. units[layer][unit][inputWeight]
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
13 private OutputFunctionPtr[] functions;
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
14 public double learningRate;
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
15 public double momentum;
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
16 private double[][][] oldWeightUpdate;
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
17
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
18 ///Constructor
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
19 public this(uint numInputs, uint[] numUnits, OutputFunctionPtr[] functions,
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
20 double learningRate = 0.03, double momentum = 0.1,
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
21 double value = 0.1, bool randomize = true) {
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
22 if(numUnits.length == 0)
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
23 throw new InputException("numUnits must be greater than 0");
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
24 if(numUnits.length != functions.length)
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
25 throw new InputException(
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
26 "numUnits and functions must be the same length");
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
27 this.numInputs = numInputs;
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
28 this.functions = functions;
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
29 this.learningRate = learningRate;
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
30 this.momentum = momentum;
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
31 units.length = numUnits.length;
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
32 oldWeightUpdate.length = numUnits.length;
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
33 initUnitLayer(0, numUnits[0], numInputs, value, randomize);
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
34 for(int i = 1; i < numUnits.length; i++) {
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
35 initUnitLayer(i, numUnits[i], numUnits[i - 1], value, randomize);
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
36 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
37 }
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
38
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
39 // Helper function to initialize a certain layer.
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
40 private void initUnitLayer(uint layer, uint num, uint numPrev,
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
41 double value, bool randomize) {
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
42 units[layer].length = num;
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
43 oldWeightUpdate[layer].length = num;
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
44 for(int i = 0; i < num; i++) {
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
45 units[layer][i].length = numPrev + 1; // include the bias weight
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
46 oldWeightUpdate[layer][i].length = numPrev + 1;
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
47
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
48 for(int j = 0; j < numPrev + 1; j++) {
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
49 if(randomize)
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
50 units[layer][i][j] = rnd() * value * 2 - value; // between -value and value
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
51 else
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
52 units[layer][i][j] = value;
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
53 oldWeightUpdate[layer][i][j] = 0;
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
54 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
55 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
56 }
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
57
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
58 ////////////////////////////////////////////////////// Evaluation //////////////////////////////////////////////////////
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
59 /// Evaluates the neural network.
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
60 public double[] evaluate(double[] inputs) {
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
61 return evaluateFull(inputs)[$ - 1]; // the last item (outputs) of the return value
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
62 }
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
63
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
64 /// Evaluates the neural network and returns the output from all units.
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
65 public double[][] evaluateFull(double[] inputs) {
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
66 if(inputs.length != numInputs)
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
67 throw new InputException("Wrong length of inputs.");
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
68 double[][] outputs;
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
69 outputs.length = units.length;
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
70 outputs[0] = evaluateLayer(0, inputs);
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
71 for(int i = 1; i < units.length; i++) {
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
72 outputs[i] = this.evaluateLayer(i, outputs[i - 1]);
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
73 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
74 return outputs;
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
75 }
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
76
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
77 // Helper function to evaluate the outputs of a single layer.
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
78 private double[] evaluateLayer(uint layer, double[] layerInputs) {
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
79 double[] output;
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
80 output.length = units[layer].length;
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
81 //printArray(layerInputs);
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
82 for(int i = 0; i < units[layer].length; i++) {
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
83 output[i] = evaluateUnit(layer, i, layerInputs);
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
84 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
85 return output;
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
86 }
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
87
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
88 // Helper function to evaluate the output of a single unit.
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
89 private double evaluateUnit(uint layer, uint unit, double[] layerInputs) {
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
90 //writef("(%d,%d)=",layer,unit);
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
91 //printArray(layerInputs);
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
92 double total = units[layer][unit][0]; //bias
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
93 for(int i = 1; i < layerInputs.length + 1; i++) {
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
94 total += layerInputs[i - 1] * units[layer][unit][i]; // wi * xi
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
95 //writef("@");
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
96 }
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
97 //writefln(" ! %f",total);
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
98 if(functions[layer] != null)
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
99 return functions[layer](total); // apply the function (if there is one)
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
100 writefln("no function");
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
101 return total; // just return the result instead
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
102 }
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
103
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
104 ////////////////////////////////////////////////////// Training //////////////////////////////////////////////////////
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
105 /// Trains the neural network.
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
106 /// TODO: Pull error calculation into a separate function. (maybe)
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
107 public void train(double[][] trainingInputs, double[][] trainingOutputs,
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
108 bool stochastic = false) {
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
109 if(trainingInputs.length != trainingOutputs.length)
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
110 throw new InputException(
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
111 "trainingInputs and trainingOutputs must be the same size");
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
112 double[][][] weightUpdate;
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
113 double[][] outputsError;
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
114 double[][] outputs;
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
115 double total; //temp variable
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
116
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
117 // Initialize the weightUpdate and outputsError variables
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
118 weightUpdate.length = units.length;
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
119 outputsError.length = units.length;
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
120 //writefln("#%d,%d",weightUpdate.length,outputsError.length);
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
121 for(int i = 0; i < units.length; i++) {
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
122 weightUpdate[i].length = units[i].length;
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
123 outputsError[i].length = units[i].length;
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
124 //writefln("##(%d)%d,%d",i,weightUpdate[i].length,outputsError[i].length);
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
125 for(int j = 0; j < weightUpdate[i].length; j++) {
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
126 weightUpdate[i][j].length = units[i][j].length;
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
127 for(int k = 0; k < weightUpdate[i][j].length; k++)
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
128 weightUpdate[i][j][k] = 0.0f;
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
129 //writefln("###(%d)%d",j,weightUpdate[i][j].length);
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
130 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
131 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
132
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
133
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
134 // Loop through each of the training examples
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
135 for(int example = 0; example < trainingInputs.length; example++) {
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
136 outputs = evaluateFull(trainingInputs[example]);
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
137
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
138
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
139 // Computing error of output layer
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
140 for(int i = 0; i < outputs[$ - 1].length; i++) { // units of last layer
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
141 //writefln("{%d,%d,%d,%d}",example,i,outputs.length,outputsError[$-1].length);
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
142 outputsError[$ - 1][i] = outputs[$ - 1][i] * (1 - outputs[$ - 1][i]) * (trainingOutputs[example][i] - outputs[$ - 1][i]);
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
143 }
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
144 // o(1-o)(t-o)
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
145
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
146 //printArray(outputsError[$-1]);
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
147 //printArray(units[length-1]);
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
148
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
149 //*
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
150 // Loop through each of the hidden layers (backwards - BACKpropagation!)
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
151 for(int layer = units.length - 2; layer >= 0; layer--) { // -2 to skip the output layer
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
152 //writef("|");
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
153 // loop through the units in each hidden layer
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
154 for(int unit = 0; unit < units[layer].length; unit++) {
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
155 //writef("*");
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
156 total = 0;
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
157 // total up w * e for the units the output of this unit goes into
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
158 for(int k = 0; k < units[layer + 1].length; k++) {
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
159 //writef("{weight=%f,error=%f}", units[layer+1][k][unit+1/* +1 for bias*/], outputsError[layer+1][k]);
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
160 total += units[layer + 1][k][unit + 1/* +1 for bias*/] * outputsError[layer + 1][k];
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
161 }
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
162 //writefln("=%f(total)",total);
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
163 // multiply total by o(1-o), store in outputsError
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
164 outputsError[layer][unit] = outputs[layer][unit] * (1 - outputs[layer][unit]) * total;
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
165 }
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
166 }
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
167 //writefln();
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
168
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
169 //writef("outputError="); printArray(outputsError);
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
170
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
171 // special case for the units that receive the input values
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
172 for(int unit = 0; unit < units[0].length; unit++) { // unit
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
173 //writefln(":%d,%d,%d,%d",j,weightUpdate.length,weightUpdate[0].length,weightUpdate[0][j].length);
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
174 weightUpdate[0][unit][0] += outputsError[0][unit]; //bias
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
175 for(int input = 1; input < units[0][unit].length; input++) { // input
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
176 weightUpdate[0][unit][input] += outputsError[0][unit] * trainingInputs[example][input - 1]; // account for bias
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
177 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
178 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
179
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
180
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
181 // Update the weightUpdate array OR update the weights
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
182 for(int i = 1; i < units.length; i++) { // layer
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
183 for(int j = 0; j < units[i].length; j++) { // unit
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
184 weightUpdate[i][j][0] += outputsError[i][j]; //bias
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
185 for(int k = 1; k < units[i][j].length; k++) { // input
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
186 //writefln("[%d,%d,%d]=%f; %f; %f",i,j,k,weightUpdate[i][j][k],outputsError[i][j],outputs[i-1][k-1]);
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
187 weightUpdate[i][j][k] += outputsError[i][j] * outputs[i - 1][k - 1]; // previous layer, account for bias
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
188 }
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
189 }
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
190 }
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
191 if(stochastic) {
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
192 // Apply the weightUpdate array to the weights
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
193 for(int i = 0; i < units.length; i++) { // layer
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
194 for(int j = 0; j < units[i].length; j++) { // unit
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
195 for(int k = 0; k < units[i][j].length; k++) { // input
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
196 units[i][j][k] += this.learningRate * weightUpdate[i][j][k] + (this.momentum * this.oldWeightUpdate[i][j][k]);
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
197 this.oldWeightUpdate[i][j][k] = weightUpdate[i][j][k];
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
198 weightUpdate[i][j][k] = 0;
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
199 }
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
200 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
201 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
202 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
203 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
204
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
205 if(!stochastic) {
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
206 // Apply the weightUpdate array to the weights
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
207 for(int i = 0; i < units.length; i++) { // layer
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
208 for(int j = 0; j < units[i].length; j++) { // unit
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
209 for(int k = 0; k < units[i][j].length; k++) { // input
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
210 //writefln("[%d,%d,%d]=%f; %f",i,j,k,units[i][j][k],weightUpdate[i][j][k]);
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
211 units[i][j][k] += this.learningRate * weightUpdate[i][j][k] + (this.momentum * this.oldWeightUpdate[i][j][k]);
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
212 }
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
213 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
214 }
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
215 this.oldWeightUpdate = weightUpdate;
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
216 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
217 }
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
218
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
219 /// Calculate the output error
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
220 double calculateError(double[][] trainingInputs, double[][] trainingOutputs) {
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
221 if(trainingInputs.length != trainingOutputs.length)
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
222 throw new InputException(
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
223 "trainingInputs and trainingOutputs must be the same size");
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
224 double[] outputs;
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
225 double total = 0, temp;
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
226 for(int i = 0; i < trainingInputs.length; i++) {
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
227 outputs = evaluate(trainingInputs[i]);
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
228 if(outputs.length != trainingOutputs[i].length)
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
229 throw new InputException("Wrong output length");
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
230 for(int j = 0; j < outputs.length; j++) {
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
231 temp = trainingOutputs[i][j] - outputs[j];
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
232 //writefln("&%f,%f",temp*temp,total);
3
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
233 total += temp * temp;
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
234 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
235 }
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
236 return 0.5 * total;
314d68bafeff Backprop and backprop_test added (no testing).
revcompgeek
parents:
diff changeset
237 }
5
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
238
810d58835f86 Added momentum and stochastic training to backprop.
revcompgeek
parents: 4
diff changeset
239 double[][][] getWeights() {
4
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
240 return units.dup;
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
241 }
73beed484455 Backprop working correctly.
revcompgeek
parents: 3
diff changeset
242 }