21 #include <Eigen/Dense>
23 #include "activation.hpp"
27 #include "network.hpp"
28 #include "training.hpp"
29 #include "foreach.hpp"
36 std::vector<Layer> &layers = net.
layers;
37 Neuron &neuron = layers[l][n];
43 if(l == layers.size() - 1)
50 foreach(c, 0, neuron.
size())
79 if(total_size != data[0].size())
85 std::vector<Layer> &layers = net.
layers;
87 foreach(l, 0, layers.size())
89 foreach(n, 0, layers[l].size())
91 Neuron &neuron = layers[l][n];
92 foreach(c, 0, neuron.
size())
104 foreach(l, 0, net.
layers.size())
106 foreach(n, 0, net.
layers[l].size())
110 foreach(c, 0, neuron.
size())
111 neuron[c].derror = 0;
126 std::vector<Layer> &layers = net.
layers;
127 const double eta_f =
eta * factor;
129 foreach(l, 0, layers.size())
131 foreach(n, 0, layers[l].size())
133 Neuron &neuron = layers[l][n];
135 foreach(c, 0, neuron.
size())
173 std::vector<Layer> &layers = net.
layers;
175 double last_mse = -1;
178 for(e = 1; e <= max_epochs; ++e)
180 foreach(p, 0, data.size())
183 net.
run(data[p],
true);
185 foreach(n, 0, layers[1].size())
192 const double mse = net.
calc_mse(data);
195 if(mse <= dmse)
break;
214 std::vector<Layer> &layers = net.
layers;
216 double last_mse = -1, isize = data.size();
219 for(e = 1; e <= max_epochs; ++e)
223 foreach(p, 0, data.size())
225 net.
run(data[p],
true);
227 foreach(n, 0, layers[1].size())
235 const double mse = net.
calc_mse(data);
238 if(mse <= dmse)
break;
248 IRprop::IRprop() : delta_df(0.5), delta_if(1.2), delta_min(0), delta_max(50), delta_zero(0.1)
258 std::vector<Layer> &layers = net.
layers;
260 foreach(l, 0, layers.size())
262 foreach(n, 0, layers[l].size())
264 std::vector<Connection> &conns = layers[l][n].conns;
265 foreach(c, 0, conns.size())
276 std::vector<Layer> &layers = net.
layers;
278 foreach(l, 0, layers.size())
280 foreach(n, 0, layers[l].size())
282 Neuron &neuron = layers[l][n];
283 foreach(c, 0, neuron.
size())
317 std::vector<Layer> &layers = net.
layers;
320 for(e = 1; e <= max_epochs; ++e)
324 foreach(p, 0, data.size())
326 net.
run(data[p],
true);
328 foreach(n, 0, layers[1].size())
336 if(dmse > 0 && net.
calc_mse(data) <= dmse)
354 std::vector<Layer> &layers = net.
layers;
356 Neuron &neuron = layers[l][n];
362 if(l == layers.size() - 1)
365 neuron.
delta = neuron.
err > 0 ? gd : (neuron.
err < 0 ? -gd : 0);
369 foreach(c, 0, neuron.
size())
436 p->
hessian = Eigen::MatrixXd(n_weights, n_weights);
437 p->
gradient = Eigen::VectorXd(n_weights);
438 p->
weights = Eigen::VectorXd(n_weights);
443 void NBN::get_weights(
const Network &net)
447 foreach(l, 0, net.
layers.size())
451 foreach(n, 0, layer.size())
453 foreach(c, 0, layer[n].size())
455 p->
weights(i) = layer[n][c].weight;
462 void NBN::set_weights(Network &net)
const
466 foreach(l, 0, net.layers.size())
468 Layer &layer = net.layers[l];
470 foreach(n, 0, layer.size())
472 foreach(c, 0, layer[n].size())
474 layer[n][c].weight = p->
weights(i);
481 double NBN::delta(Network &net,
unsigned l,
unsigned n,
unsigned m)
483 std::vector<Layer> &layers = net.layers;
484 Neuron &neuron = layers[l][n];
488 neuron.delta_ok =
true;
490 if(l == layers.size() - 1)
497 foreach(c, 0, neuron.size())
499 const Connection &conn = neuron[c];
500 const double d = delta(net, conn.to_layer, conn.to_neuron, m);
502 neuron.delta += d * conn.weight * gd;
510 void NBN::calc_jacobian_transp_line(Network &net,
unsigned m)
512 std::vector<Layer> &layers = net.layers;
514 net.clear_neurons(
true,
false);
517 foreach(n, 0, layers[1].size())
524 foreach(l, 0, layers.size())
526 const Layer &layer = layers[l];
528 foreach(n, 0, layer.size())
530 foreach(c, 0, layer[n].size())
539 void NBN::update_hessian_gradient(
double err)
545 void NBN::update_weights(Network &net)
547 const unsigned n = n_weights;
549 (p->
hessian + Eigen::MatrixXd::Identity(n, n) * p->
mu).colPivHouseholderQr().solve(
554 bool NBN::update_mu(
double mse,
double last_mse)
579 double last_mse = net.
calc_mse(data);
585 for(e = 1; e <= max_epochs; ++e)
590 foreach(p, 0, data.size())
592 net.
run(data[p],
true);
594 foreach(m, 0, last.size())
596 calc_jacobian_transp_line(net, m);
597 update_hessian_gradient(last[m].err);
608 if(update_mu(mse, last_mse))
622 double dmse,
unsigned max_epochs)
630 Data validation, training;
631 data.
k_fold(n, k, validation, training);
634 algo.
train(cur_net, training, dmse, max_epochs);
636 acc += cur_net.
calc_mse(validation);
646 double best_mse = net.
calc_mse(validation);
647 unsigned best_it = 0;
649 unsigned stuck = 0, it = 0;
651 while( (stuck < max_stuck) && (it < max_epochs) )
653 algo.
train(net, training, -1, step_size);
656 const double new_mse = net.
calc_mse(validation);
657 if(new_mse < best_mse)
void k_fold(unsigned n, unsigned k, Data &p, Data &l) const
Fills two Data objects with complementary information, useful for cross-validation.
Neuron, that can contain connections to neurons in the next layers.
double eta_df
Learning rate decrease factor (must be <= 1)
Eigen::MatrixXd hessian
Quasi-Hessian matrix.
virtual void check(const Network &net, const Data &data) const
Checks if the dimensions match and if the training algorithm can be used with a given network and dat...
void prepare(Network &net)
Prepares the trainer and a neural network for training.
double aux2
Auxiliary storage 2, used by some training algorithms.
virtual unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=1000)
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...
Abstract class that provides the calculation of the error derivatives and the error accumulation...
virtual void prepare(Network &net)
Prepares the trainer and a neural network for training.
unsigned early_stopping(TrainingAlgorithm &algo, Network &net, const Data &training, const SuperNN::Data &validation, int step_size=1, int max_stuck=20, int max_epochs=1000)
Trains an artificial neural network by using early stopping in order to avoid over-fitting.
double mu_zero
Initial value for mu.
unsigned calc_num_weights() const
Calculates the current number of weights.
virtual void derror_acc(Network &net)
Accumulates the error partial derivative in respect to the weights, for each connection of the neural...
double delta_zero
Initial weight change.
Synaptic connection between two neurons.
virtual void update_weights(Network &net)
Updates the weights, using the partial error derivative sign change as guide.
double calc_mse(const Data &data)
Calculates the mean squared error of the network related to a data.
virtual void update_eta(double mse, double last_mse)
Calculates the new learning rate (and updates it), based on the mean squared error last change...
bool delta_ok
Marks if the delta has been calculated for the current iteration.
Eigen::VectorXd weights_backup
Backup of the weights, for backtracking.
virtual void update_weights(Network &net, double factor=1)
Updates the weights using the accumulated error partial derivative calculated by derror_acc().
std::vector< Layer > layers
Neuron layers.
double aux1
Auxiliary storage 1, used by some training algorithms.
unsigned calc_num_inputs() const
Calculates the number of neurons on the first layer that aren't biases.
Improved resilient backpropagation algorithm.
double eta_max
Maximum learning rate.
double delta_if
Weight change increase factor.
virtual unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=1000)
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...
Eigen::VectorXd weights
Weight matrix (same as the network weights, but in a suitable format to the algorithm.
virtual unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=1000)
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...
double delta_df
Weight change decrease factor.
virtual void prepare(Network &net)
Prepares the trainer and a neural network for training.
double out
Last output of the neuron ( g(net) )
double eta_min
Minimum learning rate.
virtual ~TrainingAlgorithm()
thrown when calling a function with invalid parameters
Neuron by Neuron algorithm.
Eigen::VectorXd gradient
Error gradient matrix.
double delta
Last local error gradient.
static double derivative(const Neuron &neuron)
Calls the actual derivative of the activation function, used to calculate the error gradient...
double eta_if
Learning rate increase factor (must be >= 1)
virtual double delta(Network &net, unsigned l, unsigned n)
Calculates the local error gradient for each neuron.
const Row & run(const Row &in, bool calc_error=false)
Propagates an input in the network.
double delta(Network &net, unsigned l, unsigned n)
Calculates the local error gradient for each neuron.
double eta
Initial learning rate.
double mu_min
Minimum value of mu.
unsigned size() const
Returns the number of synaptic connections.
double beta
mu multiply/divide factor
Artificial neural network structure that supports arbitrary feedforward topologies, like multilayer perceptrons and fully connected cascade networks.
double err
Last error (desired - actual).
unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=100)
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...
unsigned to_neuron
Position of the target neuron in it's layer.
The exception can be identified by the type() method.
virtual void clear_derror_acc(Network &net)
Clears the accumulated error partial derivatives.
unsigned to_layer
Layer where the target neuron is located.
Eigen::VectorXd t_jacob_line
Transposed Jacobian line.
double mu_max
Maximum value of mu.
Data used in training, validation and testing.
double delta_max
Maximum weight change.
double derror
Accumulated partial error derivative in respect to the connection weight.
thrown when the dimensions of a Row and the network does not match
double k_fold_error(TrainingAlgorithm &algo, const SuperNN::Network &net, const Data &data, unsigned k=10, double dmse=0, unsigned max_epochs=1000)
Estimates the performance of a neural network for an independent data set by using k-fold cross valid...
double delta_min
Minimum weight change.
double mu
Current value of mu.
double limit(double min, double max, double value)
Returns the value limited to a range.
virtual unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=1000)=0
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...