21 #include <Eigen/Dense>
23 #include "activation.hpp"
27 #include "network.hpp"
28 #include "training.hpp"
35 std::vector<Layer> &layers = net.
layers;
36 Neuron &neuron = layers[l][n];
42 if(l == layers.size() - 1)
49 for(
unsigned c = 0, e = neuron.
size(); c < e; ++c)
78 if(total_size != data[0].size())
84 std::vector<Layer> &layers = net.
layers;
86 for(
unsigned l = 0, e = layers.size(); l < e; ++l)
88 for(
unsigned n = 0, e = layers[l].size(); n < e; ++n)
90 Neuron &neuron = layers[l][n];
91 for(
unsigned c = 0, e = neuron.
size(); c < e; ++c)
103 for(
unsigned l = 0, e = net.
layers.size(); l < e; ++l)
105 for(
unsigned n = 0, e = net.
layers[l].size(); n < e; ++n)
109 for(
unsigned c = 0, e = neuron.
size(); c < e; ++c)
110 neuron[c].derror = 0;
125 std::vector<Layer> &layers = net.
layers;
126 const double eta_f =
eta * factor;
128 for(
unsigned l = 0, e = layers.size(); l < e; ++l)
130 for(
unsigned n = 0, e = layers[l].size(); n < e; ++n)
132 Neuron &neuron = layers[l][n];
134 for(
unsigned c = 0, e = neuron.
size(); c < e; ++c)
172 std::vector<Layer> &layers = net.
layers;
174 double last_mse = -1;
177 for(e = 1; e <= max_epochs; ++e)
179 for(
unsigned p = 0, e = data.size(); p < e; ++p)
182 net.
run(data[p],
true);
184 for(
unsigned n = 0, e = layers[1].size(); n < e; ++n)
191 const double mse = net.
calc_mse(data);
194 if(mse <= dmse)
break;
213 std::vector<Layer> &layers = net.
layers;
215 double last_mse = -1, isize = data.size();
218 for(e = 1; e <= max_epochs; ++e)
222 for(
unsigned p = 0, e = data.size(); p < e; ++p)
224 net.
run(data[p],
true);
226 for(
unsigned n = 0, e = layers[1].size(); n < e; ++n)
234 const double mse = net.
calc_mse(data);
237 if(mse <= dmse)
break;
247 IRprop::IRprop() : delta_df(0.5), delta_if(1.2), delta_min(0), delta_max(50), delta_zero(0.1)
257 std::vector<Layer> &layers = net.
layers;
259 for(
unsigned l = 0, e = layers.size(); l < e; ++l)
261 for(
unsigned n = 0, e = layers[l].size(); n < e; ++n)
263 std::vector<Connection> &conns = layers[l][n].conns;
265 for(
unsigned c = 0, e = conns.size(); c < e; ++c)
276 std::vector<Layer> &layers = net.
layers;
278 for(
unsigned l = 0, e = layers.size(); l < e; ++l)
280 for(
unsigned n = 0, e = layers[l].size(); n < e; ++n)
282 Neuron &neuron = layers[l][n];
283 for(
unsigned c = 0, e = neuron.
size(); c < e; ++c)
317 std::vector<Layer> &layers = net.
layers;
320 for(e = 1; e <= max_epochs; ++e)
324 for(
unsigned p = 0, e = data.size(); p < e; ++p)
326 net.
run(data[p],
true);
328 for(
unsigned n = 0, e = layers[1].size(); n < e; ++n)
336 if(dmse > 0 && net.
calc_mse(data) <= dmse)
354 std::vector<Layer> &layers = net.
layers;
356 Neuron &neuron = layers[l][n];
362 if(l == layers.size() - 1)
365 neuron.
delta = neuron.
err > 0 ? gd : (neuron.
err < 0 ? -gd : 0);
369 for(
unsigned c = 0, e = neuron.
size(); c < e; ++c)
404 NBN::NBN() : h(new
NBN::
Hist()), n_weights(0), mu_zero(0.1), mu(mu_zero), beta(2.0), mu_min(1e-50), mu_max(1e50), local_iters(5)
418 h->
hessian = Eigen::MatrixXd(n_weights, n_weights);
419 h->
gradient = Eigen::VectorXd(n_weights);
420 h->
weights = Eigen::VectorXd(n_weights);
425 void NBN::get_weights(
const Network &net)
429 for(
unsigned l = 0, e = net.
layers.size(); l < e; ++l)
433 for(
unsigned n = 0, e = layer.size(); n < e; ++n)
435 for(
unsigned c = 0, e = layer[n].size(); c < e; ++c)
437 h->
weights(i) = layer[n][c].weight;
444 void NBN::set_weights(Network &net)
const
448 for(
unsigned l = 0, e = net.layers.size(); l < e; ++l)
450 Layer &layer = net.layers[l];
452 for(
unsigned n = 0, e = layer.size(); n < e; ++n)
454 for(
unsigned c = 0, e = layer[n].size(); c < e; ++c)
456 layer[n][c].weight = h->
weights(i);
463 double NBN::delta(Network &net,
unsigned l,
unsigned n,
unsigned m)
465 std::vector<Layer> &layers = net.layers;
466 Neuron &neuron = layers[l][n];
470 neuron.delta_ok =
true;
472 if(l == layers.size() - 1)
479 for(
unsigned c = 0, e = neuron.size(); c < e; ++c)
481 const Connection &conn = neuron[c];
482 const double d = delta(net, conn.to_layer, conn.to_neuron, m);
484 neuron.delta += d * conn.weight * gd;
492 void NBN::calc_jacobian_transp_line(Network &net,
unsigned m)
494 std::vector<Layer> &layers = net.layers;
496 net.clear_neurons(
true,
false);
499 for(
unsigned n = 0, e = layers[1].size(); n < e; ++n)
506 for(
unsigned l = 0, e = layers.size(); l < e; ++l)
508 const Layer &layer = layers[l];
510 for(
unsigned n = 0, e = layer.size(); n < e; ++n)
512 for(
unsigned c = 0, e = layer[n].size(); c < e; ++c)
521 void NBN::update_hessian_gradient(
double err)
527 void NBN::update_weights(Network &net)
529 const unsigned n = n_weights;
531 (h->
hessian + Eigen::MatrixXd::Identity(n, n) * mu).colPivHouseholderQr().solve(
536 bool NBN::update_mu(
double mse,
double last_mse)
561 double last_mse = net.
calc_mse(data);
567 for(e = 1; e <= max_epochs; ++e)
572 for(
unsigned p = 0, e = data.size(); p < e; ++p)
574 net.
run(data[p],
true);
576 for(
unsigned m = 0, e = last.size(); m < e; ++m)
578 calc_jacobian_transp_line(net, m);
579 update_hessian_gradient(last[m].err);
586 for(
unsigned tt = 0; tt < local_iters; ++tt)
591 if(update_mu(mse, last_mse))
605 double dmse,
unsigned max_epochs)
609 for(
unsigned n = 0; n < k; ++n)
613 Data validation, training;
614 data.
k_fold(n, k, validation, training);
617 algo.
train(cur_net, training, dmse, max_epochs);
619 acc += cur_net.
calc_mse(validation);
629 double best_mse = net.
calc_mse(validation);
630 unsigned best_it = 0;
632 unsigned stuck = 0, it = 0;
634 while( (stuck < max_stuck) && (it < max_epochs) )
636 algo.
train(net, training, -1, step_size);
639 const double new_mse = net.
calc_mse(validation);
640 if(new_mse < best_mse)
void k_fold(unsigned n, unsigned k, Data &p, Data &l) const
Fills two Data objects with complementary information, useful for cross-validation.
Neuron, that can contain connections to neurons in the next layers.
double eta_df
Learning rate decrease factor (must be <= 1)
virtual void check(const Network &net, const Data &data) const
Checks if the dimensions match and if the training algorithm can be used with a given network and dat...
void prepare(Network &net)
Prepares the trainer and a neural network for training.
double aux2
Auxiliary storage 2, used by some training algorithms.
virtual unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=1000)
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...
Abstract class that provides the calculation of the error derivatives and the error accumulation...
virtual void prepare(Network &net)
Prepares the trainer and a neural network for training.
unsigned calc_num_weights() const
Calculates the current number of weights.
virtual void derror_acc(Network &net)
Accumulates the error partial derivative in respect to the weights, for each connection of the neural...
SUPERNN_EXPORT unsigned early_stopping(TrainingAlgorithm &algo, Network &net, const Data &training, const SuperNN::Data &validation, unsigned step_size=1, unsigned max_stuck=20, unsigned max_epochs=1000)
Trains an artificial neural network by using early stopping in order to avoid over-fitting.
double delta_zero
Initial weight change.
Synaptic connection between two neurons.
Eigen::VectorXd gradient
Error gradient matrix.
virtual void update_weights(Network &net)
Updates the weights, using the partial error derivative sign change as guide.
double calc_mse(const Data &data)
Calculates the mean squared error of the network related to a data.
virtual void update_eta(double mse, double last_mse)
Calculates the new learning rate (and updates it), based on the mean squared error last change...
bool delta_ok
Marks if the delta has been calculated for the current iteration.
virtual void update_weights(Network &net, double factor=1)
Updates the weights using the accumulated error partial derivative calculated by derror_acc().
std::vector< Layer > layers
Neuron layers.
double aux1
Auxiliary storage 1, used by some training algorithms.
unsigned calc_num_inputs() const
Calculates the number of neurons on the first layer that aren't biases.
Improved resilient backpropagation algorithm.
double eta_max
Maximum learning rate.
double delta_if
Weight change increase factor.
virtual unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=1000)
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...
Eigen::VectorXd t_jacob_line
Transposed Jacobian line.
virtual unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=1000)
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...
double delta_df
Weight change decrease factor.
virtual void prepare(Network &net)
Prepares the trainer and a neural network for training.
double out
Last output of the neuron ( g(net) )
double eta_min
Minimum learning rate.
Eigen::MatrixXd hessian
Quasi-Hessian matrix.
virtual ~TrainingAlgorithm()
thrown when calling a function with invalid parameters
Neuron by Neuron algorithm.
double delta
Last local error gradient.
static double derivative(const Neuron &neuron)
Calls the actual derivative of the activation function, used to calculate the error gradient...
double eta_if
Learning rate increase factor (must be >= 1)
virtual double delta(Network &net, unsigned l, unsigned n)
Calculates the local error gradient for each neuron.
const Row & run(const Row &in, bool calc_error=false)
Propagates an input in the network.
double delta(Network &net, unsigned l, unsigned n)
Calculates the local error gradient for each neuron.
double eta
Initial learning rate.
unsigned size() const
Returns the number of synaptic connections.
Artificial neural network structure that supports arbitrary feedforward topologies, like multilayer perceptrons and fully connected cascade networks.
double err
Last error (desired - actual).
unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=100)
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...
unsigned to_neuron
Position of the target neuron in it's layer.
The exception can be identified by the type() method.
virtual void clear_derror_acc(Network &net)
Clears the accumulated error partial derivatives.
unsigned to_layer
Layer where the target neuron is located.
Eigen::VectorXd weights
Weight matrix (same as the network weights, but in a suitable format to the algorithm.
Eigen::VectorXd weights_backup
Backup of the weights, for backtracking.
Data used in training, validation and testing.
double delta_max
Maximum weight change.
double derror
Accumulated partial error derivative in respect to the connection weight.
thrown when the dimensions of a Row and the network does not match
SUPERNN_EXPORT double k_fold_error(TrainingAlgorithm &algo, const SuperNN::Network &net, const Data &data, unsigned k=10, double dmse=0, unsigned max_epochs=1000)
Estimates the performance of a neural network for an independent data set by using k-fold cross valid...
double delta_min
Minimum weight change.
double limit(double min, double max, double value)
Returns the value limited to a range.
virtual unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=1000)=0
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...