SuperNN  0.7.0
training.hpp
1 /*
2  This file is part of SuperNN.
3 
4  SuperNN is free software: you can redistribute it and/or modify
5  it under the terms of the GNU Lesser General Public License as published by
6  the Free Software Foundation, either version 3 of the License, or
7  (at your option) any later version.
8 
9  SuperNN is distributed in the hope that it will be useful,
10  but WITHOUT ANY WARRANTY; without even the implied warranty of
11  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  GNU Lesser General Public License for more details.
13 
14  You should have received a copy of the GNU Lesser General Public License
15  along with SuperNN. If not, see <http://www.gnu.org/licenses/>.
16 
17  Copyright (C) 2010 - 2014 Lucas Hermann Negri
18 */
19 
20 #ifndef SUPERNN_TRAINING_HPP
21 #define SUPERNN_TRAINING_HPP
22 
23 namespace SuperNN
24 {
25 class Data;
26 class Network;
27 
33 {
34  virtual ~TrainingAlgorithm();
35 
45  virtual void prepare(Network &net);
46 
57  virtual unsigned train(Network &net, const Data &data, double dmse = 0,
58  unsigned max_epochs = 1000) = 0;
59 
77  virtual double delta(Network &net, unsigned l, unsigned n);
78 
87  virtual void derror_acc(Network &net);
88 
94  virtual void clear_derror_acc(Network &net);
95 
96 protected:
105  virtual void check(const Network& net, const Data& data) const;
106 };
107 
112 {
113  ImplBackprop();
114  virtual ~ImplBackprop();
115 
116  virtual unsigned train(Network &net, const Data &data, double dmse, unsigned max_epochs) = 0;
117 
119  double eta_df;
120 
122  double eta_if;
123 
125  double eta_min;
126 
128  double eta_max;
129 
131  double eta;
132 
133 protected:
142  virtual void update_weights(Network &net, double factor = 1);
143 
151  virtual void update_eta(double mse, double last_mse);
152 };
153 
158 struct Incremental : public ImplBackprop
159 {
160  Incremental();
161  virtual ~Incremental();
162  virtual unsigned train(Network &net, const Data &data, double dmse = 0, unsigned max_epochs = 1000);
163 };
164 
169 struct Batch : public ImplBackprop
170 {
171  Batch();
172  virtual ~Batch();
173  virtual unsigned train(Network &net, const Data &data, double dmse = 0, unsigned max_epochs = 1000);
174 };
175 
182 struct IRprop : public TrainingAlgorithm
183 {
184  IRprop();
185  virtual ~IRprop();
186 
187  virtual void prepare(Network &net);
188  virtual unsigned train(Network &net, const Data &data, double dmse = 0, unsigned max_epochs = 1000);
189 
191  double delta_df;
192 
194  double delta_if;
195 
197  double delta_min;
198 
200  double delta_max;
201 
203  double delta_zero;
204 
205 protected:
211  virtual void update_weights(Network &net);
212 };
213 
218 struct IRpropL1 : public IRprop
219 {
220 public:
221  IRpropL1();
222  virtual ~IRpropL1();
223  double delta(Network &net, unsigned l, unsigned n);
224 };
225 
232 struct NBN : public TrainingAlgorithm
233 {
234  NBN();
235  virtual ~NBN();
236 
237  void prepare(Network &net);
238  unsigned train(Network &net, const Data &data, double dmse = 0, unsigned max_epochs = 100);
239 
240 private:
252  double delta(Network &net, unsigned l, unsigned n, unsigned m);
253 
259  void get_weights(const Network &net);
260 
266  void set_weights(Network &net) const;
267 
274  void calc_jacobian_transp_line(Network &net, unsigned m);
275 
282  void update_hessian_gradient(double err);
283 
288  void update_weights(Network &net);
289 
298  bool update_mu(double mse, double last_mse);
299 
300  struct Params;
301  Params *p; // private data members
302 
305  unsigned n_weights;
306 };
307 
308 /* higher level training helpers */
309 
320 double k_fold_error(TrainingAlgorithm &algo, const SuperNN::Network &net, const Data& data,
321  unsigned k = 10, double dmse = 0, unsigned max_epochs = 1000);
322 
339 unsigned early_stopping(TrainingAlgorithm& algo, Network& net, const Data& training,
340  const SuperNN::Data &validation, int step_size = 1, int max_stuck = 20, int max_epochs = 1000);
341 
342 }
343 
344 #endif
Batch backpropagation.
Definition: training.hpp:169
double eta_df
Learning rate decrease factor (must be <= 1)
Definition: training.hpp:119
virtual void check(const Network &net, const Data &data) const
Checks if the dimensions match and if the training algorithm can be used with a given network and dat...
Definition: training.cpp:72
void prepare(Network &net)
Prepares the trainer and a neural network for training.
Definition: training.cpp:432
Modified improved resilient backpropagation algorithm.
Definition: training.hpp:218
virtual unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=1000)
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...
Definition: training.cpp:211
Abstract class that provides the calculation of the error derivatives and the error accumulation...
Definition: training.hpp:32
virtual void prepare(Network &net)
Prepares the trainer and a neural network for training.
Definition: training.cpp:68
Incremental backpropagation.
Definition: training.hpp:158
unsigned early_stopping(TrainingAlgorithm &algo, Network &net, const Data &training, const SuperNN::Data &validation, int step_size=1, int max_stuck=20, int max_epochs=1000)
Trains an artificial neural network by using early stopping in order to avoid over-fitting.
Definition: training.cpp:642
virtual void derror_acc(Network &net)
Accumulates the error partial derivative in respect to the weights, for each connection of the neural...
Definition: training.cpp:83
double delta_zero
Initial weight change.
Definition: training.hpp:203
virtual void update_weights(Network &net)
Updates the weights, using the partial error derivative sign change as guide.
Definition: training.cpp:274
virtual void update_eta(double mse, double last_mse)
Calculates the new learning rate (and updates it), based on the mean squared error last change...
Definition: training.cpp:144
virtual ~Batch()
Definition: training.cpp:207
virtual void update_weights(Network &net, double factor=1)
Updates the weights using the accumulated error partial derivative calculated by derror_acc().
Definition: training.cpp:124
Improved resilient backpropagation algorithm.
Definition: training.hpp:182
double eta_max
Maximum learning rate.
Definition: training.hpp:128
double delta_if
Weight change increase factor.
Definition: training.hpp:194
virtual unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=1000)
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...
Definition: training.cpp:314
virtual unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=1000)
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...
Definition: training.cpp:170
double delta_df
Weight change decrease factor.
Definition: training.hpp:191
virtual ~NBN()
Definition: training.cpp:427
virtual void prepare(Network &net)
Prepares the trainer and a neural network for training.
Definition: training.cpp:256
double eta_min
Minimum learning rate.
Definition: training.hpp:125
virtual ~ImplBackprop()
Definition: training.cpp:120
virtual ~IRpropL1()
Definition: training.cpp:348
Neuron by Neuron algorithm.
Definition: training.hpp:232
Base class for the standard backpropagation algorithm.
Definition: training.hpp:111
double eta_if
Learning rate increase factor (must be >= 1)
Definition: training.hpp:122
virtual double delta(Network &net, unsigned l, unsigned n)
Calculates the local error gradient for each neuron.
Definition: training.cpp:34
virtual ~Incremental()
Definition: training.cpp:166
double delta(Network &net, unsigned l, unsigned n)
Calculates the local error gradient for each neuron.
Definition: training.cpp:352
double eta
Initial learning rate.
Definition: training.hpp:131
Artificial neural network structure that supports arbitrary feedforward topologies, like multilayer perceptrons and fully connected cascade networks.
Definition: network.hpp:78
virtual ~IRprop()
Definition: training.cpp:252
unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=100)
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...
Definition: training.cpp:575
virtual void clear_derror_acc(Network &net)
Clears the accumulated error partial derivatives.
Definition: training.cpp:102
virtual unsigned train(Network &net, const Data &data, double dmse, unsigned max_epochs)=0
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...
Data used in training, validation and testing.
Definition: data.hpp:95
double delta_max
Maximum weight change.
Definition: training.hpp:200
double k_fold_error(TrainingAlgorithm &algo, const SuperNN::Network &net, const Data &data, unsigned k=10, double dmse=0, unsigned max_epochs=1000)
Estimates the performance of a neural network for an independent data set by using k-fold cross valid...
Definition: training.cpp:621
double delta_min
Minimum weight change.
Definition: training.hpp:197
virtual unsigned train(Network &net, const Data &data, double dmse=0, unsigned max_epochs=1000)=0
Adjusts the synaptic weight of an artificial neural network in order to minimize the error (MSE for s...