SuperNN  1.0.0
training.hpp
1 /*
2  This file is part of SuperNN.
3 
4  SuperNN is free software: you can redistribute it and/or modify
5  it under the terms of the GNU Lesser General Public License as published by
6  the Free Software Foundation, either version 3 of the License, or
7  (at your option) any later version.
8 
9  SuperNN is distributed in the hope that it will be useful,
10  but WITHOUT ANY WARRANTY; without even the implied warranty of
11  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  GNU Lesser General Public License for more details.
13 
14  You should have received a copy of the GNU Lesser General Public License
15  along with SuperNN. If not, see <http://www.gnu.org/licenses/>.
16 
17  Copyright (C) 2010 - 2015 Lucas Hermann Negri
18 */
19 
20 #ifndef SUPERNN_TRAINING_HPP
21 #define SUPERNN_TRAINING_HPP
22 
23 #include "utils.hpp"
24 
25 namespace SuperNN
26 {
27 struct Data;
28 struct Network;
29 
34 struct SUPERNN_EXPORT TrainingAlgorithm
35 {
36  virtual ~TrainingAlgorithm();
37 
47  virtual void prepare(Network &net);
48 
59  virtual unsigned train(Network &net, const Data &data, double dmse = 0,
60  unsigned max_epochs = 1000) = 0;
61 
79  virtual double delta(Network &net, unsigned l, unsigned n);
80 
89  virtual void derror_acc(Network &net);
90 
96  virtual void clear_derror_acc(Network &net);
97 
98 protected:
107  virtual void check(const Network& net, const Data& data) const;
108 };
109 
113 struct SUPERNN_EXPORT ImplBackprop : public TrainingAlgorithm
114 {
115  ImplBackprop();
116  virtual ~ImplBackprop();
117 
118  virtual unsigned train(Network &net, const Data &data, double dmse, unsigned max_epochs) = 0;
119 
121  double eta_df;
122 
124  double eta_if;
125 
127  double eta_min;
128 
130  double eta_max;
131 
133  double eta;
134 
135 protected:
144  virtual void update_weights(Network &net, double factor = 1);
145 
153  virtual void update_eta(double mse, double last_mse);
154 };
155 
160 struct SUPERNN_EXPORT Incremental : public ImplBackprop
161 {
162  Incremental();
163  virtual ~Incremental();
164  virtual unsigned train(Network &net, const Data &data, double dmse = 0, unsigned max_epochs = 1000);
165 };
166 
171 struct SUPERNN_EXPORT Batch : public ImplBackprop
172 {
173  Batch();
174  virtual ~Batch();
175  virtual unsigned train(Network &net, const Data &data, double dmse = 0, unsigned max_epochs = 1000);
176 };
177 
184 struct SUPERNN_EXPORT IRprop : public TrainingAlgorithm
185 {
186  IRprop();
187  virtual ~IRprop();
188 
189  virtual void prepare(Network &net);
190  virtual unsigned train(Network &net, const Data &data, double dmse = 0, unsigned max_epochs = 1000);
191 
193  double delta_df;
194 
196  double delta_if;
197 
199  double delta_min;
200 
202  double delta_max;
203 
205  double delta_zero;
206 
207 protected:
213  virtual void update_weights(Network &net);
214 };
215 
220 struct SUPERNN_EXPORT IRpropL1 : public IRprop
221 {
222 public:
223  IRpropL1();
224  virtual ~IRpropL1();
225  double delta(Network &net, unsigned l, unsigned n);
226 };
227 
234 struct SUPERNN_EXPORT NBN : public TrainingAlgorithm
235 {
236  NBN();
237  virtual ~NBN();
238 
239  void prepare(Network &net);
240  unsigned train(Network &net, const Data &data, double dmse = 0, unsigned max_epochs = 100);
241 
242 private:
254  double delta(Network &net, unsigned l, unsigned n, unsigned m);
255 
261  void get_weights(const Network &net);
262 
268  void set_weights(Network &net) const;
269 
276  void calc_jacobian_transp_line(Network &net, unsigned m);
277 
284  void update_hessian_gradient(double err);
285 
290  void update_weights(Network &net);
291 
300  bool update_mu(double mse, double last_mse);
301 
306  double mu_zero;
307 
311  double mu;
312 
316  double mu_min;
317 
321  double mu_max;
322 
326  double beta;
327 
331  unsigned local_iters;
332 
337  unsigned n_weights;
338 
339 private:
340  struct Hist;
341  Hist *h; // private data members
342 };
343 
344 /* higher level training helpers */
345 
356 SUPERNN_EXPORT double k_fold_error(TrainingAlgorithm &algo, const SuperNN::Network &net, const Data& data,
357  unsigned k = 10, double dmse = 0, unsigned max_epochs = 1000);
358 
375 SUPERNN_EXPORT unsigned early_stopping(TrainingAlgorithm& algo, Network& net, const Data& training,
376  const SuperNN::Data &validation, unsigned step_size = 1, unsigned max_stuck = 20, unsigned max_epochs = 1000);
377 
378 }
379 
380 #endif
Batch backpropagation.
Definition: training.hpp:171
double eta_df
Learning rate decrease factor (must be <= 1)
Definition: training.hpp:121
Modified improved resilient backpropagation algorithm.
Definition: training.hpp:220
Abstract class that provides the calculation of the error derivatives and the error accumulation...
Definition: training.hpp:34
Incremental backpropagation.
Definition: training.hpp:160
SUPERNN_EXPORT unsigned early_stopping(TrainingAlgorithm &algo, Network &net, const Data &training, const SuperNN::Data &validation, unsigned step_size=1, unsigned max_stuck=20, unsigned max_epochs=1000)
Trains an artificial neural network by using early stopping in order to avoid over-fitting.
Definition: training.cpp:625
double delta_zero
Initial weight change.
Definition: training.hpp:205
Improved resilient backpropagation algorithm.
Definition: training.hpp:184
double eta_max
Maximum learning rate.
Definition: training.hpp:130
double delta_if
Weight change increase factor.
Definition: training.hpp:196
double delta_df
Weight change decrease factor.
Definition: training.hpp:193
double eta_min
Minimum learning rate.
Definition: training.hpp:127
Neuron by Neuron algorithm.
Definition: training.hpp:234
Base class for the standard backpropagation algorithm.
Definition: training.hpp:113
double eta_if
Learning rate increase factor (must be >= 1)
Definition: training.hpp:124
double eta
Initial learning rate.
Definition: training.hpp:133
Artificial neural network structure that supports arbitrary feedforward topologies, like multilayer perceptrons and fully connected cascade networks.
Definition: network.hpp:78
Data used in training, validation and testing.
Definition: data.hpp:95
double delta_max
Maximum weight change.
Definition: training.hpp:202
SUPERNN_EXPORT double k_fold_error(TrainingAlgorithm &algo, const SuperNN::Network &net, const Data &data, unsigned k=10, double dmse=0, unsigned max_epochs=1000)
Estimates the performance of a neural network for an independent data set by using k-fold cross valid...
Definition: training.cpp:604
double delta_min
Minimum weight change.
Definition: training.hpp:199