SuperNN  0.7.0
network.cpp
1 /*
2  This file is part of SuperNN.
3 
4  SuperNN is free software: you can redistribute it and/or modify
5  it under the terms of the GNU Lesser General Public License as published by
6  the Free Software Foundation, either version 3 of the License, or
7  (at your option) any later version.
8 
9  SuperNN is distributed in the hope that it will be useful,
10  but WITHOUT ANY WARRANTY; without even the implied warranty of
11  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  GNU Lesser General Public License for more details.
13 
14  You should have received a copy of the GNU Lesser General Public License
15  along with SuperNN. If not, see <http://www.gnu.org/licenses/>.
16 
17  Copyright (C) 2010 - 2014 Lucas Hermann Negri
18 */
19 
20 #include <locale>
21 #include <fstream>
22 #include "activation.hpp"
23 #include "utils.hpp"
24 #include "neuron.hpp"
25 #include "network.hpp"
26 #include "foreach.hpp"
27 
28 namespace SuperNN
29 {
30 
32 {
33 }
34 
36 {
37 }
38 
40 {
41  push_back(n);
42 }
43 
44 void Layer::add_neurons(unsigned n_neurons, bool bias)
45 {
46  reserve(size() + n_neurons);
47 
48  foreach(n, 0, n_neurons)
49  {
50  Neuron l(bias);
51  add_neuron(l);
52  }
53 }
54 
55 void Layer::set_activation(ActFuncType type, double s)
56 {
57  foreach(n, 0, size())
58  at(n).set_activation(type, s);
59 }
60 
61 void Layer::connect(unsigned to_layer, unsigned to_neuron)
62 {
63  foreach(n, 0, size())
64  at(n).connect(to_layer, to_neuron);
65 }
66 
68 {
69  layers.push_back(l);
70 }
71 
72 void Network::add_layers(unsigned n_layers)
73 {
74  layers.reserve(layers.size() + n_layers);
75 
76  foreach(l, 0, n_layers)
77  {
78  Layer nl;
79  add_layer(nl);
80  }
81 }
82 
83 Network::Network() : n_input(0)
84 {
85 }
86 
88 {
89 }
90 
91 Network Network::make_mlp(unsigned input, unsigned hidden, unsigned output)
92 {
93  Network net;
94  net.add_layers(3);
95 
96  Layer &input_layer = net.layers[0];
97  Layer &hidden_layer = net.layers[1];
98  Layer &output_layer = net.layers[2];
99 
100  input_layer.add_neurons(input);
101  input_layer.add_neurons(1, true);
102  hidden_layer.add_neurons(hidden);
103  output_layer.add_neurons(output);
104 
105  net.connect(0, 1);
106  net.connect_neuron_to_layer(0, input, 2);
107  net.connect(1, 2);
108 
109  return net;
110 }
111 
112 Network Network::make_mlp(unsigned input, unsigned hidden1, unsigned hidden2, unsigned output)
113 {
114  Network net;
115  net.add_layers(4);
116 
117  Layer &input_layer = net.layers[0];
118  Layer &hidden1_layer = net.layers[1];
119  Layer &hidden2_layer = net.layers[2];
120  Layer &output_layer = net.layers[3];
121 
122  input_layer.add_neurons(input);
123  input_layer.add_neurons(1, true);
124  hidden1_layer.add_neurons(hidden1);
125  hidden2_layer.add_neurons(hidden2);
126  output_layer.add_neurons(output);
127 
128  net.connect(0, 1);
129 
130  net.connect_neuron_to_layer(0, input, 2);
131  net.connect(1, 2);
132 
133  net.connect_neuron_to_layer(0, input, 3);
134  net.connect(2, 3);
135 
136  return net;
137 }
138 
139 Network Network::make_mlp(unsigned input, unsigned output)
140 {
141  Network net;
142  net.add_layers(2);
143 
144  Layer &input_layer = net.layers[0];
145  Layer &output_layer = net.layers[1];
146 
147  input_layer.add_neurons(input);
148  input_layer.add_neurons(1, true);
149  output_layer.add_neurons(output);
150 
151  net.connect(0, 1);
152 
153  return net;
154 }
155 
156 Network Network::make_fcc(unsigned input, unsigned hidden, unsigned output)
157 {
158  Network net;
159  net.add_layers(hidden + 2);
160 
161  net.layers[0].add_neurons(input);
162  net.layers[0].add_neurons(1, true);
163 
164  foreach(l, 1, hidden + 1)
165  net.layers[l].add_neurons(1);
166 
167  unsigned last = net.layers.size() - 1;
168  net.layers[last].add_neurons(output);
169 
170  foreach(l1, 0, last)
171  foreach(l2, l1 + 1, last + 1)
172  net.connect(l1, l2);
173 
174  return net;
175 }
176 
177 const Row &Network::run(const Row &in, bool calc_error)
178 {
179  const unsigned last_layer = layers.size() - 1;
180  const unsigned in_size = in.size();
181  clear_neurons(true, true);
182 
183  /* copies the inputs to the first neuron layer and count the non-biases */
184  n_input = 0;
185 
186  foreach(i, 0, layers[0].size())
187  {
188  if(!layers[0][i].bias)
189  {
190  if(n_input >= in_size)
192  else
193  layers[0][i].out = in[n_input++];
194  }
195  }
196 
197  /* propagate the signal */
198  foreach(l, 0, last_layer)
199  {
200  foreach(n, 0, layers[l].size())
201  {
202  Neuron &neuron = layers[l][n];
203 
204  foreach(c, 0, neuron.size())
205  {
206  Connection &conn = neuron[c];
207  double v = neuron.out * conn.weight;
208  layers[conn.to_layer][conn.to_neuron].net += v;
209  }
210  }
211 
212  foreach(n, 0, layers[l + 1].size())
213  {
214  Neuron &neuron = layers[l + 1][n];
215  neuron.out = ActFunc::activation(neuron);
216  }
217  }
218 
219  const unsigned n_out = layers[last_layer].size();
220  last_output.resize(n_out);
221 
222  foreach(n, 0, n_out)
223  {
224  Neuron &neuron = layers[last_layer][n];
225  last_output[n] = neuron.out;
226 
227  if(calc_error)
228  neuron.err = in[n_input + n] - neuron.out;
229  }
230 
231  return last_output;
232 }
233 
234 double Network::calc_mse(const Data &data)
235 {
236  const unsigned n_out = layers[layers.size() - 1].size();
237  double mse = 0;
238 
239  foreach(p, 0, data.size())
240  {
241  const Row& r = run(data[p], true);
242 
243  foreach(n, 0, n_out)
244  {
245  double err = data[p][n_input + n] - r[n];
246  mse += err * err;
247  }
248  }
249 
250  mse /= data.size();
251 
252  return mse;
253 }
254 
255 double Network::calc_mae(const Data &data)
256 {
257  const unsigned n_out = layers[layers.size() - 1].size();
258  double mae = 0;
259 
260  foreach(p, 0, data.size())
261  {
262  const Row& r = run(data[p], true);
263 
264  foreach(n, 0, n_out)
265  {
266  double err = data[p][n_input + n] - r[n];
267  mae += std::abs(err);
268  }
269  }
270 
271  mae /= data.size();
272 
273  return mae;
274 }
275 
276 double Network::calc_class(const Data &data, double limit)
277 {
278  unsigned nt = 0;
279 
280  foreach(p, 0, data.size())
281  {
282  run(data[p]);
283  const unsigned last = layers.size() - 1;
284  unsigned s = 0;
285  const unsigned n_output = layers[last].size();
286 
287  foreach(n, 0, n_output)
288 
289  if(std::abs(layers[last][n].out - data[p][n_input + n]) < limit)
290  ++s;
291 
292  if(s == n_output)
293  ++nt;
294  }
295 
296  return nt * 100 / (double)data.size();
297 }
298 
299 double Network::calc_class_higher(const Data &data)
300 {
301  unsigned nt = 0;
302 
303  foreach(p, 0, data.size())
304  {
305  run(data[p]);
306  const unsigned last = layers.size() - 1;
307  const unsigned n_output = layers[last].size();
308  unsigned max1 = -1, max2 = -1;
309  double val1 = -1e30, val2 = -1e30;
310 
311  foreach(n, 0, n_output)
312  {
313  if(layers[last][n].out > val1)
314  val1 = layers[last][n].out, max1 = n;
315 
316  if(data[p][n_input + n] > val2)
317  val2 = data[p][n_input + n], max2 = n;
318  }
319 
320  if(max1 == max2)
321  ++nt;
322  }
323 
324  return nt * 100 / (double)data.size();
325 }
326 
327 void Network::init_weights(double min, double max)
328 {
329  foreach(l, 0, layers.size())
330  {
331  foreach(n, 0, layers[l].size())
332  {
333  Neuron &neuron = layers[l][n];
334 
335  foreach(c, 0, neuron.size())
336  neuron[c].weight = Utils::rand_double(max - min) + min;
337  }
338  }
339 }
340 
342 {
343  foreach(l, 0, layers.size())
344  layers[l].set_activation(type, s);
345 }
346 
347 void Network::save_file(const std::string &path) const
348 {
349  std::ofstream out;
350  out.open(path.c_str());
351 
352  if(!out.is_open())
354 
355  out.imbue(std::locale("C"));
356 
357  out << layers.size() << std::endl;
358  out.precision(file_precision);
359 
360  foreach(l, 0, layers.size())
361  {
362  out << layers[l].size() << std::endl;
363 
364  foreach(n, 0, layers[l].size())
365  {
366  const Neuron &neuron = layers[l][n];
367  out << neuron.act_func << " " << neuron.steep << " " << neuron.bias << std::endl;
368  out << neuron.size() << std::endl;
369 
370  foreach(c, 0, neuron.size())
371  {
372  const Connection &conn = neuron[c];
373  out << conn.weight << " " << conn.to_layer << " " << conn.to_neuron << std::endl;
374  }
375  }
376  }
377 
378  out.close();
379 }
380 
381 void Network::load_file(const std::string &path)
382 {
383  layers.clear();
384  last_output.clear();
385  n_input = 0;
386 
387  std::ifstream inp;
388  inp.open(path.c_str());
389 
390  if(!inp.is_open())
392 
393  inp.imbue(std::locale("C"));
394 
395  int n_layers;
396  inp >> n_layers;
397 
398  if(n_layers < 1)
399  {
400  inp.close();
402  }
403 
404  layers.resize(n_layers);
405 
406  foreach(l, 0, n_layers)
407  {
408  int n_neurons;
409  inp >> n_neurons;
410 
411  if(n_neurons < 1)
412  {
413  inp.close();
415  }
416 
417  layers[l].resize(n_neurons);
418 
419  foreach(n, 0, n_neurons)
420  {
421  Neuron &neuron = layers[l][n];
422  int act, bias;
423  inp >> act >> neuron.steep >> bias;
424  neuron.act_func = (ActFuncType)act;
425  neuron.bias = bias;
426 
427  int n_conns;
428  inp >> n_conns;
429 
430  if(n_conns < 0)
431  {
432  inp.close();
434  }
435 
436  neuron.conns.resize(n_conns);
437 
438  foreach(c, 0, n_conns)
439  {
440  Connection &conn = neuron[c];
441  inp >> conn.weight >> conn.to_layer >> conn.to_neuron;
442  }
443  }
444  }
445 
446  inp.close();
447 }
448 
449 void Network::connect(unsigned from_layer, unsigned to_layer)
450 {
451  foreach(n, 0, layers[to_layer].size())
452  layers[from_layer].connect(to_layer, n);
453 }
454 
455 void Network::connect_neuron_to_layer(unsigned from_layer, unsigned from_neuron, unsigned to_layer)
456 {
457  Neuron &neuron = layers[from_layer][from_neuron];
458 
459  foreach(n, 0, layers[to_layer].size())
460  neuron.connect(to_layer, n);
461 }
462 
463 void Network::clear_neurons(bool clear_delta, bool clear_run)
464 {
465  foreach(l, 0, layers.size())
466  {
467  foreach(n, 0, layers[l].size())
468  {
469  Neuron &neuron = layers[l][n];
470 
471  if(clear_delta)
472  {
473  neuron.delta = 0;
474  neuron.delta_ok = false;
475  }
476 
477  if(clear_run)
478  {
479  neuron.out = neuron.bias ? 1 : 0;
480  neuron.net = 0;
481  neuron.err = 0;
482  }
483  }
484  }
485 }
486 
488 {
489  unsigned n_weights = 0;
490 
491  foreach(l, 0, layers.size())
492  {
493  const Layer &layer = layers[l];
494  foreach(n, 0, layer.size())
495  n_weights += layer[n].size();
496  }
497 
498  return n_weights;
499 }
500 
502 {
503  unsigned n_neurons = 0;
504 
505  foreach(l, 0, layers.size())
506  n_neurons += layers[l].size();
507 
508  return n_neurons;
509 }
510 
511 unsigned Network::calc_num_inputs() const
512 {
513  unsigned n_inputs = 0;
514 
515  foreach(i, 0, layers[0].size())
516  {
517  if(!layers[0][i].bias)
518  ++n_inputs;
519  }
520 
521  return n_inputs;
522 }
523 
524 const Layer &Network::operator[](unsigned l) const
525 {
526  return layers[l];
527 }
528 
530 {
531  return layers[l];
532 }
533 
534 unsigned Network::size() const
535 {
536  return layers.size();
537 }
538 
539 }
Neuron, that can contain connections to neurons in the next layers.
Definition: neuron.hpp:70
std::vector< Connection > conns
Synaptic connections.
Definition: neuron.hpp:133
double calc_class(const Data &data, double limit=0.5)
Calculates the classification rate of the network related to a data.
Definition: network.cpp:276
void connect_neuron_to_layer(unsigned from_layer, unsigned from_neuron, unsigned to_layer)
Connects a neuron to all the neurons of another layer.
Definition: network.cpp:455
thrown when a file couldn't be opened
Definition: utils.hpp:35
unsigned calc_num_weights() const
Calculates the current number of weights.
Definition: network.cpp:487
virtual ~Layer()
Definition: network.cpp:35
void add_neurons(unsigned n_neurons, bool bias=false)
Adds a number of neurons to the layer.
Definition: network.cpp:44
Synaptic connection between two neurons.
Definition: neuron.hpp:32
void set_activation(ActFuncType type, double s=1)
Sets the activation function for all the neurons currently in the network.
Definition: network.cpp:341
double calc_mse(const Data &data)
Calculates the mean squared error of the network related to a data.
Definition: network.cpp:234
bool delta_ok
Marks if the delta has been calculated for the current iteration.
Definition: neuron.hpp:154
std::vector< Layer > layers
Neuron layers.
Definition: network.hpp:292
const unsigned file_precision
Precision used when writting floating point number to files.
Definition: utils.hpp:29
unsigned calc_num_inputs() const
Calculates the number of neurons on the first layer that aren't biases.
Definition: network.cpp:511
void add_layer(Layer &l)
Adds a layer to the network.
Definition: network.cpp:67
double calc_mae(const Data &data)
Calculates the mean absolute error of the network related to a data.
Definition: network.cpp:255
void add_neuron(Neuron &n)
Adds a neuron to the layer.
Definition: network.cpp:39
static Network make_mlp(unsigned input, unsigned hidden, unsigned output)
Constructs a 'standard' feed forward neural network with one hidden layer.
Definition: network.cpp:91
void add_layers(unsigned n_layers)
Adds a number of layers to the network.
Definition: network.cpp:72
double steep
Activation function steepness.
Definition: neuron.hpp:151
static double activation(const Neuron &neuron)
Calls the actual activation function.
Definition: activation.hpp:187
static Network make_fcc(unsigned input, unsigned hidden, unsigned output)
Constructs a fully connected cascade neural network.
Definition: network.cpp:156
void connect(unsigned to_layer, unsigned to_neuron)
Adds a connection to a neuron.
Definition: neuron.cpp:46
double weight
Weight.
Definition: neuron.hpp:49
double out
Last output of the neuron ( g(net) )
Definition: neuron.hpp:139
double rand_double(double max)
Returns a pseudo-random double.
Definition: utils.cpp:36
double delta
Last local error gradient.
Definition: neuron.hpp:145
unsigned n_input
Last computed number of neurons in the input layer that aren't biases, computed by run()...
Definition: network.hpp:303
void init_weights(double min=-0.5, double max=0.5)
Initializes the weights with pseudo-ramdom numbers.
Definition: network.cpp:327
void connect(unsigned from_layer, unsigned to_layer)
Connects all the neurons from a layer to all the neurons of another layer.
Definition: network.cpp:449
unsigned calc_num_neurons() const
Calculates the current number of neurons.
Definition: network.cpp:501
const Row & run(const Row &in, bool calc_error=false)
Propagates an input in the network.
Definition: network.cpp:177
unsigned size() const
Returns the number of synaptic connections.
Definition: neuron.hpp:111
double calc_class_higher(const Data &data)
Calculates the classification rate of the network related to a data.
Definition: network.cpp:299
ActFuncType act_func
Used activation function.
Definition: neuron.hpp:148
Artificial neural network structure that supports arbitrary feedforward topologies, like multilayer perceptrons and fully connected cascade networks.
Definition: network.hpp:78
void set_activation(ActFuncType type, double s=1)
Sets the activation function for all the neurons currently in the layer.
Definition: network.cpp:55
double err
Last error (desired - actual).
Definition: neuron.hpp:142
const Layer & operator[](unsigned l) const
Returns a const reference to a layer.
Definition: network.cpp:524
unsigned to_neuron
Position of the target neuron in it's layer.
Definition: neuron.hpp:64
The exception can be identified by the type() method.
Definition: utils.hpp:59
ActFuncType
Activation functions built-in in the library.
void connect(unsigned to_layer, unsigned to_neuron)
Connects all the neurons of the layer to a neuron.
Definition: network.cpp:61
void load_file(const std::string &path)
Loads the network contents from a file.
Definition: network.cpp:381
unsigned to_layer
Layer where the target neuron is located.
Definition: neuron.hpp:61
double net
Last sum of the neuron inputs.
Definition: neuron.hpp:136
std::vector< double > Row
Data row.
Definition: data.hpp:90
Array of neurons.
Definition: network.hpp:36
unsigned size() const
Returns the number of layers.
Definition: network.cpp:534
Row last_output
Structure that holds the last output values.
Definition: network.hpp:295
Data used in training, validation and testing.
Definition: data.hpp:95
thrown when a file has invalid contents
Definition: utils.hpp:38
bool bias
Marks if it's a bias neuron.
Definition: neuron.hpp:157
thrown when the dimensions of a Row and the network does not match
Definition: utils.hpp:50
void clear_neurons(bool clear_delta, bool clear_run)
Clears the neuron state.
Definition: network.cpp:463
void save_file(const std::string &path) const
Saves the network contents to a file, for latter use.
Definition: network.cpp:347
virtual ~Network()
Definition: network.cpp:87