Line data Source code
1 : /*************************************************************************
2 : *
3 : * Project
4 : * _____ _____ __ __ _____
5 : * / ____| __ \| \/ | __ \
6 : * ___ _ __ ___ _ __ | | __| |__) | \ / | |__) |
7 : * / _ \| '_ \ / _ \ '_ \| | |_ | ___/| |\/| | ___/
8 : *| (_) | |_) | __/ | | | |__| | | | | | | |
9 : * \___/| .__/ \___|_| |_|\_____|_| |_| |_|_|
10 : * | |
11 : * |_|
12 : *
13 : * Copyright (C) Akiel Aries, <akiel@akiel.org>, et al.
14 : *
15 : * This software is licensed as described in the file LICENSE, which
16 : * you should have received as part of this distribution. The terms
17 : * among other details are referenced in the official documentation
18 : * seen here : https://akielaries.github.io/openGPMP/ along with
19 : * important files seen in this project.
20 : *
21 : * You may opt to use, copy, modify, merge, publish, distribute
22 : * and/or sell copies of the Software, and permit persons to whom
23 : * the Software is furnished to do so, under the terms of the
24 : * LICENSE file. As this is an Open Source effort, all implementations
25 : * must be of the same methodology.
26 : *
27 : *
28 : *
29 : * This software is distributed on an AS IS basis, WITHOUT
30 : * WARRANTY OF ANY KIND, either express or implied.
31 : *
32 : ************************************************************************/
33 : #include <algorithm>
34 : #include <ctime>
35 : #include <fstream>
36 : #include <iostream>
37 : #include <openGPMP/ml/encoder.hpp>
38 : #include <random>
39 : #include <vector>
40 :
41 0 : void gpmp::ml::AutoEncoder::save(const std::string &filename) const {
42 0 : std::ofstream file(filename, std::ios::out | std::ios::binary);
43 :
44 0 : if (file.is_open()) {
45 0 : file.write(reinterpret_cast<const char *>(&weights_input_hidden[0][0]),
46 0 : weights_input_hidden.size() *
47 0 : weights_input_hidden[0].size() * sizeof(double));
48 0 : file.write(reinterpret_cast<const char *>(&weights_hidden_output[0][0]),
49 0 : weights_hidden_output.size() *
50 0 : weights_hidden_output[0].size() * sizeof(double));
51 :
52 0 : file.close();
53 0 : std::cout << "Model saved successfully." << std::endl;
54 : } else {
55 0 : std::cerr << "Unable to open the file for saving." << std::endl;
56 : }
57 0 : }
58 :
59 0 : void gpmp::ml::AutoEncoder::load(const std::string &filename) {
60 0 : std::ifstream file(filename, std::ios::in | std::ios::binary);
61 :
62 0 : if (file.is_open()) {
63 0 : file.read(reinterpret_cast<char *>(&weights_input_hidden[0][0]),
64 0 : weights_input_hidden.size() * weights_input_hidden[0].size() *
65 : sizeof(double));
66 0 : file.read(reinterpret_cast<char *>(&weights_hidden_output[0][0]),
67 0 : weights_hidden_output.size() *
68 0 : weights_hidden_output[0].size() * sizeof(double));
69 :
70 0 : file.close();
71 0 : std::cout << "Model loaded successfully." << std::endl;
72 : } else {
73 0 : std::cerr << "Unable to open the file for loading." << std::endl;
74 : }
75 0 : }
76 :
77 0 : void gpmp::ml::AutoEncoder::lrate_set(double initial_rate) {
78 0 : learning_rate = initial_rate;
79 0 : }
80 :
81 0 : void gpmp::ml::AutoEncoder::lrate_update(int epoch) {
82 : // reduce the learning rate by half every N epochs
83 0 : const int decay_interval = 10;
84 0 : if (epoch % decay_interval == 0) {
85 0 : learning_rate /= 2.0;
86 0 : std::cout << "Learning rate updated to: " << learning_rate
87 0 : << " at epoch " << epoch << std::endl;
88 : }
89 : // TODO?
90 0 : }
91 :
92 0 : gpmp::ml::AutoEncoder::AutoEncoder(int in_size,
93 : int h_size,
94 : int out_size,
95 0 : double l_rate)
96 0 : : input_size(in_size), hidden_size(h_size), output_size(out_size),
97 0 : learning_rate(l_rate) {
98 :
99 : // initialize weights randomly
100 0 : weights_input_hidden.resize(input_size, std::vector<double>(hidden_size));
101 0 : weights_hidden_output.resize(hidden_size, std::vector<double>(output_size));
102 :
103 0 : for (int i = 0; i < input_size; ++i) {
104 0 : for (int j = 0; j < hidden_size; ++j) {
105 : // random values between 0 and 1
106 0 : weights_input_hidden[i][j] = (rand() % 1000) / 1000.0;
107 : }
108 : }
109 0 : for (int i = 0; i < hidden_size; ++i) {
110 0 : for (int j = 0; j < output_size; ++j) {
111 0 : weights_hidden_output[i][j] = (rand() % 1000) / 1000.0;
112 : }
113 : }
114 0 : }
115 :
116 : std::vector<double>
117 0 : gpmp::ml::AutoEncoder::sigmoid(const std::vector<double> &x) {
118 0 : std::vector<double> result;
119 0 : for (double val : x) {
120 0 : result.push_back(1.0 / (1.0 + exp(-val)));
121 : }
122 0 : return result;
123 0 : }
124 :
125 : std::vector<double>
126 0 : gpmp::ml::AutoEncoder::forward(const std::vector<double> &input) {
127 : // forward passes
128 0 : std::vector<double> hidden(hidden_size);
129 0 : std::vector<double> output(output_size);
130 :
131 : // calculate hidden layer values
132 0 : for (int i = 0; i < hidden_size; ++i) {
133 0 : hidden[i] = 0;
134 0 : for (int j = 0; j < input_size; ++j) {
135 0 : hidden[i] += input[j] * weights_input_hidden[j][i];
136 : }
137 0 : hidden[i] = sigmoid({hidden[i]})[0];
138 : }
139 :
140 : // calculate output layer values
141 0 : for (int i = 0; i < output_size; ++i) {
142 0 : output[i] = 0;
143 0 : for (int j = 0; j < hidden_size; ++j) {
144 0 : output[i] += hidden[j] * weights_hidden_output[j][i];
145 : }
146 0 : output[i] = sigmoid({output[i]})[0];
147 : }
148 :
149 0 : return output;
150 0 : }
151 :
152 0 : void gpmp::ml::AutoEncoder::train(
153 : const std::vector<std::vector<double>> &training_data,
154 : int epochs) {
155 0 : for (int epoch = 0; epoch < epochs; ++epoch) {
156 0 : for (const auto &input : training_data) {
157 : // forward pass
158 0 : std::vector<double> hidden(hidden_size);
159 0 : std::vector<double> output(output_size);
160 :
161 : // calculate hidden layer values
162 0 : for (int i = 0; i < hidden_size; ++i) {
163 0 : hidden[i] = 0;
164 0 : for (int j = 0; j < input_size; ++j) {
165 0 : hidden[i] += input[j] * weights_input_hidden[j][i];
166 : }
167 0 : hidden[i] = sigmoid({hidden[i]})[0];
168 : }
169 :
170 : // calculate output layer values
171 0 : for (int i = 0; i < output_size; ++i) {
172 0 : output[i] = 0;
173 0 : for (int j = 0; j < hidden_size; ++j) {
174 0 : output[i] += hidden[j] * weights_hidden_output[j][i];
175 : }
176 0 : output[i] = sigmoid({output[i]})[0];
177 : }
178 :
179 : // backward pass (gradient descent)
180 0 : for (int i = 0; i < output_size; ++i) {
181 0 : for (int j = 0; j < hidden_size; ++j) {
182 0 : weights_hidden_output[j][i] -=
183 0 : learning_rate * (output[i] - input[i]) * hidden[j];
184 : }
185 : }
186 :
187 0 : for (int i = 0; i < hidden_size; ++i) {
188 0 : for (int j = 0; j < input_size; ++j) {
189 0 : double error = 0;
190 0 : for (int k = 0; k < output_size; ++k) {
191 0 : error += (output[k] - input[k]) *
192 0 : weights_hidden_output[i][k];
193 : }
194 0 : weights_input_hidden[j][i] -= learning_rate * error *
195 0 : input[j] * (1 - hidden[i]) *
196 0 : hidden[i];
197 : }
198 : }
199 0 : }
200 : }
201 0 : }
202 :
203 0 : void gpmp::ml::AutoEncoder::display() {
204 0 : std::cout << "Input to Hidden Weights:\n";
205 0 : for (int i = 0; i < input_size; ++i) {
206 0 : for (int j = 0; j < hidden_size; ++j) {
207 0 : std::cout << weights_input_hidden[i][j] << " ";
208 : }
209 0 : std::cout << "\n";
210 : }
211 :
212 0 : std::cout << "\nHidden to Output Weights:\n";
213 0 : for (int i = 0; i < hidden_size; ++i) {
214 0 : for (int j = 0; j < output_size; ++j) {
215 0 : std::cout << weights_hidden_output[i][j] << " ";
216 : }
217 0 : std::cout << "\n";
218 : }
219 0 : }
220 :
221 0 : gpmp::ml::SparseAutoEncoder::SparseAutoEncoder(int in_size,
222 : int h_size,
223 : int out_size,
224 : double l_rate,
225 : double s_weight,
226 0 : double s_target)
227 0 : : AutoEncoder(in_size, h_size, out_size, l_rate), sparsity_weight(s_weight),
228 0 : sparsity_target(s_target) {
229 0 : }
230 :
231 0 : void gpmp::ml::SparseAutoEncoder::train(
232 : const std::vector<std::vector<double>> &training_data,
233 : int epochs) {
234 :
235 0 : const double SPARSITY_TARGET_DECAY = 0.1;
236 :
237 0 : for (int epoch = 0; epoch < epochs; ++epoch) {
238 0 : for (const auto ¤t_input : training_data) {
239 : // forward pass
240 0 : std::vector<double> hidden = forward(current_input);
241 :
242 : // backward pass (gradient descent)
243 0 : for (int i = 0; i < output_size; ++i) {
244 0 : for (int j = 0; j < hidden_size; ++j) {
245 0 : weights_hidden_output[j][i] -=
246 0 : learning_rate * (hidden[i] - current_input[i]) *
247 0 : hidden[j];
248 : }
249 : }
250 :
251 0 : for (int i = 0; i < hidden_size; ++i) {
252 0 : for (int j = 0; j < input_size; ++j) {
253 0 : double error = 0;
254 0 : for (int k = 0; k < output_size; ++k) {
255 0 : error += (hidden[k] - current_input[k]) *
256 0 : weights_hidden_output[i][k];
257 : }
258 :
259 : double sparsity_term =
260 0 : sparsity_weight * (sparsity_target - hidden[i]);
261 :
262 0 : weights_input_hidden[j][i] -=
263 0 : learning_rate * (error + sparsity_term) *
264 0 : current_input[j] * (1 - hidden[i]) * hidden[i];
265 : }
266 : }
267 :
268 0 : for (int i = 0; i < hidden_size; ++i) {
269 0 : double average_activation = 0.0;
270 0 : for (const auto &input : training_data) {
271 0 : std::vector<double> current_hidden = forward(input);
272 0 : average_activation += current_hidden[i];
273 0 : }
274 0 : average_activation /= training_data.size();
275 0 : sparsity_target =
276 0 : (1.0 - SPARSITY_TARGET_DECAY) * sparsity_target +
277 0 : SPARSITY_TARGET_DECAY * average_activation;
278 : }
279 0 : }
280 : }
281 0 : }
282 :
283 0 : gpmp::ml::DenoisingAutoEncoder::DenoisingAutoEncoder(int in_size,
284 : int h_size,
285 : int out_size,
286 : double l_rate,
287 0 : double c_level)
288 : : AutoEncoder(in_size, h_size, out_size, l_rate),
289 0 : corruption_level(c_level) {
290 0 : }
291 :
292 0 : void gpmp::ml::DenoisingAutoEncoder::train(
293 : const std::vector<std::vector<double>> &training_data,
294 : int epochs) {
295 0 : std::srand(std::time(0));
296 :
297 0 : for (int epoch = 0; epoch < epochs; ++epoch) {
298 0 : for (const auto &input : training_data) {
299 : // add noise to the input data
300 0 : std::vector<double> noisy_input = input;
301 0 : for (double &val : noisy_input) {
302 0 : if ((std::rand() / (RAND_MAX + 1.0)) < corruption_level) {
303 : // set to zero with probability corruption_level
304 0 : val = 0.0;
305 : }
306 : }
307 :
308 : // forward pass
309 0 : std::vector<double> hidden = forward(noisy_input);
310 :
311 : // backward pass (gradient descent)
312 0 : for (int i = 0; i < output_size; ++i) {
313 0 : for (int j = 0; j < hidden_size; ++j) {
314 0 : weights_hidden_output[j][i] -=
315 0 : learning_rate * (hidden[i] - input[i]) * hidden[j];
316 : }
317 : }
318 :
319 0 : for (int i = 0; i < hidden_size; ++i) {
320 0 : for (int j = 0; j < input_size; ++j) {
321 0 : double error = 0;
322 0 : for (int k = 0; k < output_size; ++k) {
323 0 : error += (hidden[k] - input[k]) *
324 0 : weights_hidden_output[i][k];
325 : }
326 0 : weights_input_hidden[j][i] -= learning_rate * error *
327 0 : noisy_input[j] *
328 0 : (1 - hidden[i]) * hidden[i];
329 : }
330 : }
331 0 : }
332 : }
333 0 : }
334 :
335 0 : gpmp::ml::ContractiveAutoEncoder::ContractiveAutoEncoder(int in_size,
336 : int h_size,
337 : int out_size,
338 : double l_rate,
339 0 : double c_weight)
340 : : AutoEncoder(in_size, h_size, out_size, l_rate),
341 0 : contractive_weight(c_weight) {
342 0 : }
343 :
344 0 : void gpmp::ml::ContractiveAutoEncoder::train(
345 : const std::vector<std::vector<double>> &training_data,
346 : int epochs) {
347 0 : for (int epoch = 0; epoch < epochs; ++epoch) {
348 0 : for (const auto &input : training_data) {
349 : // forward pass
350 0 : std::vector<double> hidden = forward(input);
351 :
352 : // backward pass (gradient descent)
353 0 : for (int i = 0; i < output_size; ++i) {
354 0 : for (int j = 0; j < hidden_size; ++j) {
355 0 : weights_hidden_output[j][i] -=
356 0 : learning_rate * (hidden[i] - input[i]) * hidden[j];
357 : }
358 : }
359 :
360 0 : for (int i = 0; i < hidden_size; ++i) {
361 0 : for (int j = 0; j < input_size; ++j) {
362 0 : double error = 0;
363 0 : for (int k = 0; k < output_size; ++k) {
364 0 : error += (hidden[k] - input[k]) *
365 0 : weights_hidden_output[i][k];
366 : }
367 : double contractive_term =
368 0 : contractive_weight * hidden[i] * (1 - hidden[i]);
369 0 : weights_input_hidden[j][i] -=
370 0 : learning_rate * (error + contractive_term) * input[j] *
371 0 : (1 - hidden[i]) * hidden[i];
372 : }
373 : }
374 0 : }
375 : }
376 0 : }
377 :
378 0 : gpmp::ml::MDLAutoEncoder::MDLAutoEncoder(int in_size,
379 : int h_size,
380 : int out_size,
381 : double l_rate,
382 0 : double m_wt)
383 0 : : AutoEncoder(in_size, h_size, out_size, l_rate), mdl_weight(m_wt) {
384 0 : }
385 :
386 0 : void gpmp::ml::MDLAutoEncoder::train(
387 : const std::vector<std::vector<double>> &training_data,
388 : int epochs) {
389 0 : for (int epoch = 0; epoch < epochs; ++epoch) {
390 0 : for (const auto &input : training_data) {
391 : // forward pass
392 0 : std::vector<double> hidden = forward(input);
393 :
394 : // backward pass (gradient descent)
395 0 : for (int i = 0; i < output_size; ++i) {
396 0 : for (int j = 0; j < hidden_size; ++j) {
397 0 : weights_hidden_output[j][i] -=
398 0 : learning_rate * (hidden[i] - input[i]) * hidden[j];
399 : }
400 : }
401 :
402 0 : for (int i = 0; i < hidden_size; ++i) {
403 0 : for (int j = 0; j < input_size; ++j) {
404 0 : double error = 0;
405 0 : for (int k = 0; k < output_size; ++k) {
406 0 : error += (hidden[k] - input[k]) *
407 0 : weights_hidden_output[i][k];
408 : }
409 0 : double mdl_term = mdl_weight * log(1.0 + fabs(error));
410 0 : weights_input_hidden[j][i] -=
411 0 : learning_rate * (error + mdl_term) * input[j] *
412 0 : (1 - hidden[i]) * hidden[i];
413 : }
414 : }
415 0 : }
416 : }
417 0 : }
418 :
419 0 : gpmp::ml::ConcreteAutoEncoder::ConcreteAutoEncoder(int in_size,
420 : int h_size,
421 : int out_size,
422 : double l_rate,
423 0 : double temp)
424 0 : : AutoEncoder(in_size, h_size, out_size, l_rate), temperature(temp) {
425 0 : }
426 :
427 0 : void gpmp::ml::ConcreteAutoEncoder::train(
428 : const std::vector<std::vector<double>> &training_data,
429 : int epochs) {
430 0 : std::default_random_engine generator;
431 0 : std::uniform_real_distribution<double> uniform_distribution(0.0, 1.0);
432 :
433 0 : for (int epoch = 0; epoch < epochs; ++epoch) {
434 0 : for (const auto &input : training_data) {
435 : // forward pass with Concrete distribution
436 0 : std::vector<double> hidden;
437 0 : for (int i = 0; i < hidden_size; ++i) {
438 0 : double u = uniform_distribution(generator);
439 : // Gumbel noise
440 0 : double g = -log(-log(u));
441 0 : double s = (input[i] + g) / temperature;
442 0 : double p = 1.0 / (1.0 + exp(-s));
443 0 : hidden.push_back(p);
444 : }
445 :
446 : // backward pass (gradient descent)
447 0 : for (int i = 0; i < output_size; ++i) {
448 0 : for (int j = 0; j < hidden_size; ++j) {
449 0 : weights_hidden_output[j][i] -=
450 0 : learning_rate * (hidden[i] - input[i]) * hidden[j];
451 : }
452 : }
453 :
454 0 : for (int i = 0; i < hidden_size; ++i) {
455 0 : for (int j = 0; j < input_size; ++j) {
456 0 : double error = 0;
457 0 : for (int k = 0; k < output_size; ++k) {
458 0 : error += (hidden[k] - input[k]) *
459 0 : weights_hidden_output[i][k];
460 : }
461 0 : weights_input_hidden[j][i] -= learning_rate * error *
462 0 : input[j] * (1 - hidden[i]) *
463 0 : hidden[i];
464 : }
465 : }
466 0 : }
467 : }
468 0 : }
469 :
470 0 : gpmp::ml::VariationalAutoEncoder::VariationalAutoEncoder(int in_size,
471 : int h_size,
472 : int out_size,
473 0 : double l_rate)
474 0 : : AutoEncoder(in_size, h_size, out_size, l_rate) {
475 0 : }
476 :
477 0 : double gpmp::ml::VariationalAutoEncoder::sample_dist() {
478 0 : static std::default_random_engine generator;
479 0 : static std::normal_distribution<double> distribution(0.0, 1.0);
480 0 : return distribution(generator);
481 : }
482 :
483 0 : double gpmp::ml::VariationalAutoEncoder::reparameterize(double mean,
484 : double log_variance) {
485 0 : double standard_normal_sample = sample_dist();
486 0 : return mean + exp(0.5 * log_variance) * standard_normal_sample;
487 : }
488 :
489 : std::vector<double>
490 0 : gpmp::ml::VariationalAutoEncoder::encoder(const std::vector<double> &input) {
491 0 : std::vector<double> hidden(hidden_size);
492 :
493 0 : for (int i = 0; i < hidden_size; ++i) {
494 0 : hidden[i] = 0;
495 0 : for (int j = 0; j < input_size; ++j) {
496 0 : hidden[i] += input[j] * weights_input_hidden[j][i];
497 : }
498 0 : hidden[i] = sigmoid({hidden[i]})[0];
499 : }
500 :
501 0 : return hidden;
502 0 : }
503 :
504 0 : std::vector<double> gpmp::ml::VariationalAutoEncoder::decoder(
505 : const std::vector<double> &hidden_sampled) {
506 0 : std::vector<double> output(output_size);
507 :
508 0 : for (int i = 0; i < output_size; ++i) {
509 0 : output[i] = 0;
510 0 : for (int j = 0; j < hidden_size; ++j) {
511 0 : output[i] += hidden_sampled[j] * weights_hidden_output[j][i];
512 : }
513 0 : output[i] = sigmoid({output[i]})[0];
514 : }
515 :
516 0 : return output;
517 0 : }
518 :
519 0 : void gpmp::ml::VariationalAutoEncoder::gradient_descent(
520 : const std::vector<double> &input,
521 : const std::vector<double> &output,
522 : const std::vector<double> &hidden_sampled) {
523 0 : for (int i = 0; i < output_size; ++i) {
524 0 : for (int j = 0; j < hidden_size; ++j) {
525 0 : weights_hidden_output[j][i] -=
526 0 : learning_rate * (output[i] - input[i]) * hidden_sampled[j];
527 : }
528 : }
529 :
530 0 : for (int i = 0; i < hidden_size; ++i) {
531 0 : for (int j = 0; j < input_size; ++j) {
532 0 : double error = 0;
533 0 : for (int k = 0; k < output_size; ++k) {
534 0 : error += (output[k] - input[k]) * weights_hidden_output[i][k];
535 : }
536 : double hidden_gradient =
537 0 : hidden_sampled[i] * (1 - hidden_sampled[i]);
538 : // derivative of softplus
539 : double log_variance_gradient =
540 0 : 1.0 / (1.0 + exp(-hidden_log_variance[i]));
541 0 : weights_input_hidden[j][i] -=
542 0 : learning_rate *
543 0 : (error * hidden_gradient +
544 0 : (hidden_sampled[i] - hidden_mean[i]) * hidden_gradient +
545 0 : (hidden_log_variance[i] - log_variance_gradient) *
546 0 : hidden_gradient) *
547 0 : input[j];
548 : }
549 : }
550 0 : }
551 :
552 0 : void gpmp::ml::VariationalAutoEncoder::train(
553 : const std::vector<std::vector<double>> &training_data,
554 : int epochs) {
555 0 : std::default_random_engine generator;
556 0 : std::normal_distribution<double> normal_distribution(0.0, 1.0);
557 0 : std::vector<double> hidden_sampled;
558 :
559 0 : for (int epoch = 0; epoch < epochs; ++epoch) {
560 0 : for (const auto &input : training_data) {
561 : // forward pass (encoder)
562 0 : hidden_mean = encoder(input);
563 0 : hidden_log_variance = encoder(input);
564 :
565 0 : for (int i = 0; i < hidden_size; ++i) {
566 0 : hidden_sampled.push_back(
567 0 : reparameterize(hidden_mean[i], hidden_log_variance[i]));
568 : }
569 :
570 : // forward pass (decoder)
571 0 : std::vector<double> output = decoder(hidden_sampled);
572 :
573 : // backward pass (gradient descent)
574 0 : gradient_descent(input, output, hidden_sampled);
575 0 : }
576 : }
577 0 : }
578 :
579 0 : gpmp::ml::RecurrentAutoEncoder::RecurrentAutoEncoder(int in_size,
580 : int h_size,
581 : int out_size,
582 0 : double l_rate)
583 : : AutoEncoder(in_size, h_size, out_size, l_rate),
584 0 : weights_recurrent(h_size, std::vector<double>(h_size, 0.0)) {
585 0 : }
586 :
587 0 : void gpmp::ml::RecurrentAutoEncoder::train(
588 : const std::vector<std::vector<double>> &training_data,
589 : int epochs) {
590 0 : for (int epoch = 0; epoch < epochs; ++epoch) {
591 0 : std::vector<double> previous_hidden(hidden_size, 0.0);
592 :
593 0 : for (const auto &input : training_data) {
594 : // forward pass
595 0 : std::vector<double> hidden = recurr_fwd(input, previous_hidden);
596 0 : std::vector<double> output = forward(hidden);
597 :
598 : // backward pass (gradient descent)
599 0 : for (int i = 0; i < output_size; ++i) {
600 0 : for (int j = 0; j < hidden_size; ++j) {
601 0 : weights_hidden_output[j][i] -=
602 0 : learning_rate * (output[i] - input[i]) * hidden[j];
603 : }
604 : }
605 :
606 0 : for (int i = 0; i < hidden_size; ++i) {
607 0 : for (int j = 0; j < input_size; ++j) {
608 0 : double error = 0;
609 0 : for (int k = 0; k < output_size; ++k) {
610 0 : error += (output[k] - input[k]) *
611 0 : weights_hidden_output[i][k];
612 : }
613 0 : weights_input_hidden[j][i] -= learning_rate * error *
614 0 : input[j] * (1 - hidden[i]) *
615 0 : hidden[i];
616 : }
617 : }
618 :
619 : // recurrent weights update
620 0 : for (int i = 0; i < hidden_size; ++i) {
621 0 : for (int j = 0; j < hidden_size; ++j) {
622 0 : weights_recurrent[j][i] -=
623 0 : learning_rate * (hidden[i] - previous_hidden[i]) *
624 0 : hidden[j];
625 : }
626 : }
627 :
628 0 : previous_hidden = hidden;
629 0 : }
630 0 : }
631 0 : }
632 :
633 0 : std::vector<double> gpmp::ml::RecurrentAutoEncoder::recurr_fwd(
634 : const std::vector<double> &input,
635 : const std::vector<double> &previous_hidden) {
636 0 : std::vector<double> recurrent_input(hidden_size, 0.0);
637 :
638 : // sum the weighted contributions from the current input and the previous
639 : // hidden state
640 0 : for (int i = 0; i < hidden_size; ++i) {
641 0 : recurrent_input[i] = 0.0;
642 0 : for (int j = 0; j < input_size; ++j) {
643 0 : recurrent_input[i] += weights_input_hidden[j][i] * input[j];
644 : }
645 0 : for (int j = 0; j < hidden_size; ++j) {
646 0 : recurrent_input[i] += weights_recurrent[j][i] * previous_hidden[j];
647 : }
648 : // activation function
649 : // recurrent_input[i] = 1.0 / (1.0 + std::exp(-recurrent_input[i]));
650 0 : recurrent_input[i] = sigmoid({recurrent_input[i]})[0];
651 : }
652 :
653 0 : return recurrent_input;
654 0 : }
655 :
656 0 : gpmp::ml::FullAutoEncoder::FullAutoEncoder(int in_size,
657 : int h_size,
658 : int out_size,
659 0 : double l_rate)
660 0 : : AutoEncoder(in_size, h_size, out_size, l_rate) {
661 0 : }
662 :
663 0 : void gpmp::ml::FullAutoEncoder::train(
664 : const std::vector<std::vector<double>> &training_data,
665 : int epochs) {
666 0 : for (int epoch = 0; epoch < epochs; ++epoch) {
667 0 : for (const auto &input : training_data) {
668 : // forward pass
669 0 : std::vector<double> hidden = forward(input);
670 0 : std::vector<double> output = forward(hidden);
671 :
672 : // backward pass (gradient descent)
673 0 : for (int i = 0; i < output_size; ++i) {
674 0 : for (int j = 0; j < hidden_size; ++j) {
675 0 : weights_hidden_output[j][i] -=
676 0 : learning_rate * (output[i] - input[i]) * hidden[j];
677 : }
678 : }
679 :
680 0 : for (int i = 0; i < hidden_size; ++i) {
681 0 : for (int j = 0; j < input_size; ++j) {
682 0 : double error = 0;
683 0 : for (int k = 0; k < output_size; ++k) {
684 0 : error += (output[k] - input[k]) *
685 0 : weights_hidden_output[i][k];
686 : }
687 0 : weights_input_hidden[j][i] -= learning_rate * error *
688 0 : input[j] * (1 - hidden[i]) *
689 0 : hidden[i];
690 : }
691 : }
692 0 : }
693 : }
694 0 : }
|
