GPUMLib  0.2.2
GPU Machine Learning Library
HostRBM.h
1 /*
2  Noel Lopes is an Assistant Professor at the Polytechnic Institute of Guarda, Portugal
3  Copyright (C) 2009, 2010, 2011 Noel de Jesus Mendonša Lopes
4 
5  This file is part of GPUMLib.
6 
7  GPUMLib is free software: you can redistribute it and/or modify
8  it under the terms of the GNU General Public License as published by
9  the Free Software Foundation, either version 3 of the License, or
10  (at your option) any later version.
11 
12  This program is distributed in the hope that it will be useful,
13  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  GNU General Public License for more details.
16 
17  You should have received a copy of the GNU General Public License
18  along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20 
21 #ifndef RBMH_h
22 #define RBMH_h
23 
24 #include <stdlib.h>
25 #include <math.h>
26 #include <float.h>
27 #include <limits>
28 
29 #include "RBMconfig.h"
30 
31 #include "../memory/HostArray.h"
32 
33 namespace GPUMLib {
34 
37 
39 class RBMhost {
40  friend class DBNhost;
41 
42  private:
43  int I; // number of visible units
44  int J; // number of hidden units
45  int samples;
46 
47  HostArray<cudafloat> a; // bias of the visible layer
48  HostArray<cudafloat> b; // bias of the hidden layer
49  HostMatrix<cudafloat> w; // weights
50 
51  HostArray<cudafloat> lastDeltaA;
52  HostArray<cudafloat> lastDeltaB;
53  HostMatrix<cudafloat> lastDeltaW;
54 
55  HostMatrix<cudafloat> * v; // visible units / inputs
56 
57  HostArray<cudafloat> learningRateA;
58  HostArray<cudafloat> learningRateB;
59  HostMatrix<cudafloat> learningRateW;
60 
61  HostArray<cudafloat> lastDeltaWithoutLearningMomentumA;
62  HostArray<cudafloat> lastDeltaWithoutLearningMomentumB;
63  HostMatrix<cudafloat> lastDeltaWithoutLearningMomentumW;
64 
65  cudafloat stdWeights;
66  cudafloat initialLearningRate;
67  cudafloat momentum;
68 
69  bool useBinaryValuesVisibleReconstruction;
70 
71  HostMatrix<cudafloat> h_data;
72  HostMatrix<cudafloat> h_recon;
73  HostMatrix<cudafloat> v_recon;
74 
75  int epoch;
76 
77  // Used by DBNs
78  RBMhost(RBMhost * previousLayer, int hiddenUnits, cudafloat initialLearningRate, cudafloat momentum, bool useBinaryValuesVisibleReconstruction, cudafloat stdWeights) {
79  v = &previousLayer->h_data;
80 
81  this->stdWeights = stdWeights;
82  this->initialLearningRate = initialLearningRate;
83  this->useBinaryValuesVisibleReconstruction = useBinaryValuesVisibleReconstruction;
84 
85  I = previousLayer->J;
86  J = hiddenUnits;
87 
88  w.ResizeWithoutPreservingData(hiddenUnits, I);
90  b.ResizeWithoutPreservingData(hiddenUnits);
91 
92  learningRateW.ResizeWithoutPreservingData(hiddenUnits, I);
93  learningRateA.ResizeWithoutPreservingData(I);
94  learningRateB.ResizeWithoutPreservingData(hiddenUnits);
95 
96  lastDeltaWithoutLearningMomentumW.ResizeWithoutPreservingData(hiddenUnits, I);
97  lastDeltaWithoutLearningMomentumA.ResizeWithoutPreservingData(I);
98  lastDeltaWithoutLearningMomentumB.ResizeWithoutPreservingData(hiddenUnits);
99 
100  lastDeltaW.ResizeWithoutPreservingData(hiddenUnits, I);
101  lastDeltaA.ResizeWithoutPreservingData(I);
102  lastDeltaB.ResizeWithoutPreservingData(hiddenUnits);
103 
104  samples = v->Rows();
105 
106  h_data.ResizeWithoutPreservingData(samples, hiddenUnits);
107  h_recon.ResizeWithoutPreservingData(samples, hiddenUnits);
108  v_recon.ResizeWithoutPreservingData(samples, I);
109 
110  this->momentum = momentum;
111 
112  epoch = 0;
113  }
114 
115  public:
124  RBMhost(int visibleUnits, int hiddenUnits, HostMatrix<cudafloat> & inputs, cudafloat initialLearningRate, cudafloat momentum = DEFAULT_MOMENTUM, bool useBinaryValuesVisibleReconstruction = false, cudafloat stdWeights = STD_WEIGHTS) {
125  assert(visibleUnits == inputs.Columns());
126 
127  v = &inputs;
128 
129  this->useBinaryValuesVisibleReconstruction = useBinaryValuesVisibleReconstruction;
130 
131  I = visibleUnits;
132  J = hiddenUnits;
133 
134  w.ResizeWithoutPreservingData(hiddenUnits, visibleUnits);
135  a.ResizeWithoutPreservingData(visibleUnits);
136  b.ResizeWithoutPreservingData(hiddenUnits);
137 
138  learningRateW.ResizeWithoutPreservingData(hiddenUnits, visibleUnits);
139  learningRateA.ResizeWithoutPreservingData(visibleUnits);
140  learningRateB.ResizeWithoutPreservingData(hiddenUnits);
141 
142  lastDeltaWithoutLearningMomentumW.ResizeWithoutPreservingData(hiddenUnits, visibleUnits);
143  lastDeltaWithoutLearningMomentumA.ResizeWithoutPreservingData(visibleUnits);
144  lastDeltaWithoutLearningMomentumB.ResizeWithoutPreservingData(hiddenUnits);
145 
146  lastDeltaW.ResizeWithoutPreservingData(hiddenUnits, visibleUnits);
147  lastDeltaA.ResizeWithoutPreservingData(visibleUnits);
148  lastDeltaB.ResizeWithoutPreservingData(hiddenUnits);
149 
150  samples = inputs.Rows();
151 
152  h_data.ResizeWithoutPreservingData(samples, hiddenUnits);
153  h_recon.ResizeWithoutPreservingData(samples, hiddenUnits);
154  v_recon.ResizeWithoutPreservingData(samples, visibleUnits);
155 
156  RandomizeWeights(stdWeights, initialLearningRate);
157  this->momentum = momentum;
158  }
159 
162  void RandomizeWeights(cudafloat stdWeights) {
163  this->stdWeights = stdWeights;
165  }
166 
170  void RandomizeWeights(cudafloat stdWeights, cudafloat initialLearningRate) {
171  this->initialLearningRate = initialLearningRate;
172  RandomizeWeights(stdWeights);
173  }
174 
177  int nWeights = w.Elements();
178 
179  cudafloat * weights = w.Pointer();
180  cudafloat * lastDelta = lastDeltaW.Pointer();
181 
182  cudafloat * lastDeltaWM = lastDeltaWithoutLearningMomentumW.Pointer();
183  cudafloat * stepSizes = learningRateW.Pointer();
184 
185  for (int w = 0; w < nWeights; w++) {
186  weights[w] = CUDA_VALUE(2.0) * stdWeights * ((cudafloat) rand() / RAND_MAX) - stdWeights;
187  lastDelta[w] = CUDA_VALUE(0.0);
188  lastDeltaWM[w] = CUDA_VALUE(0.0);
189  stepSizes[w] = initialLearningRate;
190  }
191 
192  for (int j = 0; j < J; j++) {
193  b[j] = INITIAL_BIAS_HIDDEN_UNITS;
194  lastDeltaB[j] = CUDA_VALUE(0.0);
195  lastDeltaWithoutLearningMomentumB[j] = CUDA_VALUE(0.0);
196  learningRateB[j] = initialLearningRate;
197  }
198 
199  for (int i = 0; i < I; i++) {
200  a[i] = CUDA_VALUE(0.0);
201  lastDeltaA[i] = CUDA_VALUE(0.0);
202  lastDeltaWithoutLearningMomentumA[i] = CUDA_VALUE(0.0);
203  learningRateA[i] = initialLearningRate;
204  }
205 
206  for(int s = 0; s < samples; s++) {
207  for (int i = 0; i < I; i++) {
208  if ((*v)(s, i) > CUDA_VALUE(0.0)) a[i]++;
209  }
210  }
211 
212  for (int i = 0; i < I; i++) {
213  cudafloat pi = a[i] / samples;
214  a[i] = log(pi / (CUDA_VALUE(1.0) - pi));
215  if (IsInfOrNaN(a[i])) a[i] = CUDA_VALUE(0.0);
216  }
217 
218  epoch = 0;
219  }
220 
221  void ContrastiveDivergence(int n) {
222  ComputeStatusHiddenUnits(*v, h_data);
223  ComputeStatusVisibleUnits(h_data, v_recon);
224 
225  for (int k = 1; k < n; k++) {
226  ComputeStatusHiddenUnits(v_recon, h_recon);
227  ComputeStatusVisibleUnits(h_recon, v_recon);
228  }
229 
230  ComputeStatusHiddenUnits(v_recon, h_recon, false);
231 
232  for (int j = 0; j < J; j++) {
233  for(int i = 0; i < I; i++) {
234  cudafloat delta = CUDA_VALUE(0.0);
235 
236  for (int s = 0; s < v->Rows(); s++) {
237  delta += (*v)(s, i) * h_data(s, j) - v_recon(s, i) * h_recon(s, j);
238  }
239 
240  delta /= samples;
241 
242  cudafloat learnRate = learningRateW(j, i);
243  cudafloat factor = SAME_DIRECTION(lastDeltaWithoutLearningMomentumW(j, i), delta) ? U_FACTOR : D_FACTOR;
244  learnRate *= factor;
245  if (learnRate > MAX_STEP_SIZE) learnRate = MAX_STEP_SIZE;
246  learningRateW(j, i) = learnRate;
247 
248  lastDeltaWithoutLearningMomentumW(j, i) = delta;
249 
250  cudafloat momentum = this->momentum * learnRate;
251  if (momentum < CUDA_VALUE(0.1)) momentum = CUDA_VALUE(0.1);
252  if (momentum > CUDA_VALUE(0.9)) momentum = CUDA_VALUE(0.9);
253 
254  delta += momentum * lastDeltaW(j, i);
255  lastDeltaW(j, i) = delta;
256 
257  cudafloat neww = w(j, i) + (learnRate * delta);
258  if (IsInfOrNaN(neww)) {
259  lastDeltaW(j, i) = CUDA_VALUE(0.0);
260  lastDeltaWithoutLearningMomentumW(j, i) = CUDA_VALUE(0.0);
261  } else {
262  w(j, i) = neww;
263  }
264  }
265 
266  cudafloat deltaB = CUDA_VALUE(0.0);
267  for (int s = 0; s < v->Rows(); s++) {
268  deltaB += h_data(s, j) - h_recon(s, j);
269  }
270 
271  deltaB /= samples;
272 
273  cudafloat learnRate = learningRateB[j];
274  cudafloat factor = SAME_DIRECTION(lastDeltaWithoutLearningMomentumB[j], deltaB) ? U_FACTOR : D_FACTOR;
275  learnRate *= factor;
276  if (learnRate > MAX_STEP_SIZE) learnRate = MAX_STEP_SIZE;
277  learningRateB[j] = learnRate;
278 
279  lastDeltaWithoutLearningMomentumB[j] = deltaB;
280 
281  cudafloat momentum = this->momentum * learnRate;
282  if (momentum < CUDA_VALUE(0.1)) momentum = CUDA_VALUE(0.1);
283  if (momentum > CUDA_VALUE(0.9)) momentum = CUDA_VALUE(0.9);
284 
285  deltaB += momentum * lastDeltaB[j];
286  lastDeltaB[j] = deltaB;
287 
288  cudafloat newb = b[j] + (learnRate * deltaB);
289  if (IsInfOrNaN(newb)) {
290  lastDeltaB[j] = CUDA_VALUE(0.0);
291  lastDeltaWithoutLearningMomentumB[j] = CUDA_VALUE(0.0);
292  } else {
293  b[j] = newb;
294  }
295  }
296 
297  for(int i = 0; i < I; i++) {
298  cudafloat deltaA = CUDA_VALUE(0.0);
299 
300  for (int s = 0; s < v->Rows(); s++) {
301  deltaA += (*v)(s, i) - v_recon(s, i);
302  }
303 
304  deltaA /= samples;
305 
306  cudafloat learnRate = learningRateA[i];
307  cudafloat factor = SAME_DIRECTION(lastDeltaWithoutLearningMomentumA[i], deltaA) ? U_FACTOR : D_FACTOR;
308  learnRate *= factor;
309  if (learnRate > MAX_STEP_SIZE) learnRate = MAX_STEP_SIZE;
310  learningRateA[i] = learnRate;
311 
312  lastDeltaWithoutLearningMomentumA[i] = deltaA;
313 
314  cudafloat momentum = this->momentum * learnRate;
315  if (momentum < CUDA_VALUE(0.1)) momentum = CUDA_VALUE(0.1);
316  if (momentum > CUDA_VALUE(0.9)) momentum = CUDA_VALUE(0.9);
317 
318  deltaA += momentum * lastDeltaA[i];
319  lastDeltaA[i] = deltaA;
320 
321  cudafloat newa = a[i] + (learnRate * deltaA);
322  if (IsInfOrNaN(newa)) {
323  lastDeltaA[i] = CUDA_VALUE(0.0);
324  lastDeltaWithoutLearningMomentumA[i] = CUDA_VALUE(0.0);
325  } else {
326  a[i] = newa;
327  }
328  }
329 
330  epoch++;
331  }
332 
336  assert(epoch > 0);
337 
338  cudafloat error;
339 
340  error = CUDA_VALUE(0.0);
341  for (int s = 0; s < (*v).Rows(); s++) {
342  for(int i = 0; i < I; i++) {
343  cudafloat e = (*v)(s, i) - v_recon(s, i);
344  error += e * e;
345  }
346  }
347 
348  return error / ((cudafloat) samples * I);
349  }
350 
353  int Epoch() const {
354  return epoch;
355  }
356 
357  private:
358  static cudafloat Binarize(cudafloat probability) {
359  return (probability > ((cudafloat) rand() / RAND_MAX)) ? CUDA_VALUE(1.0) : CUDA_VALUE(0.0);
360  }
361 
362  void ComputeStatusHiddenUnits(HostMatrix<cudafloat> & v, HostMatrix<cudafloat> & h, bool binary = true) {
363  for (int s = 0; s < samples; s++) {
364  for (int j = 0; j < J; j++) {
365  cudafloat sum = b[j];
366  for(int i = 0; i < I; i++) sum += v(s, i) * w(j, i);
367 
368  cudafloat result = CUDA_SIGMOID(sum);
369  if (binary) result = Binarize(result);
370  h(s, j) = result;
371  }
372  }
373  }
374 
375  void ComputeStatusVisibleUnits(HostMatrix<cudafloat> & h, HostMatrix<cudafloat> & v) {
376  for (int s = 0; s < samples; s++) {
377  for(int i = 0; i < I; i++) {
378  cudafloat sum = a[i];
379  for (int j = 0; j < J; j++) sum += h(s, j) * w(j, i);
380 
381  cudafloat result = CUDA_SIGMOID(sum);
382  if (useBinaryValuesVisibleReconstruction) result = Binarize(result);
383  v(s, i) = result;
384  }
385  }
386  }
387 
388  public:
392  return HostMatrix<cudafloat>(w);
393  }
394 
398  return HostArray<cudafloat>(a);
399  }
400 
404  return HostArray<cudafloat>(b);
405  }
406 
407  HostMatrix<cudafloat> GetOutputs() {
408  ComputeStatusHiddenUnits(*v, h_data);
409  return HostMatrix<cudafloat>(h_data);
410  }
411 };
412 
415 
417 
418 }
419 
420 #endif
Represents a Deep Belief Network (Host - CPU).
Definition: HostDBN.h:32
#define SAME_DIRECTION(X, Y)
Verifies if X and Y have the same signal.
void RandomizeWeights(cudafloat stdWeights, cudafloat initialLearningRate)
Definition: HostRBM.h:170
int Elements() const
Definition: BaseMatrix.h:94
int ResizeWithoutPreservingData(int size)
Definition: BaseArray.h:77
void RandomizeWeights(cudafloat stdWeights)
Definition: HostRBM.h:162
void RandomizeWeights()
Randomizes the weights of the RBM.
Definition: HostRBM.h:176
int Epoch() const
Definition: HostRBM.h:353
Type * Pointer() const
Definition: BaseMatrix.h:88
RBMhost(int visibleUnits, int hiddenUnits, HostMatrix< cudafloat > &inputs, cudafloat initialLearningRate, cudafloat momentum=DEFAULT_MOMENTUM, bool useBinaryValuesVisibleReconstruction=false, cudafloat stdWeights=STD_WEIGHTS)
Definition: HostRBM.h:124
int Columns() const
Definition: BaseMatrix.h:80
cudafloat MeanSquareError() const
Definition: HostRBM.h:335
int ResizeWithoutPreservingData(int rows, int columns)
Definition: BaseMatrix.h:119
HostMatrix< cudafloat > GetWeights()
Definition: HostRBM.h:391
#define CUDA_SIGMOID(X)
int Rows() const
Definition: BaseMatrix.h:74
#define CUDA_VALUE(X)
HostArray< cudafloat > GetHiddenBias()
Definition: HostRBM.h:403
HostArray< cudafloat > GetVisibleBias()
Definition: HostRBM.h:397
float cudafloat
Represents a Restricted Boltzman Machine (Host - CPU).
Definition: HostRBM.h:39