GPUMLib  0.2.2
GPU Machine Learning Library
ATS.cpp

Example Autonomous Training System (uses the Back-Propagation and Multiple Back-Propagation algorithms).

/*
Noel Lopes is an Assistant Professor at the Polytechnic Institute of Guarda, Portugal
Copyright (C) 2009, 2010, 2011, 2012, 2013 Noel de Jesus Mendonša Lopes
This file is part of GPUMLib.
GPUMLib is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define RMS_STOP CUDA_VALUE(0.01)
#include "../common/CudaInit.h"
#include "../common/OutputFile.h"
#include "../../MBP/BackPropagation.h"
#include "../../MBP/MultipleBackPropagation.h"
#include "Configuration.h"
#include "../common/ConfusionMatrix.h"
void SaveSelectiveInputWeights(OutputFile & f, HostArray<cudafloat> & weights, HostArray<cudafloat> & bias) {
int numWeights = weights.Length();
for(int w = 0; w < numWeights; w++) {
if (weights[w] != CUDA_VALUE(0.0) || bias[w] != CUDA_VALUE(0.0)) {
f.WriteLine(bias[w]);
f.WriteLine("0.0"); //delta
f.WriteLine("0.0"); //deltaWithoutLearningMomentum
f.WriteLine(INITIAL_LEARNING_RATE);
f.WriteLine(weights[w]);
f.WriteLine("0.0"); //delta
f.WriteLine("0.0"); //deltaWithoutLearningMomentum
f.WriteLine(INITIAL_LEARNING_RATE);
}
}
}
void SaveWeights(OutputFile & f, HostArray<cudafloat> & weights) {
int numWeights = weights.Length();
for(int w = 0; w < numWeights; w++) {
f.WriteLine(weights[w]);
f.WriteLine("0.0"); //delta
f.WriteLine("0.0"); //deltaWithoutLearningMomentum
f.WriteLine(INITIAL_LEARNING_RATE);
}
}
void SaveNetwork(BackPropagation * network, Configuration & cfg) {
ostringstream sstream;
sstream << cfg.RandomGenerator() << ".bpn";
OutputFile f(sstream.str().c_str());
const char * MBPVersion = "Multiple Back-Propagation Version 2.2.1";
f.WriteLine(MBPVersion);
f.WriteLine("Multiple Back-Propagation can be freely obtained at http://dit.ipg.pt/MBP - This file was generated by the ATS.");
f.WriteLine(cfg.TrainDataFile().c_str()); // train file
f.WriteLine(cfg.TestDataFile().c_str()); // test file
f.WriteLine("32"); // priority normal
f.WriteLine("0"); // update screen
f.WriteLine("1"); // delta bar delta
f.WriteLine(network->GetUpStepSizeFactor());
f.WriteLine(network->GetDownStepSizeFactor());
f.WriteLine(network->GetMaxStepSize());
f.WriteLine(network->GetRobustLearning());
f.WriteLine(network->GetRobustFactor());
f.WriteLine(1.0 + network->GetMaxPercentageRMSGrow()); // rmsGrowToApplyRobustLearning
f.WriteLine("0.0"); // weightDecay
f.WriteLine("0"); // autoUpdateLearning
f.WriteLine("0"); // autoUpdateMomentum
f.WriteLine("0.01"); //percentIncDecLearnRate
f.WriteLine("0.01"); //percentIncDecMomentum
f.WriteLine("0.01"); //percentIncDecSpaceLearnRate
f.WriteLine("0.01"); //percentIncDecSpaceMomentum
f.WriteLine(INITIAL_LEARNING_RATE); //mainNetLearningMomentumInformation.learningRate.value
f.WriteLine("1000"); //mainNetLearningMomentumInformation.learningRate.decayEpochs
f.WriteLine("1"); //mainNetLearningMomentumInformation.learningRate.decayPercentage
f.WriteLine(network->GetMomentum()); //mainNetLearningMomentumInformation.momentum.value
f.WriteLine("0"); //mainNetLearningMomentumInformation.momentum.decayEpochs
f.WriteLine("0"); //mainNetLearningMomentumInformation.momentum.decayPercentage
f.WriteLine(INITIAL_LEARNING_RATE); //spaceNetLearningMomentumInformation.learningRate.value
f.WriteLine("1000"); //spaceNetLearningMomentumInformation.learningRate.decayEpochs
f.WriteLine("1"); //spaceNetLearningMomentumInformation.learningRate.decayPercentage
f.WriteLine(network->GetMomentum()); //spaceNetLearningMomentumInformation.momentum.value
f.WriteLine("0"); //spaceNetLearningMomentumInformation.momentum.decayEpochs
f.WriteLine("0"); //spaceNetLearningMomentumInformation.momentum.decayPercentage
f.WriteLine("0"); //epochsStop
f.WriteLine(cfg.RmsStop());
f.WriteLine((int)cfg.MaximumEpochs()); // numberEpochsToStop
f.WriteLine("0.0"); //spaceRmsStop
f.WriteLine("1"); //batchTraining
f.WriteLine("0"); //randomizePatterns
f.WriteLine(cfg.MBP() ? 3 : 0); //Network Type
//main network
f.Write(network->GetNumberInputs());
int numLayers = network->GetNumberLayers();
for(int l = 0; l < numLayers; l++) {
f.Write("-");
f.Write(network->GetNumberNeurons(l));
}
f.WriteLine();
//space network additional layers
f.WriteLine("");
// layers information
for(int l = 0; l < numLayers; l++) {
int numNeurons = network->GetNumberNeurons(l);
f.WriteLine((l == 0 && cfg.MBP()) ? numNeurons : 0); // NeuronsWithSelectiveActivation
for(int n = 0; n < numNeurons; n++) {
f.WriteLine("0"); // ActivationFunction
f.WriteLine("1.0"); //ActivationFunctionParameter
}
}
// space layers information
if (cfg.MBP()) {
int numNeurons = network->GetNumberNeurons(0);
for(int n = 0; n < numNeurons; n++) {
f.WriteLine("0"); // ActivationFunction
f.WriteLine("1.0"); //ActivationFunctionParameter
}
}
f.WriteLine("0"); //ConnectInputLayerWithOutputLayer main
f.WriteLine("0"); //ConnectInputLayerWithOutputLayer space
if (network->HasSelectiveInputs()) {
HostArray<cudafloat> weights = network->GetSelectiveInputWeights();
HostArray<cudafloat> bias = network->GetSelectiveInputBias();
SaveSelectiveInputWeights(f, weights, bias);
}
for(int l = 0; l < numLayers; l++) {
HostArray<cudafloat> weights = network->GetLayerWeights(l);
SaveWeights(f, weights);
}
if (cfg.MBP()) {
MultipleBackPropagation * mbpnet = static_cast<MultipleBackPropagation *>(network);
if (mbpnet->HasSelectiveInputs()) {
HostArray<cudafloat> weights = mbpnet->GetSelectiveInputWeightsSpaceNetwork();
HostArray<cudafloat> bias = mbpnet->GetSelectiveInputBiasSpaceNetwork();
SaveSelectiveInputWeights(f, weights, bias);
}
HostArray<cudafloat> weights = mbpnet->GetLayerWeightsSpaceNetwork(0);
SaveWeights(f, weights);
}
f.WriteLine("0"); //epoch must be 0
f.WriteLine("0"); //rmsInterval
f.WriteLine("0"); //trainingTime
}
// WARNING: Assuming single class outputs
void AnalyzeTestData(BackPropagation * network, Configuration & cfg, ConfusionMatrix & cm) {
HostMatrix<cudafloat> & inputs = cfg.TestInputs();
HostMatrix<cudafloat> outputs = network->GetOutputs(inputs);
HostMatrix<cudafloat> & desiredOutputs = cfg.DesiredTestOutputs();
cm.Reset();
int nOutputs = outputs.Columns();
int patterns = outputs.Rows();
for(int p = 0; p < patterns; p++) {
int predicted = -1;
int correct = -1;
double max = -1.0;
if(nOutputs > 1) {
for (int o = 0; o < nOutputs; o++) {
if (desiredOutputs(p, o) >= CUDA_VALUE(0.5)) correct = o;
double output = outputs(p, o);
if (output > max) {
predicted = o;
max = output;
}
}
} else {
predicted = (outputs(p, 0) >= 0.5) ? 1 : 0;
correct = (desiredOutputs(p, 0) >= 0.5) ? 1 : 0;
}
cm.Classify(correct, predicted);
}
}
int main(int argc, char * argv[]) {
cout << "ATS - Autonomous Training System (version 0.2)" << endl;
cout << "This version supports missing values (using a Neural Selective Input Model - NSIM)." << endl << endl;
cout << "WARNING: This version is designed for classification problems." << endl << endl;
// Initialize the device
CudaDevice device;
if(!device.SupportsCuda()) {
cout << "Device does not support cuda" << endl;
return 0;
}
// Load information and check parameters
Configuration cfg(argc, argv);
if (cfg.HasInvalidParameters()) {
cout << endl << "Invalid parameter or no parameters specified. ";
cfg.ShowParameterInfo();
return 0;
}
if (!cfg.LoadedTrainData()) {
cout << "Could not load the training data. Check the filename and the number of inputs, outputs and samples." << endl;
return 0;
}
if (!cfg.LoadedTestData()) {
cout << "Could not load the test data. Check the filename and the number of inputs, outputs and samples." << endl;
return 0;
}
if (!cfg.Quiet()) {
device.ShowInfo();
cout << "Random Generator.........: " << cfg.RandomGenerator() << endl;
cout << "Network..................: " << cfg.Network() << cfg.Topology() << endl << endl;
}
int numberNetsTrain = cfg.NumberNetworksTrain();
HostArray<int> & layers = cfg.Layers();
HostArray<bool> selectiveNeurons(layers.Length() - 1);
selectiveNeurons[0] = cfg.MBP();
for(int l = 1; l < selectiveNeurons.Length(); l++) selectiveNeurons[l] = false;
HostArray<int> additionalSpaceLayers;
HostMatrix<cudafloat> & inputs = cfg.Inputs();
HostMatrix<cudafloat> & desiredOutputs = cfg.DesiredOutputs();
int bestNumberHiddenNeurons = layers[1];
int currentNumberHiddenNeurons = bestNumberHiddenNeurons;
bool down = false;
int numberTestPatterns = cfg.NumberTestPatterns();
bool automated = (numberNetsTrain > 1 && numberTestPatterns > 0 && !cfg.FixedTopology());
int nOutputs = desiredOutputs.Columns();
ConfusionMatrix cm((nOutputs == 1) ? 2 : nOutputs);
double bestFMeasure = 0.0;
int increment = 0;
cout << "Network\tName\tEpoch\tTime\tTrain RMS\tF-Meausure\tAccuracy" << endl;
cudafloat rmsStop = cfg.RmsStop();
long maxEpochs = cfg.MaximumEpochs();
BackPropagation * network = nullptr;
double sumFmeasures = 0.0;
double sumAccuracy = 0.0;
for(int net = 0; net < numberNetsTrain; net++) {
cout << cfg.Network() << cfg.Topology() << "\t" << cfg.RandomGenerator() << ".bpn\t";
if (cfg.MBP()) {
network = new MultipleBackPropagation(layers, selectiveNeurons, additionalSpaceLayers, inputs, desiredOutputs);
} else {
network = new BackPropagation(layers, inputs, desiredOutputs);
}
network->SetRobustLearning(cfg.RobustLearning());
clock_t initialTime = clock();
network->Train(maxEpochs, rmsStop);
cudaThreadSynchronize();
unsigned time = (clock() - initialTime);
cout << network->GetEpoch() << "\t" << (double) time / CLOCKS_PER_SEC << "s\t" << network->GetRMS() << "\t";
bool saveNetwork = true;
int prevInc = increment;
if (numberTestPatterns > 0) {
AnalyzeTestData(network, cfg, cm);
double fmeasure = cm.FMeasure();
double accuracy = cm.Accuracy();
cout << fmeasure << '\t' << accuracy << '\t';
sumFmeasures += fmeasure;
sumAccuracy += accuracy;
saveNetwork = (fmeasure >= bestFMeasure);
if (saveNetwork) {
bestFMeasure = fmeasure;
bestNumberHiddenNeurons = currentNumberHiddenNeurons;
increment++;
cout << "*";
if (bestFMeasure == 1.0) automated = false; // best topology was found
}
}
if (automated) {
if (currentNumberHiddenNeurons < bestNumberHiddenNeurons) {
if (down) {
down = false;
if (increment > 1) increment--;
}
} else if (currentNumberHiddenNeurons > bestNumberHiddenNeurons) {
if (!down) {
down = true;
if (increment > 1) increment--;
}
}
currentNumberHiddenNeurons += increment * ((down) ? -1 : 1);
if (currentNumberHiddenNeurons < 1) currentNumberHiddenNeurons = 1;
if (down && currentNumberHiddenNeurons == 1) down = false;
layers[1] = currentNumberHiddenNeurons;
}
cout << endl;
if (!cfg.Quiet()) cm.Show();
if (saveNetwork) SaveNetwork(network, cfg);
delete network;
cfg.ResetRandomGenerator();
}
cout << endl << numberNetsTrain << " network(s) trained" << endl;
if (numberTestPatterns > 0) {
cout << endl << "Average F-Measure: " << sumFmeasures / numberNetsTrain;
cout << endl << "Average Accuracy : " << sumAccuracy / numberNetsTrain << endl;
}
return 0;
}