Flood: MeanSquaredError.cpp Source File

00001 /****************************************************************************************************************/
00002 /*                                                                                                              */
00003 /*   Flood: An Open Source Neural Networks C++ Library                                                          */
00004 /*   www.cimne.com/flood                                                                                        */
00005 /*                                                                                                              */
00006 /*   M E A N   S Q U A R E D   E R R O R   C L A S S                                                            */
00007 /*                                                                                                              */
00008 /*   Roberto Lopez                                                                                              */
00009 /*   International Center for Numerical Methods in Engineering (CIMNE)                                          */
00010 /*   Technical University of Catalonia (UPC)                                                                    */
00011 /*   Barcelona, Spain                                                                                           */
00012 /*   E-mail: rlopez@cimne.upc.edu                                                                               */
00013 /*                                                                                                              */
00014 /****************************************************************************************************************/
00015 
00016 // Flood includes
00017 
00018 #include "MeanSquaredError.h"
00019 
00020 // System includes
00021 
00022 #include <string>
00023 #include <sstream>
00024 #include <iostream>
00025 #include <fstream>
00026 #include <limits>
00027 #include <math.h>
00028 
00029 namespace Flood
00030 {
00031 // DEFAULT CONSTRUCTOR
00032 
00036 
00037 MeanSquaredError::MeanSquaredError(void) : ObjectiveFunctional()
00038 {
00039    input_target_data_set_pointer = NULL;
00040 }
00041 
00042 
00043 // MULTILAYER PERCEPTRON CONSTRUCTOR
00044 
00049 
00050 MeanSquaredError::MeanSquaredError(MultilayerPerceptron* new_multilayer_perceptron_pointer)
00051 : ObjectiveFunctional(new_multilayer_perceptron_pointer)
00052 {
00053    input_target_data_set_pointer = NULL;
00054 }
00055 
00056 
00057 // INPUT-TARGET DATA SET CONSTRUCTOR
00058 
00063 
00064 MeanSquaredError::MeanSquaredError(InputTargetDataSet* new_input_target_data_set_pointer)
00065 : ObjectiveFunctional()
00066 {
00067    input_target_data_set_pointer = new_input_target_data_set_pointer;
00068 }
00069 
00070 
00071 // GENERAL CONSTRUCTOR
00072 
00078 
00079 MeanSquaredError::MeanSquaredError(MultilayerPerceptron* new_multilayer_perceptron_pointer, 
00080 InputTargetDataSet* new_input_target_data_set_pointer)
00081 : ObjectiveFunctional(new_multilayer_perceptron_pointer)
00082 {
00083    input_target_data_set_pointer = new_input_target_data_set_pointer;
00084 }
00085 
00086 
00087 // DESTRUCTOR
00088 
00090 
00091 MeanSquaredError::~MeanSquaredError(void)
00092 {
00093 
00094 }
00095 
00096 
00097 // METHODS
00098 
00099 // void set(void) method
00100 
00103 
00104 void MeanSquaredError::set(void)
00105 {
00106    multilayer_perceptron_pointer = NULL;
00107    input_target_data_set_pointer = NULL;
00108    set_default();
00109 }
00110 
00111 
00112 // void set(MultilayerPerceptron*) method
00113 
00117 
00118 void MeanSquaredError::set(MultilayerPerceptron* new_multilayer_perceptron_pointer)
00119 {
00120    multilayer_perceptron_pointer = new_multilayer_perceptron_pointer;
00121    input_target_data_set_pointer = NULL;
00122    set_default();
00123 }
00124 
00125 
00126 // void set(InputTargetDataSet*) method
00127 
00131 
00132 void MeanSquaredError::set(InputTargetDataSet* new_input_target_data_set_pointer)
00133 {
00134    multilayer_perceptron_pointer = NULL;
00135    input_target_data_set_pointer = new_input_target_data_set_pointer;
00136    set_default();
00137 }
00138 
00139 
00140 // void set(MultilayerPerceptron*, InputTargetDataSet*) method
00141 
00146 
00147 void MeanSquaredError::set(MultilayerPerceptron* new_multilayer_perceptron_pointer, InputTargetDataSet* new_input_target_data_set_pointer)
00148 {
00149    multilayer_perceptron_pointer = new_multilayer_perceptron_pointer;
00150    input_target_data_set_pointer = new_input_target_data_set_pointer;
00151    set_default();
00152 }
00153 
00154 
00155 // void set_input_target_data_set_pointer(InputTargetDataSet*) method
00156 
00159 
00160 void MeanSquaredError::set_input_target_data_set_pointer(InputTargetDataSet* new_input_target_data_set_pointer)
00161 {
00162    input_target_data_set_pointer = new_input_target_data_set_pointer;
00163 }
00164 
00165 
00166 // double calculate_objective(void) method
00167 
00170 
00171 double MeanSquaredError::calculate_objective(void)
00172 {
00173    // Control sentence (if debug)
00174 
00175    #ifdef _DEBUG 
00176 
00177    if(multilayer_perceptron_pointer == NULL)
00178    {
00179       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00180                 << "double calculate_objective(void) method." << std::endl
00181                 << "Pointer to multilayer perceptron object cannot be NULL." << std::endl;
00182 
00183         exit(1);
00184    }
00185    else if(input_target_data_set_pointer == NULL)
00186    {
00187       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00188                 << "double calculate_objective(void) method." << std::endl
00189                 << "Pointer to input-target data set object cannot be NULL." << std::endl;
00190 
00191       exit(1);
00192    }
00193 
00194    #endif
00195 
00196    int training_instances_number = input_target_data_set_pointer->get_training_instances_number();
00197 
00198    int inputs_number = multilayer_perceptron_pointer->get_inputs_number();
00199    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00200 
00201    #ifdef _DEBUG 
00202 
00203    int input_variables_number = input_target_data_set_pointer->get_input_variables_number();
00204 
00205    if(inputs_number != input_variables_number)
00206    {
00207       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00208                 << "double calculate_objective(void) method." << std::endl
00209                 << "Number of inputs in multilayer perceptron must be equal to " 
00210                 << "number of input variables in input-target data set." << std::endl;
00211 
00212       exit(1);
00213    }
00214 
00215    int target_variables_number = input_target_data_set_pointer->get_target_variables_number();
00216 
00217    if(outputs_number != target_variables_number)
00218    {
00219       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00220                 << "double calculate_objective(void) method." << std::endl
00221                 << "Number of outputs in multilayer perceptron must be equal to " 
00222                 << "number of target variables in input-target data set." << std::endl;
00223 
00224       exit(1);
00225    }
00226 
00227    #endif
00228 
00229    Vector<double> input(inputs_number);
00230    Vector<double> output(outputs_number);
00231    Vector<double> target(outputs_number);
00232    Vector<double> instance_error(outputs_number);
00233 
00234    double training_error = 0.0;
00235 
00236    for(int i = 0; i < training_instances_number; i++)
00237    {
00238       // Input vector
00239 
00240           input = input_target_data_set_pointer->get_training_input_instance(i);
00241 
00242       // Output vector
00243 
00244       output = multilayer_perceptron_pointer->calculate_output(input);
00245 
00246       // Target vector
00247 
00248       target = input_target_data_set_pointer->get_training_target_instance(i);
00249 
00250       // Error
00251 
00252            instance_error = output - target;
00253 
00254       // Sum of squares error
00255 
00256       training_error += instance_error.dot(instance_error);           
00257    }
00258 
00259    return(training_error/(double)training_instances_number);
00260 }
00261 
00262 
00263 // double calculate_validation_error(void) method
00264 
00267 
00268 double MeanSquaredError::calculate_validation_error(void)
00269 {
00270    // Control sentence (if debug)
00271 
00272    #ifdef _DEBUG 
00273 
00274    if(multilayer_perceptron_pointer == NULL)
00275    {
00276       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00277                 << "double calculate_validation_error(void) method." << std::endl
00278                 << "Pointer to multilayer perceptron object cannot be NULL." << std::endl;
00279 
00280         exit(1);
00281    }
00282    else if(input_target_data_set_pointer == NULL)
00283    {
00284       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00285                 << "double calculate_validation_error(void) method." << std::endl
00286                 << "Pointer to input-target data set object cannot be NULL." << std::endl;
00287 
00288       exit(1);
00289    }
00290 
00291    #endif
00292 
00293    int inputs_number = multilayer_perceptron_pointer->get_inputs_number();
00294    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00295 
00296    #ifdef _DEBUG 
00297 
00298    int input_variables_number = input_target_data_set_pointer->get_input_variables_number();
00299    int target_variables_number = input_target_data_set_pointer->get_target_variables_number();
00300 
00301    if(inputs_number != input_variables_number || outputs_number != target_variables_number)
00302    {
00303       std::cout << "Flood Error: MeanSquaredError class." << std::endl
00304                 << "double calculate_validation_error(void) method." << std::endl
00305                 << "Number of inputs and outputs in multilayer perceptron must be equal to " 
00306                 << "number of input and output variables in input-target data set." << std::endl;
00307 
00308       exit(1);
00309    }
00310 
00311    #endif
00312 
00313    int validation_instances_number = input_target_data_set_pointer->get_validation_instances_number();
00314 
00315    if(validation_instances_number == 0)
00316    {
00317       return(0.0);
00318    }
00319    else
00320    {
00321       Vector<double> input(inputs_number);
00322       Vector<double> output(outputs_number);
00323       Vector<double> target(outputs_number);
00324 
00325       double validation_error = 0.0;
00326 
00327       for(int i = 0; i < validation_instances_number; i++)
00328       {
00329          // Input vector
00330 
00331              input = input_target_data_set_pointer->get_validation_input_instance(i);
00332 
00333          // Output vector
00334 
00335          output = multilayer_perceptron_pointer->calculate_output(input);
00336 
00337          // Target vector
00338 
00339          target = input_target_data_set_pointer->get_validation_target_instance(i);
00340 
00341          // Sum of squares error
00342 
00343          validation_error += (output - target).dot(output - target);           
00344       }
00345 
00346       return(validation_error/(double)validation_instances_number);
00347    }
00348 }
00349 
00350 
00351 // Vector<double> calculate_objective_gradient(void) method
00352 
00355 
00356 Vector<double> MeanSquaredError::calculate_objective_gradient(void)
00357 {
00358    // Control sentence (if debug)
00359 
00360    #ifdef _DEBUG 
00361 
00362    if(multilayer_perceptron_pointer == NULL)
00363    {
00364       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00365                 << "double calculate_objective_gradient(void) method." << std::endl
00366                 << "Pointer to multilayer perceptron object cannot be NULL." << std::endl;
00367 
00368         exit(1);
00369    }
00370    else if(input_target_data_set_pointer == NULL)
00371    {
00372       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00373                 << "double calculate_objective_gradient(void) method." << std::endl
00374                 << "Pointer to input-target data set object cannot be NULL." << std::endl;
00375 
00376       exit(1);
00377    }
00378 
00379    #endif
00380 
00381    // Multilayer perceptron 
00382 
00383    int hidden_layers_number = multilayer_perceptron_pointer->get_hidden_layers_number();
00384 
00385    Vector<int> hidden_layers_size = multilayer_perceptron_pointer->get_hidden_layers_size();
00386 
00387    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00388 
00389    int forward_propagation_derivative_size = 2*hidden_layers_number + 2;
00390 
00391    Vector< Vector<double> > forward_propagation_derivative(forward_propagation_derivative_size);
00392 
00393    // Input-target data set
00394 
00395    int training_instances_number = input_target_data_set_pointer->get_training_instances_number();
00396 
00397    int input_variables_number = input_target_data_set_pointer->get_input_variables_number();
00398    int target_variables_number = input_target_data_set_pointer->get_target_variables_number();    Vector<double> training_input_instance(input_variables_number);
00399    Vector<double> training_target_instance(target_variables_number);
00400 
00401    // Output and hidden errors
00402 
00403    Vector<double> output_errors(outputs_number);
00404    Vector< Vector<double> > hidden_errors(hidden_layers_number);
00405 
00406    for(int h = 0; h < hidden_layers_number; h++)
00407    {
00408       hidden_errors[h].set_size(hidden_layers_size[h]);
00409    }
00410 
00411    // Hidden and output layers gradients
00412 
00413    int hidden_layers_parameters_number = multilayer_perceptron_pointer->get_hidden_layers_parameters_number();
00414    int output_layer_parameters_number = multilayer_perceptron_pointer->get_output_layer_parameters_number();
00415 
00416    Vector<double> hidden_layers_error_gradient(hidden_layers_parameters_number, 0.0);
00417    Vector<double> output_layer_error_gradient(output_layer_parameters_number, 0.0);
00418 
00419    // Main loop
00420 
00421    for(int i = 0; i < training_instances_number; i++)
00422    {
00423       training_input_instance = input_target_data_set_pointer->get_training_input_instance(i);
00424 
00425       forward_propagation_derivative = multilayer_perceptron_pointer->calculate_forward_propagation_derivative(training_input_instance);
00426 
00427       training_target_instance = input_target_data_set_pointer->get_training_target_instance(i);
00428 
00429           output_errors = calculate_output_errors(forward_propagation_derivative, training_target_instance);
00430 
00431       hidden_errors = calculate_hidden_errors(forward_propagation_derivative, output_errors);
00432 
00433       hidden_layers_error_gradient += calculate_hidden_layers_error_gradient(training_input_instance, forward_propagation_derivative, hidden_errors);
00434   
00435       output_layer_error_gradient += calculate_output_layer_error_gradient(forward_propagation_derivative, output_errors);
00436    }
00437 
00438    return(hidden_layers_error_gradient.assemble(output_layer_error_gradient));   
00439 }
00440 
00441 
00442 // Vector<double> calculate_output_errors(const Vector< Vector<double> >&, const Vector<double>&) method
00443 
00447 
00448 Vector<double> MeanSquaredError::calculate_output_errors
00449 (const Vector< Vector<double> >& forward_propagation_derivative, const Vector<double>& target)
00450 {
00451    int forward_propagation_derivative_size = forward_propagation_derivative.get_size();
00452    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00453 
00454    // Control sentence (if debug)
00455 
00456    #ifdef _DEBUG 
00457 
00458    int hidden_layers_number = multilayer_perceptron_pointer->get_hidden_layers_number();
00459 
00460    if(forward_propagation_derivative_size != 2*hidden_layers_number+2)
00461    {
00462       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00463                 << "double calculate_output_errors(void) method." << std::endl
00464                 << "Size of forward propagation derivative vector must be equal to 2*hidden_layers_number+2." 
00465                                 << std::endl;
00466 
00467         exit(1);
00468    }
00469       
00470    int output_layer_output_size = forward_propagation_derivative[forward_propagation_derivative_size-2].get_size();
00471       
00472    if(output_layer_output_size != outputs_number)
00473    {
00474       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00475                 << "double calculate_output_errors(void) method." << std::endl
00476                 << "Size of output layer output ("<< output_layer_output_size << ") must be equal to "
00477                 << "number of outputs (" << outputs_number << ")." << std::endl;
00478 
00479       exit(1);
00480    }
00481 
00482    int output_layer_output_derivative_size 
00483    = forward_propagation_derivative[forward_propagation_derivative_size-1].get_size();
00484 
00485    if(output_layer_output_derivative_size != outputs_number)
00486    {
00487       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00488                 << "double calculate_output_errors(void) method." << std::endl
00489                 << "Size of output layer output derivative (" << output_layer_output_derivative_size << ")must be equal to "               << "number of outputs (" << outputs_number << ")." << std::endl;
00490 
00491       exit(1);
00492    }
00493 
00494    int target_size = target.get_size();
00495 
00496    if(target_size != outputs_number)
00497    {
00498       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00499                 << "double calculate_output_errors(void) method." << std::endl
00500                 << "Size of target must be equal to number of outputs." << std::endl;
00501 
00502       exit(1);
00503    }
00504 
00505    #endif
00506 
00507    Vector<double> output_layer_output_derivative = forward_propagation_derivative[forward_propagation_derivative_size-1];
00508    Vector<double> output_layer_output = forward_propagation_derivative[forward_propagation_derivative_size-2];
00509 
00510    int training_instances_number = input_target_data_set_pointer->get_training_instances_number();
00511 
00512    Vector<double> output_errors(outputs_number);
00513 
00514    Vector<double> error(outputs_number);
00515 
00516    MultilayerPerceptron::ScalingMethod outputs_unscaling_method
00517    = multilayer_perceptron_pointer->get_outputs_unscaling_method();
00518 
00519    switch(outputs_unscaling_method)
00520    {
00521       case MultilayerPerceptron::None:
00522       {   
00523          error = output_layer_output-target;
00524 
00525          output_errors = output_layer_output_derivative*2.0*error/(double)training_instances_number;
00526           } 
00527       break;
00528 
00529       case MultilayerPerceptron::MeanStandardDeviation:
00530       {         
00531          Vector<double> output_variables_standard_deviation = multilayer_perceptron_pointer->get_output_variables_standard_deviation();
00532 
00533          // Control sentence (if debug)
00534 
00535          #ifdef _DEBUG 
00536  
00537                  int output_variables_standard_deviation_size = output_variables_standard_deviation.get_size();
00538 
00539          if(output_variables_standard_deviation_size != outputs_number)
00540          {
00541            std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00542                      << "Vector<double> calculate_output_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00543                      << "Size of output variables standard deviation must be equal to number of outputs." << std::endl;
00544 
00545            exit(1);
00546          }
00547 
00548          #endif
00549 
00550                  output_errors = output_layer_output_derivative*2.0*error*output_variables_standard_deviation/(double)training_instances_number;
00551 
00552       }
00553       break;
00554 
00555       case MultilayerPerceptron::MinimumMaximum:
00556       {
00557          // Control sentence (if debug)
00558 
00559          #ifdef _DEBUG 
00560  
00561                  int output_variables_minimum_size = multilayer_perceptron_pointer->get_output_variables_minimum().get_size();
00562                  int output_variables_maximum_size = multilayer_perceptron_pointer->get_output_variables_maximum().get_size();
00563 
00564          if(output_variables_minimum_size != outputs_number)
00565          {
00566            std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00567                      << "Vector<double> calculate_output_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00568                      << "Size of output variables minimum must be equal to number of outputs." << std::endl;
00569 
00570            exit(1);
00571          }
00572          else if(output_variables_maximum_size != outputs_number)
00573          {
00574            std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00575                      << "Vector<double> calculate_output_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00576                      << "Size of output variables maximum must be equal to number of outputs." << std::endl;
00577 
00578            exit(1);
00579          }
00580 
00581          #endif
00582 
00583          Vector<double> output_variables_minimum = multilayer_perceptron_pointer->get_output_variables_minimum();
00584          Vector<double> output_variables_maximum = multilayer_perceptron_pointer->get_output_variables_maximum();
00585 
00586          Vector<double> output_variables_range = output_variables_maximum-output_variables_minimum;
00587 
00588              output_errors = output_layer_output_derivative*(output_layer_output-target)*output_variables_range/(double)training_instances_number;
00589       }          
00590       break;
00591       
00592       default:
00593       {
00594          std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00595                    << "double calculate_output_errors(void) method." << std::endl
00596                    << "Unknown outputs unscaling method." << std::endl;
00597 
00598          exit(1);
00599       }          
00600       break;     
00601    }
00602 
00603    return(output_errors);
00604 }
00605 
00606 
00607 // Vector< Vector<double> > calculate_hidden_errors(const Vector< Vector<double> >&, const Vector<double>&) method
00608 
00612 
00613 Vector< Vector<double> > MeanSquaredError::calculate_hidden_errors
00614 (const Vector< Vector<double> >& forward_propagation_derivative, const Vector<double>& output_errors)
00615 {
00616    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00617 
00618    int hidden_layers_number = multilayer_perceptron_pointer->get_hidden_layers_number();
00619 
00620    // Control sentence (if debug)
00621 
00622    #ifdef _DEBUG 
00623 
00624    int forward_propagation_derivative_size = forward_propagation_derivative.get_size();
00625 
00626    if(forward_propagation_derivative_size != 2*hidden_layers_number+2)
00627    {
00628       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00629                 << "Vector< Vector<double> > calculate_hidden_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00630                 << "Size of forward propagation derivative vector must be equal to 2*hidden_layers_number+2." 
00631                                 << std::endl;
00632 
00633         exit(1);
00634    }
00635       
00636    int output_layer_output_size = forward_propagation_derivative[forward_propagation_derivative_size-2].get_size();
00637       
00638    if(output_layer_output_size != outputs_number)
00639    {
00640       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00641                 << "Vector< Vector<double> > calculate_hidden_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00642                 << "Size of output layer output ("<< output_layer_output_size << ") must be equal to "
00643                 << "number of outputs (" << outputs_number << ")." << std::endl;
00644 
00645       exit(1);
00646    }
00647 
00648    int output_layer_output_derivative_size 
00649    = forward_propagation_derivative[forward_propagation_derivative_size-1].get_size();
00650 
00651    if(output_layer_output_derivative_size != outputs_number)
00652    {
00653       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00654                 << "Vector< Vector<double> > calculate_hidden_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00655                 << "Size of output layer output derivative (" << output_layer_output_derivative_size << ")must be equal to " 
00656                                 << "number of outputs (" << outputs_number << ")." << std::endl;
00657 
00658       exit(1);
00659    }
00660 
00661    int output_errors_size = output_errors.get_size();
00662 
00663    if(output_errors_size != outputs_number)
00664    {
00665       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00666                 << "Vector< Vector<double> > calculate_hidden_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00667                 << "Size of target must be equal to number of outputs." << std::endl;
00668 
00669       exit(1);
00670    }
00671 
00672    #endif
00673 
00674    // Set hidden errors vector of vectors
00675 
00676    Vector<int> hidden_layers_size = multilayer_perceptron_pointer->get_hidden_layers_size();
00677 
00678    Vector< Vector<double> > hidden_errors(hidden_layers_number);
00679 
00680    for(int h = 0; h < hidden_layers_number; h++)
00681    {
00682       hidden_errors[h].set_size(hidden_layers_size[h]);
00683    }
00684 
00685    // Multilayer perceptron
00686 
00687    Vector<Perceptron>& output_layer = multilayer_perceptron_pointer->get_output_layer();
00688    Vector< Vector<Perceptron> >& hidden_layers = multilayer_perceptron_pointer->get_hidden_layers();
00689 
00690    Vector< Vector<double> > hidden_layers_output_derivative(hidden_layers_number);
00691 
00692    for(int i = 0; i < hidden_layers_number; i++)
00693    {
00694       hidden_layers_output_derivative[i] = forward_propagation_derivative[1+2*i];
00695    }
00696 
00697    Vector<double> synaptic_weights;
00698 
00699    double sum;
00700     
00701    // Last hidden layer
00702 
00703    for(int j = 0; j < hidden_layers_size[hidden_layers_number-1]; j++)
00704    {
00705       sum = 0.0;
00706 
00707       for(int k = 0; k < outputs_number; k++)
00708       {
00709          synaptic_weights = output_layer[k].get_synaptic_weights();
00710 
00711          sum += (synaptic_weights[j])*output_errors[k];
00712       }
00713 
00714       hidden_errors[hidden_layers_number-1][j] = hidden_layers_output_derivative[hidden_layers_number-1][j]*sum;
00715    }
00716 
00717    // Rest of hidden layers
00718 
00719    for(int h = hidden_layers_number-2; h >= 0; h--) 
00720    {   
00721       for(int j = 0; j < hidden_layers_size[h]; j++)
00722       {
00723          sum = 0.0;
00724 
00725          for(int k = 0; k < hidden_layers_size[h+1]; k++)
00726          {
00727             synaptic_weights = hidden_layers[h+1][k].get_synaptic_weights();
00728 
00729             sum += (synaptic_weights[j])*hidden_errors[h+1][k];
00730          }               
00731 
00732          hidden_errors[h][j] = hidden_layers_output_derivative[h][j]*sum;
00733       }
00734    }
00735 
00736    return(hidden_errors);
00737 }
00738 
00739 
00740 // Vector<double> calculate_hidden_layers_error_gradient
00741 // (const Vector<double>&, const Vector< Vector<double> >&, const Vector< Vector<double> >&) ) method
00742 
00747 
00748 Vector<double> MeanSquaredError::calculate_hidden_layers_error_gradient
00749 (const Vector<double>& input, 
00750  const Vector< Vector<double> >& forward_propagation_derivative, 
00751  const Vector< Vector<double> >& hidden_errors)
00752 {
00753    int inputs_number = multilayer_perceptron_pointer->get_inputs_number();
00754    int hidden_layers_number = multilayer_perceptron_pointer->get_hidden_layers_number();
00755 
00756    // Control sentence (if debug)
00757 
00758    #ifdef _DEBUG 
00759 
00760    int input_size = input.get_size();
00761 
00762    if(input_size != inputs_number)
00763    {
00764       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00765                 << "Vector< Vector<double> > calculate_hidden_layers_error_gradient(const Vector< Vector<double> >&, const Vector<double>&, const Vector<double>&) method." << std::endl
00766                 << "Size of input (" << input_size << ") must be equal to inputs number (" << inputs_number << ")." << std::endl;
00767 
00768         exit(1);
00769    }
00770 
00771 
00772    int forward_propagation_derivative_size = forward_propagation_derivative.get_size();
00773 
00774    if(forward_propagation_derivative_size != 2*hidden_layers_number+2)
00775    {
00776       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00777                 << "Vector< Vector<double> > calculate_hidden_layers_error_gradient(const Vector< Vector<double> >&, const Vector<double>&, const Vector<double>&) method." << std::endl
00778                 << "Size of forward propagation derivative (" << forward_propagation_derivative_size << ") must be equal to 2*hidden_layers_number+2 (" << 2*hidden_layers_number+2 << ")." 
00779                                 << std::endl;
00780 
00781         exit(1);
00782    }
00783       
00784    int hidden_errors_size = hidden_errors.get_size();
00785       
00786    if(hidden_errors_size != hidden_layers_number)
00787    {
00788       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00789                 << "Vector< Vector<double> > calculate_hidden_layers_error_gradient(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00790                 << "Size of output errors ("<< hidden_errors_size << ") must be equal to number of hidden layers (" << hidden_layers_number << ")." << std::endl;
00791 
00792       exit(1);
00793    }
00794 
00795    #endif
00796 
00797    // Multilayer perceptron stuff 
00798 
00799    Vector<int> hidden_layers_size = multilayer_perceptron_pointer->get_hidden_layers_size();
00800 
00801    int hidden_layers_parameters_number = multilayer_perceptron_pointer->get_hidden_layers_parameters_number();
00802 
00803    Vector< Vector<Perceptron> >& hidden_layers = multilayer_perceptron_pointer->get_hidden_layers();
00804 
00805    Vector<double> synaptic_weights;
00806 
00807    Vector< Vector<double> > hidden_layers_output(hidden_layers_number);
00808 
00809    for(int i = 0; i < hidden_layers_number; i++)
00810    {
00811       hidden_layers_output[i] = forward_propagation_derivative[2*i];
00812    }
00813 
00814    int index = 0;
00815 
00816    Vector<double> hidden_layers_error_gradient(hidden_layers_parameters_number, 0.0);
00817 
00818    // First hidden layer
00819 
00820    for(int j = 0; j < hidden_layers_size[0]; j++)
00821    {
00822       // Bias
00823 
00824       hidden_layers_error_gradient[index] += hidden_errors[0][j];
00825       index++;
00826 
00827       // Synaptic weights
00828 
00829       synaptic_weights = hidden_layers[0][j].get_synaptic_weights();
00830 
00831       for(int k = 0; k < inputs_number; k++)
00832       {
00833          hidden_layers_error_gradient[index] += hidden_errors[0][j]*input[k];
00834          index++;   
00835       }
00836    }
00837 
00838    // Rest of hidden layers     
00839     
00840    for(int h = 1; h < hidden_layers_number; h++)
00841    {      
00842       for(int j = 0; j < hidden_layers_size[h]; j++)
00843       {
00844          // Bias
00845 
00846          hidden_layers_error_gradient[index] += hidden_errors[h][j];
00847          index++;
00848 
00849          // Synaptic weights
00850 
00851          synaptic_weights = hidden_layers[h][j].get_synaptic_weights();
00852 
00853          for(int k = 0; k < hidden_layers_size[h-1]; k++)
00854          {
00855             hidden_layers_error_gradient[index] += hidden_errors[h][j]*hidden_layers_output[h-1][k];
00856             index++;   
00857          }
00858       }
00859    }
00860 
00861    return(hidden_layers_error_gradient);
00862 }
00863 
00864 
00865 // Vector<double> calculate_output_layer_error_gradient(const Vector< Vector<double> >&, const Vector<double>&) method
00866 
00870 
00871 Vector<double> MeanSquaredError::calculate_output_layer_error_gradient
00872 (const Vector< Vector<double> >& forward_propagation_derivative, const Vector<double>& output_errors)
00873 {
00874    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00875    int hidden_layers_number = multilayer_perceptron_pointer->get_hidden_layers_number();
00876 
00877    // Control sentence (if debug)
00878 
00879    #ifdef _DEBUG 
00880 
00881    int forward_propagation_derivative_size = forward_propagation_derivative.get_size();
00882 
00883    if(forward_propagation_derivative_size != 2*hidden_layers_number+2)
00884    {
00885       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00886                 << "Vector< Vector<double> > calculate_output_layer_error_gradient(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00887                 << "Size of forward propagation derivative (" << forward_propagation_derivative_size << ") must be equal to 2*hidden_layers_number+2 (" << 2*hidden_layers_number+2 << ")." 
00888                                 << std::endl;
00889 
00890         exit(1);
00891    }
00892       
00893    int output_errors_size = output_errors.get_size();
00894       
00895    if(output_errors_size != outputs_number)
00896    {
00897       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00898                 << "Vector< Vector<double> > calculate_output_layer_error_gradient(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00899                 << "Size of output errors ("<< output_errors_size << ") must be equal to number of outputs (" << outputs_number << ")." << std::endl;
00900 
00901       exit(1);
00902    }
00903 
00904    #endif
00905 
00906    // Multilayer perceptron stuff
00907 
00908    Vector<int> hidden_layers_size = multilayer_perceptron_pointer->get_hidden_layers_size();
00909 
00910    int output_layer_parameters_number = multilayer_perceptron_pointer->get_output_layer_parameters_number();
00911 
00912    Vector< Vector<double> > hidden_layers_output(hidden_layers_number);
00913 
00914    for(int i = 0; i < hidden_layers_number; i++)
00915    {
00916       hidden_layers_output[i] = forward_propagation_derivative[2*i];
00917    }
00918 
00919    // Calculate gradient elements of output neurons
00920 
00921    Vector<double> output_layer_error_gradient(output_layer_parameters_number, 0.0);
00922 
00923    int index = 0;
00924 
00925    for(int j = 0; j < outputs_number; j++)
00926    {
00927       // Bias
00928 
00929       output_layer_error_gradient[index] += output_errors[j];
00930       index++;
00931 
00932       // Synaptic weights
00933 
00934       for(int k = 0; k < hidden_layers_size[hidden_layers_number-1]; k++)
00935       {
00936          output_layer_error_gradient[index] = hidden_layers_output[hidden_layers_number-1][k]*output_errors[j];
00937          index++;
00938       }
00939    }
00940 
00941    return(output_layer_error_gradient);
00942 }
00943 
00944 
00945 
00946 
00947 // Matrix<double> calculate_Jacobian(void) method
00948 
00951 
00952 Matrix<double> MeanSquaredError::calculate_Jacobian(void)
00953 {
00954    // Control sentence (if debug)
00955 
00956    #ifdef _DEBUG 
00957 
00958    if(multilayer_perceptron_pointer == NULL)
00959    {
00960       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00961                 << "double calculate_Jacobian(void) method." << std::endl
00962                 << "Pointer to multilayer perceptron object cannot be NULL." << std::endl;
00963 
00964         exit(1);
00965    }
00966    else if(input_target_data_set_pointer == NULL)
00967    {
00968       std::cerr << "Flood Error: MeanSquaredError class." << std::endl
00969                 << "double calculate_Jacobian(void) method." << std::endl
00970                 << "Pointer to input-target data set object cannot be NULL." << std::endl;
00971 
00972       exit(1);
00973    }
00974 
00975    #endif
00976 
00977    int training_instances_number = input_target_data_set_pointer->get_training_instances_number();
00978    int neural_parameters_number = multilayer_perceptron_pointer->get_neural_parameters_number();
00979 
00980    Matrix<double> Jacobian(training_instances_number, neural_parameters_number);
00981 
00982    // Multilayer perceptron 
00983 
00984    int hidden_layers_number = multilayer_perceptron_pointer->get_hidden_layers_number();
00985 
00986    Vector<int> hidden_layers_size = multilayer_perceptron_pointer->get_hidden_layers_size();
00987 
00988    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00989 
00990    int forward_propagation_derivative_size = 2*hidden_layers_number + 2;
00991 
00992    Vector< Vector<double> > forward_propagation_derivative(forward_propagation_derivative_size);
00993 
00994    // Input-target data set
00995 
00996    int target_variables_number = input_target_data_set_pointer->get_target_variables_number();    Vector<double> training_input_instance(target_variables_number);
00997    Vector<double> training_target_instance(target_variables_number);
00998 
00999    // Output and hidden errors
01000 
01001    Vector<double> output_errors(outputs_number);
01002    Vector< Vector<double> > hidden_errors(hidden_layers_number);
01003 
01004    for(int h = 0; h < hidden_layers_number; h++)
01005    {
01006       hidden_errors[h].set_size(hidden_layers_size[h]);
01007    }
01008 
01009    // Hidden and output layers gradients
01010 
01011    int hidden_layers_parameters_number = multilayer_perceptron_pointer->get_hidden_layers_parameters_number();
01012    int output_layer_parameters_number = multilayer_perceptron_pointer->get_output_layer_parameters_number();
01013 
01014    Vector<double> hidden_layers_error_gradient(hidden_layers_parameters_number, 0.0);
01015    Vector<double> output_layer_error_gradient(output_layer_parameters_number, 0.0);
01016 
01017    Vector<double> gradient(outputs_number);
01018 
01019    // Main loop
01020 
01021    for(int i = 0; i < training_instances_number; i++)
01022    {
01023       training_input_instance = input_target_data_set_pointer->get_training_input_instance(i);
01024 
01025       forward_propagation_derivative = multilayer_perceptron_pointer->calculate_forward_propagation_derivative(training_input_instance);
01026 
01027       training_target_instance = input_target_data_set_pointer->get_training_target_instance(i);
01028 
01029       output_errors = calculate_output_errors(forward_propagation_derivative, training_target_instance);
01030 
01031       hidden_errors = calculate_hidden_errors(forward_propagation_derivative, output_errors);
01032 
01033       hidden_layers_error_gradient = calculate_hidden_layers_error_gradient(training_input_instance, forward_propagation_derivative, hidden_errors);
01034 
01035       output_layer_error_gradient = calculate_output_layer_error_gradient(forward_propagation_derivative, output_errors);
01036 
01037       gradient = hidden_layers_error_gradient.assemble(output_layer_error_gradient);
01038 
01039       Jacobian.set_row(i, gradient);
01040    }
01041 
01042    return(Jacobian);
01043 }
01044 
01045 }
01046 
01047 // Flood: An Open Source Neural Networks C++ Library.
01048 // Copyright (C) 2005-2010 Roberto Lopez 
01049 //
01050 // This library is free software; you can redistribute it and/or
01051 // modify it under the s of the GNU Lesser General Public
01052 // License as published by the Free Software Foundation; either
01053 // version 2.1 of the License, or any later version.
01054 //
01055 // This library is distributed in the hope that it will be useful,
01056 // but WITHOUT ANY WARRANTY; without even the implied warranty of
01057 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
01058 // Lesser General Public License for more details.
01059 
01060 // You should have received a copy of the GNU Lesser General Public
01061 // License along with this library; if not, write to the Free Software
01062 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA