Flood: SumSquaredError.cpp Source File

00001 /****************************************************************************************************************/
00002 /*                                                                                                              */
00003 /*   Flood: An Open Source Neural Networks C++ Library                                                          */
00004 /*   www.cimne.com/flood                                                                                        */
00005 /*                                                                                                              */
00006 /*   S U M   S Q U A R E D   E R R O R   C L A S S                                                              */
00007 /*                                                                                                              */
00008 /*   Roberto Lopez                                                                                              */
00009 /*   International Center for Numerical Methods in Engineering (CIMNE)                                          */
00010 /*   Technical University of Catalonia (UPC)                                                                    */
00011 /*   Barcelona, Spain                                                                                           */
00012 /*   E-mail: rlopez@cimne.upc.edu                                                                               */
00013 /*                                                                                                              */
00014 /****************************************************************************************************************/
00015 
00016 // Flood includes
00017 
00018 #include "SumSquaredError.h"
00019 
00020 // System includes
00021 
00022 #include <string>
00023 #include <sstream>
00024 #include <iostream>
00025 #include <fstream>
00026 #include <cmath>
00027 
00028 namespace Flood
00029 {
00030 
00031 // DEFAULT CONSTRUCTOR
00032 
00036 
00037 SumSquaredError::SumSquaredError(void) : ObjectiveFunctional()
00038 {
00039    input_target_data_set_pointer = NULL;
00040 }
00041 
00042 
00043 // MULTILAYER PERCEPTRON CONSTRUCTOR
00044 
00049 
00050 SumSquaredError::SumSquaredError(MultilayerPerceptron* new_multilayer_perceptron_pointer) 
00051 : ObjectiveFunctional(new_multilayer_perceptron_pointer)
00052 {
00053    input_target_data_set_pointer = NULL;
00054 }
00055 
00056 
00057 // INPUT-TARGET DATA SET CONSTRUCTOR
00058 
00063 
00064 SumSquaredError::SumSquaredError(InputTargetDataSet* new_input_target_data_set_pointer)
00065 : ObjectiveFunctional()
00066 {
00067    input_target_data_set_pointer = new_input_target_data_set_pointer;
00068 }
00069 
00070 
00071 // GENERAL CONSTRUCTOR
00072 
00078 
00079 SumSquaredError::SumSquaredError(MultilayerPerceptron* new_multilayer_perceptron_pointer, 
00080 InputTargetDataSet* new_input_target_data_set_pointer): ObjectiveFunctional(new_multilayer_perceptron_pointer)
00081 {
00082    input_target_data_set_pointer = new_input_target_data_set_pointer;
00083 }
00084 
00085 
00086 // DESTRUCTOR
00087 
00089 
00090 SumSquaredError::~SumSquaredError(void) 
00091 {
00092 }
00093 
00094 
00095 // METHODS
00096 
00097 
00098 // void set(void) method
00099 
00102 
00103 void SumSquaredError::set(void)
00104 {
00105    multilayer_perceptron_pointer = NULL;
00106    input_target_data_set_pointer = NULL;
00107    set_default();
00108 }
00109 
00110 
00111 // void set(MultilayerPerceptron*) method
00112 
00116 
00117 void SumSquaredError::set(MultilayerPerceptron* new_multilayer_perceptron_pointer)
00118 {
00119    multilayer_perceptron_pointer = new_multilayer_perceptron_pointer;
00120    input_target_data_set_pointer = NULL;
00121    set_default();
00122 }
00123 
00124 
00125 // void set(InputTargetDataSet*) method
00126 
00130 
00131 void SumSquaredError::set(InputTargetDataSet* new_input_target_data_set_pointer)
00132 {
00133    multilayer_perceptron_pointer = NULL;
00134    input_target_data_set_pointer = new_input_target_data_set_pointer;
00135    set_default();
00136 }
00137 
00138 
00139 // void set(MultilayerPerceptron*, InputTargetDataSet*) method
00140 
00145 
00146 void SumSquaredError::set(MultilayerPerceptron* new_multilayer_perceptron_pointer, InputTargetDataSet* new_input_target_data_set_pointer)
00147 {
00148    multilayer_perceptron_pointer = new_multilayer_perceptron_pointer;
00149    input_target_data_set_pointer = new_input_target_data_set_pointer;
00150    set_default();
00151 }
00152 
00153 
00154 // void set_input_target_data_set_pointer(InputTargetDataSet*) method
00155 
00158 
00159 void SumSquaredError::set_input_target_data_set_pointer(InputTargetDataSet* new_input_target_data_set_pointer)
00160 {
00161    input_target_data_set_pointer = new_input_target_data_set_pointer;
00162 }
00163 
00164 
00165 // double calculate_objective(void) method
00166 
00169 
00170 double SumSquaredError::calculate_objective(void)
00171 {
00172    // Control sentence (if debug)
00173 
00174    #ifdef _DEBUG 
00175 
00176    if(multilayer_perceptron_pointer == NULL)
00177    {
00178       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00179                 << "double calculate_objective(void) method." << std::endl
00180                 << "Pointer to multilayer perceptron object cannot be NULL." << std::endl;
00181 
00182         exit(1);
00183    }
00184    else if(input_target_data_set_pointer == NULL)
00185    {
00186       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00187                 << "double calculate_objective(void) method." << std::endl
00188                 << "Pointer to input-target data set object cannot be NULL." << std::endl;
00189 
00190       exit(1);
00191    }
00192 
00193    #endif
00194 
00195    int training_instances_number = input_target_data_set_pointer->get_training_instances_number();
00196 
00197    int inputs_number = multilayer_perceptron_pointer->get_inputs_number();
00198    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00199 
00200    #ifdef _DEBUG 
00201 
00202    int input_variables_number = input_target_data_set_pointer->get_input_variables_number();
00203 
00204    if(inputs_number != input_variables_number)
00205    {
00206       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00207                 << "double calculate_objective(void) method." << std::endl
00208                 << "Number of inputs in multilayer perceptron must be equal to " 
00209                 << "number of input variables in input-target data set." << std::endl;
00210 
00211       exit(1);
00212    }
00213 
00214    int target_variables_number = input_target_data_set_pointer->get_target_variables_number();
00215 
00216    if(outputs_number != target_variables_number)
00217    {
00218       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00219                 << "double calculate_objective(void) method." << std::endl
00220                 << "Number of outputs in multilayer perceptron must be equal to " 
00221                 << "number of target variables in input-target data set." << std::endl;
00222 
00223       exit(1);
00224    }
00225 
00226    #endif
00227 
00228    Vector<double> input(inputs_number);
00229    Vector<double> output(outputs_number);
00230    Vector<double> target(outputs_number);
00231    Vector<double> instance_error(outputs_number);
00232 
00233    double training_error = 0.0;
00234 
00235    for(int i = 0; i < training_instances_number; i++)
00236    {
00237       // Input vector
00238 
00239           input = input_target_data_set_pointer->get_training_input_instance(i);
00240 
00241       // Output vector
00242 
00243       output = multilayer_perceptron_pointer->calculate_output(input);
00244 
00245       // Target vector
00246 
00247       target = input_target_data_set_pointer->get_training_target_instance(i);
00248 
00249       // Error
00250 
00251            instance_error = output - target;
00252 
00253       // Sum of squares error
00254 
00255       training_error += instance_error.dot(instance_error);           
00256    }
00257 
00258    return(training_error);
00259 }
00260 
00261 
00262 // double calculate_validation_error(void) method
00263 
00266 
00267 double SumSquaredError::calculate_validation_error(void)
00268 {
00269    // Control sentence (if debug)
00270 
00271    #ifdef _DEBUG 
00272 
00273    if(multilayer_perceptron_pointer == NULL)
00274    {
00275       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00276                 << "double calculate_validation_error(void) method." << std::endl
00277                 << "Pointer to multilayer perceptron object cannot be NULL." << std::endl;
00278 
00279         exit(1);
00280    }
00281    else if(input_target_data_set_pointer == NULL)
00282    {
00283       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00284                 << "double calculate_validation_error(void) method." << std::endl
00285                 << "Pointer to input-target data set object cannot be NULL." << std::endl;
00286 
00287       exit(1);
00288    }
00289 
00290    #endif
00291 
00292    int inputs_number = multilayer_perceptron_pointer->get_inputs_number();
00293    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00294 
00295    #ifdef _DEBUG 
00296 
00297    int input_variables_number = input_target_data_set_pointer->get_input_variables_number();
00298    int target_variables_number = input_target_data_set_pointer->get_target_variables_number();
00299 
00300    if(inputs_number != input_variables_number || outputs_number != target_variables_number)
00301    {
00302       std::cout << "Flood Error: SumSquaredError class." << std::endl
00303                 << "double calculate_validation_error(void) method." << std::endl
00304                 << "Number of inputs and outputs in multilayer perceptron must be equal to " 
00305                 << "number of input and output variables in input-target data set." << std::endl;
00306 
00307       exit(1);
00308    }
00309 
00310    #endif
00311 
00312    int validation_instances_number = input_target_data_set_pointer->get_validation_instances_number();
00313 
00314    Vector<double> input(inputs_number);
00315    Vector<double> output(outputs_number);
00316    Vector<double> target(outputs_number);
00317 
00318    double validation_error = 0.0;
00319 
00320    for(int i = 0; i < validation_instances_number; i++)
00321    {
00322       // Input vector
00323 
00324           input = input_target_data_set_pointer->get_validation_input_instance(i);
00325 
00326       // Output vector
00327 
00328       output = multilayer_perceptron_pointer->calculate_output(input);
00329 
00330       // Target vector
00331 
00332       target = input_target_data_set_pointer->get_validation_target_instance(i);
00333 
00334       // Sum of squares error
00335 
00336       validation_error += (output - target).dot(output - target);           
00337    }
00338 
00339    return(validation_error);
00340 }
00341 
00342 
00343 // Vector<double> calculate_objective_gradient(void) method
00344 
00347 
00348 Vector<double> SumSquaredError::calculate_objective_gradient(void)
00349 {
00350    // Control sentence (if debug)
00351 
00352    #ifdef _DEBUG 
00353 
00354    if(multilayer_perceptron_pointer == NULL)
00355    {
00356       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00357                 << "double calculate_objective_gradient(void) method." << std::endl
00358                 << "Pointer to multilayer perceptron object cannot be NULL." << std::endl;
00359 
00360         exit(1);
00361    }
00362    else if(input_target_data_set_pointer == NULL)
00363    {
00364       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00365                 << "double calculate_objective_gradient(void) method." << std::endl
00366                 << "Pointer to input-target data set object cannot be NULL." << std::endl;
00367 
00368       exit(1);
00369    }
00370 
00371    #endif
00372 
00373    // Multilayer perceptron 
00374 
00375    int hidden_layers_number = multilayer_perceptron_pointer->get_hidden_layers_number();
00376 
00377    Vector<int> hidden_layers_size = multilayer_perceptron_pointer->get_hidden_layers_size();
00378 
00379    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00380 
00381    int forward_propagation_derivative_size = 2*hidden_layers_number + 2;
00382 
00383    Vector< Vector<double> > forward_propagation_derivative(forward_propagation_derivative_size);
00384 
00385    // Input-target data set
00386 
00387    int training_instances_number = input_target_data_set_pointer->get_training_instances_number();
00388 
00389    int input_variables_number = input_target_data_set_pointer->get_input_variables_number();
00390    int target_variables_number = input_target_data_set_pointer->get_target_variables_number();    Vector<double> training_input_instance(input_variables_number);
00391    Vector<double> training_target_instance(target_variables_number);
00392 
00393    // Output and hidden errors
00394 
00395    Vector<double> output_errors(outputs_number);
00396    Vector< Vector<double> > hidden_errors(hidden_layers_number);
00397 
00398    for(int h = 0; h < hidden_layers_number; h++)
00399    {
00400       hidden_errors[h].set_size(hidden_layers_size[h]);
00401    }
00402 
00403    // Hidden and output layers gradients
00404 
00405    int hidden_layers_parameters_number = multilayer_perceptron_pointer->get_hidden_layers_parameters_number();
00406    int output_layer_parameters_number = multilayer_perceptron_pointer->get_output_layer_parameters_number();
00407 
00408    Vector<double> hidden_layers_error_gradient(hidden_layers_parameters_number, 0.0);
00409    Vector<double> output_layer_error_gradient(output_layer_parameters_number, 0.0);
00410 
00411    // Main loop
00412 
00413    for(int i = 0; i < training_instances_number; i++)
00414    {
00415       training_input_instance = input_target_data_set_pointer->get_training_input_instance(i);
00416 
00417       forward_propagation_derivative = multilayer_perceptron_pointer->calculate_forward_propagation_derivative(training_input_instance);
00418 
00419       training_target_instance = input_target_data_set_pointer->get_training_target_instance(i);
00420 
00421           output_errors = calculate_output_errors(forward_propagation_derivative, training_target_instance);
00422 
00423       hidden_errors = calculate_hidden_errors(forward_propagation_derivative, output_errors);
00424 
00425       hidden_layers_error_gradient += calculate_hidden_layers_error_gradient(training_input_instance, forward_propagation_derivative, hidden_errors);
00426   
00427       output_layer_error_gradient += calculate_output_layer_error_gradient(forward_propagation_derivative, output_errors);
00428    }
00429 
00430    return(hidden_layers_error_gradient.assemble(output_layer_error_gradient));   
00431 }
00432 
00433 
00434 // Vector<double> calculate_output_errors(const Vector< Vector<double> >&, const Vector<double>&) method
00435 
00439 
00440 Vector<double> SumSquaredError::calculate_output_errors
00441 (const Vector< Vector<double> >& forward_propagation_derivative, const Vector<double>& target)
00442 {
00443    int forward_propagation_derivative_size = forward_propagation_derivative.get_size();
00444    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00445 
00446    // Control sentence (if debug)
00447 
00448    #ifdef _DEBUG 
00449 
00450    int hidden_layers_number = multilayer_perceptron_pointer->get_hidden_layers_number();
00451 
00452    if(forward_propagation_derivative_size != 2*hidden_layers_number+2)
00453    {
00454       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00455                 << "Vector<double> calculate_output_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00456                 << "Size of forward propagation derivative vector must be equal to 2*hidden_layers_number+2." 
00457                                 << std::endl;
00458 
00459         exit(1);
00460    }
00461       
00462    int output_layer_output_size = forward_propagation_derivative[forward_propagation_derivative_size-2].get_size();
00463       
00464    if(output_layer_output_size != outputs_number)
00465    {
00466       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00467                 << "Vector<double> calculate_output_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00468                 << "Size of output layer output ("<< output_layer_output_size << ") must be equal to "
00469                 << "number of outputs (" << outputs_number << ")." << std::endl;
00470 
00471       exit(1);
00472    }
00473 
00474    int output_layer_output_derivative_size 
00475    = forward_propagation_derivative[forward_propagation_derivative_size-1].get_size();
00476 
00477    if(output_layer_output_derivative_size != outputs_number)
00478    {
00479       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00480                 << "Vector<double> calculate_output_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00481                 << "Size of output layer output derivative (" << output_layer_output_derivative_size << ")must be equal to "               << "number of outputs (" << outputs_number << ")." << std::endl;
00482 
00483       exit(1);
00484    }
00485 
00486    int target_size = target.get_size();
00487 
00488    if(target_size != outputs_number)
00489    {
00490       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00491                 << "Vector<double> calculate_output_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00492                 << "Size of target must be equal to number of outputs." << std::endl;
00493 
00494       exit(1);
00495    }
00496 
00497    #endif
00498 
00499    Vector<double> output_layer_output_derivative = forward_propagation_derivative[forward_propagation_derivative_size-1];
00500    Vector<double> output_layer_output = forward_propagation_derivative[forward_propagation_derivative_size-2];
00501 
00502    Vector<double> output_errors(outputs_number);
00503 
00504    Vector<double> error(outputs_number);
00505 
00506    MultilayerPerceptron::ScalingMethod outputs_unscaling_method
00507    = multilayer_perceptron_pointer->get_outputs_unscaling_method();
00508 
00509    switch(outputs_unscaling_method)
00510    {
00511       case MultilayerPerceptron::None:
00512       {   
00513          error = output_layer_output-target;
00514 
00515          output_errors = output_layer_output_derivative*error*2.0;
00516       } 
00517       break;
00518 
00519       case MultilayerPerceptron::MeanStandardDeviation:
00520       {         
00521          Vector<double>& output_variables_standard_deviation 
00522          = multilayer_perceptron_pointer->get_output_variables_standard_deviation();
00523 
00524          // Control sentence (if debug)
00525 
00526          #ifdef _DEBUG 
00527  
00528                  int output_variables_standard_deviation_size = output_variables_standard_deviation.get_size();
00529 
00530          if(output_variables_standard_deviation_size != outputs_number)
00531          {
00532            std::cerr << "Flood Error: SumSquaredError class." << std::endl
00533                      << "Vector<double> calculate_output_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00534                      << "Size of output variables standard deviation must be equal to number of outputs." << std::endl;
00535 
00536            exit(1);
00537          }
00538 
00539          #endif
00540 
00541          output_errors = output_layer_output_derivative*error*output_variables_standard_deviation*2.0;
00542       }
00543       break;
00544 
00545       case MultilayerPerceptron::MinimumMaximum:
00546       {
00547          Vector<double>& output_variables_minimum = multilayer_perceptron_pointer->get_output_variables_minimum();
00548          Vector<double>& output_variables_maximum = multilayer_perceptron_pointer->get_output_variables_maximum();
00549 
00550          // Control sentence (if debug)
00551 
00552          #ifdef _DEBUG 
00553  
00554                  int output_variables_minimum_size = output_variables_minimum.get_size();
00555                  int output_variables_maximum_size = output_variables_maximum.get_size();
00556 
00557          if(output_variables_minimum_size != outputs_number)
00558          {
00559            std::cerr << "Flood Error: SumSquaredError class." << std::endl
00560                      << "Vector<double> calculate_output_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00561                      << "Size of output variables minimum must be equal to number of outputs." << std::endl;
00562 
00563            exit(1);
00564          }
00565          else if(output_variables_maximum_size != outputs_number)
00566          {
00567            std::cerr << "Flood Error: SumSquaredError class." << std::endl
00568                      << "Vector<double> calculate_output_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00569                      << "Size of output variables maximum must be equal to number of outputs." << std::endl;
00570 
00571            exit(1);
00572          }
00573 
00574          #endif
00575 
00576          output_errors = output_layer_output_derivative*error*(output_variables_maximum-output_variables_minimum);
00577       }          
00578       break;
00579       
00580       default:
00581       {
00582          std::cerr << "Flood Error: SumSquaredError class." << std::endl
00583                    << "Vector<double> calculate_output_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00584                    << "Unknown outputs unscaling method." << std::endl;
00585 
00586          exit(1);
00587       }          
00588       break;
00589       
00590    }
00591 
00592    return(output_errors);
00593 }
00594 
00595 
00596 // Vector< Vector<double> > calculate_hidden_errors(const Vector< Vector<double> >&, const Vector<double>&) method
00597 
00601 
00602 Vector< Vector<double> > SumSquaredError::calculate_hidden_errors
00603 (const Vector< Vector<double> >& forward_propagation_derivative, const Vector<double>& output_errors)
00604 {
00605    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00606 
00607    int hidden_layers_number = multilayer_perceptron_pointer->get_hidden_layers_number();
00608 
00609    // Control sentence (if debug)
00610 
00611    #ifdef _DEBUG 
00612 
00613    int forward_propagation_derivative_size = forward_propagation_derivative.get_size();
00614 
00615    if(forward_propagation_derivative_size != 2*hidden_layers_number+2)
00616    {
00617       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00618                 << "Vector< Vector<double> > calculate_hidden_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00619                 << "Size of forward propagation derivative vector must be equal to 2*hidden_layers_number+2." 
00620                                 << std::endl;
00621 
00622         exit(1);
00623    }
00624       
00625    int output_layer_output_size = forward_propagation_derivative[forward_propagation_derivative_size-2].get_size();
00626       
00627    if(output_layer_output_size != outputs_number)
00628    {
00629       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00630                 << "Vector< Vector<double> > calculate_hidden_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00631                 << "Size of output layer output ("<< output_layer_output_size << ") must be equal to "
00632                 << "number of outputs (" << outputs_number << ")." << std::endl;
00633 
00634       exit(1);
00635    }
00636 
00637    int output_layer_output_derivative_size 
00638    = forward_propagation_derivative[forward_propagation_derivative_size-1].get_size();
00639 
00640    if(output_layer_output_derivative_size != outputs_number)
00641    {
00642       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00643                 << "Vector< Vector<double> > calculate_hidden_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00644                 << "Size of output layer output derivative (" << output_layer_output_derivative_size << ")must be equal to " 
00645                                 << "number of outputs (" << outputs_number << ")." << std::endl;
00646 
00647       exit(1);
00648    }
00649 
00650    int output_errors_size = output_errors.get_size();
00651 
00652    if(output_errors_size != outputs_number)
00653    {
00654       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00655                 << "Vector< Vector<double> > calculate_hidden_errors(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00656                 << "Size of target must be equal to number of outputs." << std::endl;
00657 
00658       exit(1);
00659    }
00660 
00661    #endif
00662 
00663    // Set hidden errors vector of vectors
00664 
00665    Vector<int> hidden_layers_size = multilayer_perceptron_pointer->get_hidden_layers_size();
00666 
00667    Vector< Vector<double> > hidden_errors(hidden_layers_number);
00668 
00669    for(int h = 0; h < hidden_layers_number; h++)
00670    {
00671       hidden_errors[h].set_size(hidden_layers_size[h]);
00672    }
00673 
00674    // Multilayer perceptron
00675 
00676    Vector<Perceptron>& output_layer = multilayer_perceptron_pointer->get_output_layer();
00677    Vector< Vector<Perceptron> >& hidden_layers = multilayer_perceptron_pointer->get_hidden_layers();
00678 
00679    Vector< Vector<double> > hidden_layers_output_derivative(hidden_layers_number);
00680 
00681    for(int i = 0; i < hidden_layers_number; i++)
00682    {
00683       hidden_layers_output_derivative[i] = forward_propagation_derivative[1+2*i];
00684    }
00685 
00686    Vector<double> synaptic_weights;
00687 
00688    double sum;
00689     
00690    // Last hidden layer
00691 
00692    for(int j = 0; j < hidden_layers_size[hidden_layers_number-1]; j++)
00693    {
00694       sum = 0.0;
00695 
00696       for(int k = 0; k < outputs_number; k++)
00697       {
00698          synaptic_weights = output_layer[k].get_synaptic_weights();
00699 
00700          sum += (synaptic_weights[j])*output_errors[k];
00701       }
00702 
00703       hidden_errors[hidden_layers_number-1][j] = hidden_layers_output_derivative[hidden_layers_number-1][j]*sum;
00704    }
00705 
00706    // Rest of hidden layers
00707 
00708    for(int h = hidden_layers_number-2; h >= 0; h--) 
00709    {   
00710       for(int j = 0; j < hidden_layers_size[h]; j++)
00711       {
00712          sum = 0.0;
00713 
00714          for(int k = 0; k < hidden_layers_size[h+1]; k++)
00715          {
00716             synaptic_weights = hidden_layers[h+1][k].get_synaptic_weights();
00717 
00718             sum += (synaptic_weights[j])*hidden_errors[h+1][k];
00719          }               
00720 
00721          hidden_errors[h][j] = hidden_layers_output_derivative[h][j]*sum;
00722       }
00723    }
00724 
00725    return(hidden_errors);
00726 }
00727 
00728 
00729 // Vector<double> calculate_hidden_layers_error_gradient
00730 // (const Vector<double>&, const Vector< Vector<double> >&, const Vector< Vector<double> >&) ) method
00731 
00736 
00737 Vector<double> SumSquaredError::calculate_hidden_layers_error_gradient
00738 (const Vector<double>& input, 
00739  const Vector< Vector<double> >& forward_propagation_derivative, 
00740  const Vector< Vector<double> >& hidden_errors)
00741 {
00742    int inputs_number = multilayer_perceptron_pointer->get_inputs_number();
00743    int hidden_layers_number = multilayer_perceptron_pointer->get_hidden_layers_number();
00744 
00745    // Control sentence (if debug)
00746 
00747    #ifdef _DEBUG 
00748 
00749    int input_size = input.get_size();
00750 
00751    if(input_size != inputs_number)
00752    {
00753       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00754                 << "Vector< Vector<double> > calculate_hidden_layers_error_gradient(const Vector< Vector<double> >&, const Vector<double>&, const Vector<double>&) method." << std::endl
00755                 << "Size of input (" << input_size << ") must be equal to inputs number (" << inputs_number << ")." 
00756                                 << std::endl;
00757 
00758         exit(1);
00759    }
00760 
00761 
00762    int forward_propagation_derivative_size = forward_propagation_derivative.get_size();
00763 
00764    if(forward_propagation_derivative_size != 2*hidden_layers_number+2)
00765    {
00766       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00767                 << "Vector< Vector<double> > calculate_hidden_layers_error_gradient(const Vector< Vector<double> >&, const Vector<double>&, const Vector<double>&) method." << std::endl
00768                 << "Size of forward propagation derivative (" << forward_propagation_derivative_size << ") must be equal to 2*hidden_layers_number+2 (" << 2*hidden_layers_number+2 << ")." 
00769                                 << std::endl;
00770 
00771         exit(1);
00772    }
00773       
00774    int hidden_errors_size = hidden_errors.get_size();
00775       
00776    if(hidden_errors_size != hidden_layers_number)
00777    {
00778       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00779                 << "Vector< Vector<double> > calculate_hidden_layers_error_gradient(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00780                 << "Size of output errors ("<< hidden_errors_size << ") must be equal to number of hidden layers (" << hidden_layers_number << ")." << std::endl;
00781 
00782       exit(1);
00783    }
00784 
00785    #endif
00786 
00787    // Multilayer perceptron stuff 
00788 
00789    Vector<int> hidden_layers_size = multilayer_perceptron_pointer->get_hidden_layers_size();
00790 
00791    int hidden_layers_parameters_number = multilayer_perceptron_pointer->get_hidden_layers_parameters_number();
00792 
00793    Vector< Vector<Perceptron> >& hidden_layers = multilayer_perceptron_pointer->get_hidden_layers();
00794 
00795    Vector<double> synaptic_weights;
00796 
00797    Vector< Vector<double> > hidden_layers_output(hidden_layers_number);
00798 
00799    for(int i = 0; i < hidden_layers_number; i++)
00800    {
00801       hidden_layers_output[i] = forward_propagation_derivative[2*i];
00802    }
00803 
00804    int index = 0;
00805 
00806    Vector<double> hidden_layers_error_gradient(hidden_layers_parameters_number, 0.0);
00807 
00808    // First hidden layer
00809 
00810    for(int j = 0; j < hidden_layers_size[0]; j++)
00811    {
00812       // Bias
00813 
00814       hidden_layers_error_gradient[index] += hidden_errors[0][j];
00815       index++;
00816 
00817       // Synaptic weights
00818 
00819       synaptic_weights = hidden_layers[0][j].get_synaptic_weights();
00820 
00821       for(int k = 0; k < inputs_number; k++)
00822       {
00823          hidden_layers_error_gradient[index] += hidden_errors[0][j]*input[k];
00824          index++;   
00825       }
00826    }
00827 
00828    // Rest of hidden layers     
00829     
00830    for(int h = 1; h < hidden_layers_number; h++)
00831    {      
00832       for(int j = 0; j < hidden_layers_size[h]; j++)
00833       {
00834          // Bias
00835 
00836          hidden_layers_error_gradient[index] += hidden_errors[h][j];
00837          index++;
00838 
00839          // Synaptic weights
00840 
00841          synaptic_weights = hidden_layers[h][j].get_synaptic_weights();
00842 
00843          for(int k = 0; k < hidden_layers_size[h-1]; k++)
00844          {
00845             hidden_layers_error_gradient[index] += hidden_errors[h][j]*hidden_layers_output[h-1][k];
00846             index++;   
00847          }
00848       }
00849    }
00850 
00851    return(hidden_layers_error_gradient);
00852 }
00853 
00854 
00855 // Vector<double> calculate_output_layer_error_gradient(const Vector< Vector<double> >&, const Vector<double>&) method
00856 
00860 
00861 Vector<double> SumSquaredError::calculate_output_layer_error_gradient
00862 (const Vector< Vector<double> >& forward_propagation_derivative, const Vector<double>& output_errors)
00863 {
00864    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00865    int hidden_layers_number = multilayer_perceptron_pointer->get_hidden_layers_number();
00866 
00867    // Control sentence (if debug)
00868 
00869    #ifdef _DEBUG 
00870 
00871    int forward_propagation_derivative_size = forward_propagation_derivative.get_size();
00872 
00873    if(forward_propagation_derivative_size != 2*hidden_layers_number+2)
00874    {
00875       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00876                 << "Vector< Vector<double> > calculate_output_layer_error_gradient(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00877                 << "Size of forward propagation derivative (" << forward_propagation_derivative_size << ") must be equal to 2*hidden_layers_number+2 (" << 2*hidden_layers_number+2 << ")." << std::endl;
00878 
00879         exit(1);
00880    }
00881       
00882    int output_errors_size = output_errors.get_size();
00883       
00884    if(output_errors_size != outputs_number)
00885    {
00886       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00887                 << "Vector< Vector<double> > calculate_output_layer_error_gradient(const Vector< Vector<double> >&, const Vector<double>&) method." << std::endl
00888                 << "Size of output errors ("<< output_errors_size << ") must be equal to number of outputs (" << outputs_number << ")." << std::endl;
00889 
00890       exit(1);
00891    }
00892 
00893    #endif
00894 
00895    // Multilayer perceptron stuff
00896 
00897    Vector<int> hidden_layers_size = multilayer_perceptron_pointer->get_hidden_layers_size();
00898 
00899    int output_layer_parameters_number = multilayer_perceptron_pointer->get_output_layer_parameters_number();
00900 
00901    Vector< Vector<double> > hidden_layers_output(hidden_layers_number);
00902 
00903    for(int i = 0; i < hidden_layers_number; i++)
00904    {
00905       hidden_layers_output[i] = forward_propagation_derivative[2*i];
00906    }
00907 
00908    // Calculate gradient elements of output neurons
00909 
00910    Vector<double> output_layer_error_gradient(output_layer_parameters_number, 0.0);
00911 
00912    int index = 0;
00913 
00914    for(int j = 0; j < outputs_number; j++)
00915    {
00916       // Bias
00917 
00918       output_layer_error_gradient[index] += output_errors[j];
00919       index++;
00920 
00921       // Synaptic weights
00922 
00923       for(int k = 0; k < hidden_layers_size[hidden_layers_number-1]; k++)
00924       {
00925          output_layer_error_gradient[index] = hidden_layers_output[hidden_layers_number-1][k]*output_errors[j];
00926          index++;
00927       }
00928    }
00929 
00930    return(output_layer_error_gradient);
00931 }
00932 
00933 
00934 // Matrix<double> calculate_Jacobian(void) method
00935 
00939 
00940 Matrix<double> SumSquaredError::calculate_Jacobian(void)
00941 {
00942    // Control sentence (if debug)
00943 
00944    #ifdef _DEBUG 
00945 
00946    if(multilayer_perceptron_pointer == NULL)
00947    {
00948       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00949                 << "double calculate_Jacobian(void) method." << std::endl
00950                 << "Pointer to multilayer perceptron object cannot be NULL." << std::endl;
00951 
00952         exit(1);
00953    }
00954    else if(input_target_data_set_pointer == NULL)
00955    {
00956       std::cerr << "Flood Error: SumSquaredError class." << std::endl
00957                 << "double calculate_Jacobian(void) method." << std::endl
00958                 << "Pointer to input-target data set object cannot be NULL." << std::endl;
00959 
00960       exit(1);
00961    }
00962 
00963    #endif
00964 
00965    int training_instances_number = input_target_data_set_pointer->get_training_instances_number();
00966    int neural_parameters_number = multilayer_perceptron_pointer->get_neural_parameters_number();
00967 
00968    Matrix<double> Jacobian(training_instances_number, neural_parameters_number);
00969 
00970    // Multilayer perceptron 
00971 
00972    int hidden_layers_number = multilayer_perceptron_pointer->get_hidden_layers_number();
00973 
00974    Vector<int> hidden_layers_size = multilayer_perceptron_pointer->get_hidden_layers_size();
00975 
00976    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
00977 
00978    int forward_propagation_derivative_size = 2*hidden_layers_number + 2;
00979 
00980    Vector< Vector<double> > forward_propagation_derivative(forward_propagation_derivative_size);
00981 
00982    // Input-target data set
00983 
00984    int target_variables_number = input_target_data_set_pointer->get_target_variables_number();    Vector<double> training_input_instance(target_variables_number);
00985    Vector<double> training_target_instance(target_variables_number);
00986 
00987    // Output and hidden errors
00988 
00989    Vector<double> output_errors(outputs_number);
00990    Vector< Vector<double> > hidden_errors(hidden_layers_number);
00991 
00992    for(int h = 0; h < hidden_layers_number; h++)
00993    {
00994       hidden_errors[h].set_size(hidden_layers_size[h]);
00995    }
00996 
00997    // Hidden and output layers gradients
00998 
00999    int hidden_layers_parameters_number = multilayer_perceptron_pointer->get_hidden_layers_parameters_number();
01000    int output_layer_parameters_number = multilayer_perceptron_pointer->get_output_layer_parameters_number();
01001 
01002    Vector<double> hidden_layers_error_gradient(hidden_layers_parameters_number, 0.0);
01003    Vector<double> output_layer_error_gradient(output_layer_parameters_number, 0.0);
01004 
01005    Vector<double> gradient(outputs_number);
01006 
01007    // Main loop
01008 
01009    for(int i = 0; i < training_instances_number; i++)
01010    {
01011       training_input_instance = input_target_data_set_pointer->get_training_input_instance(i);
01012 
01013       forward_propagation_derivative = multilayer_perceptron_pointer->calculate_forward_propagation_derivative(training_input_instance);
01014 
01015       training_target_instance = input_target_data_set_pointer->get_training_target_instance(i);
01016 
01017       output_errors = calculate_output_errors(forward_propagation_derivative, training_target_instance);
01018 
01019       hidden_errors = calculate_hidden_errors(forward_propagation_derivative, output_errors);
01020 
01021       hidden_layers_error_gradient = calculate_hidden_layers_error_gradient(training_input_instance, forward_propagation_derivative, hidden_errors);
01022 
01023       output_layer_error_gradient = calculate_output_layer_error_gradient(forward_propagation_derivative, output_errors);
01024 
01025       gradient = hidden_layers_error_gradient.assemble(output_layer_error_gradient);
01026 
01027       Jacobian.set_row(i, gradient);
01028    }
01029 
01030    return(Jacobian);
01031 }
01032 
01033 
01034 // Matrix<double> calculate_Jacobian_numerical_differentiation(void) method
01035 
01038 
01039 Matrix<double> SumSquaredError::calculate_Jacobian_numerical_differentiation(void)
01040 {
01041    // Control sentence (if debug)
01042 
01043    #ifdef _DEBUG 
01044 
01045    if(multilayer_perceptron_pointer == NULL)
01046    {
01047       std::cerr << "Flood Error: SumSquaredError class." << std::endl
01048                 << "double calculate_Jacobian_numerical_differentiation(void) method." << std::endl
01049                 << "Pointer to multilayer perceptron object cannot be NULL." << std::endl;
01050 
01051         exit(1);
01052    }
01053    else if(input_target_data_set_pointer == NULL)
01054    {
01055       std::cerr << "Flood Error: SumSquaredError class." << std::endl
01056                 << "double calculate_Jacobian_numerical_differentiation(void) method." << std::endl
01057                 << "Pointer to input-target data set object cannot be NULL." << std::endl;
01058 
01059       exit(1);
01060    }
01061 
01062    #endif
01063 
01064    switch(numerical_differentiation_method)   
01065    {
01066       case ForwardDifferences:
01067       {
01068          return(calculate_Jacobian_forward_differences());
01069       }
01070       break;
01071 
01072       case CentralDifferences:
01073       {
01074          return(calculate_Jacobian_central_differences());
01075       }
01076       break;
01077 
01078       default:
01079       {
01080          std::cerr << "Flood Error: SumSquaredError class." << std::endl
01081                    << "std::string calculate_Jacobian_numerical_differentiation(void) method." << std::endl
01082                    << "Unknown numerical differentiation method." << std::endl;
01083  
01084          exit(1);
01085       }
01086       break;
01087    }
01088 }
01089 
01090 
01091 // Matrix<double> calculate_Jacobian_forward_differences(void) method
01092 
01095 
01096 Matrix<double> SumSquaredError::calculate_Jacobian_forward_differences(void)
01097 {
01098    int training_instances_number = input_target_data_set_pointer->get_training_instances_number();
01099    int neural_parameters_number = multilayer_perceptron_pointer->get_neural_parameters_number();
01100 
01101    Matrix<double> Jacobian(training_instances_number, neural_parameters_number);
01102 
01103    // Multilayer perceptron stuff
01104 
01105    Vector<double> neural_parameters = multilayer_perceptron_pointer->get_neural_parameters();
01106 
01107    // Objective functional stuff
01108 
01109    Vector<double> squared_errors = calculate_squared_errors(); 
01110 
01111    Vector<double> squared_errors_forward(training_instances_number);
01112 
01113    double actual_epsilon;
01114 
01115    // Calculate Jacobian
01116 
01117    for(int j = 0; j < neural_parameters_number; j++)
01118    {
01119       actual_epsilon = calculate_actual_epsilon(neural_parameters[j]);
01120 
01121       // Perturbate neural parameters
01122 
01123       neural_parameters[j] += actual_epsilon;
01124       multilayer_perceptron_pointer->set_neural_parameters(neural_parameters);
01125 
01126       // Calculate squared errors forward
01127 
01128       squared_errors_forward = calculate_squared_errors();
01129 
01130       // Restart biases and synaptic weights
01131 
01132       neural_parameters[j] -= actual_epsilon;
01133       multilayer_perceptron_pointer->set_neural_parameters(neural_parameters);
01134 
01135       // Calculate Jacobian elements
01136                         
01137       for(int i = 0; i < training_instances_number; i++)
01138       {
01139          Jacobian[i][j] = (squared_errors_forward[i] - squared_errors[i])/actual_epsilon;
01140       }
01141    }
01142 
01143    return(Jacobian);
01144 }
01145 
01146 
01147 // Matrix<double> calculate_Jacobian_central_differences(void) method
01148 
01151 
01152 Matrix<double> SumSquaredError::calculate_Jacobian_central_differences(void)
01153 {
01154    int training_instances_number = input_target_data_set_pointer->get_training_instances_number();
01155    int neural_parameters_number = multilayer_perceptron_pointer->get_neural_parameters_number();
01156 
01157    Matrix<double> Jacobian(training_instances_number, neural_parameters_number);
01158 
01159    // Multilayer perceptron stuff
01160 
01161    Vector<double> neural_parameters = multilayer_perceptron_pointer->get_neural_parameters();
01162 
01163    // Objective functional stuff
01164    
01165    Vector<double> squared_errors_forward(training_instances_number);
01166    Vector<double> squared_errors_backward(training_instances_number);
01167 
01168    double actual_epsilon;
01169 
01170    // Calculate Jacobian
01171 
01172    for(int j = 0; j < neural_parameters_number; j++)
01173    {
01174       actual_epsilon = calculate_actual_epsilon(neural_parameters[j]);
01175       
01176       // Perturbate neural parameters
01177 
01178       neural_parameters[j] += actual_epsilon;
01179       multilayer_perceptron_pointer->set_neural_parameters(neural_parameters);
01180 
01181       // Calculate squared errors     
01182  
01183       squared_errors_forward = calculate_squared_errors();
01184  
01185       // Restart neural parameters
01186 
01187       neural_parameters[j] -= actual_epsilon;
01188       multilayer_perceptron_pointer->set_neural_parameters(neural_parameters);
01189 
01190       // Perturbate neural parameters
01191 
01192       neural_parameters[j] -= actual_epsilon;
01193       multilayer_perceptron_pointer->set_neural_parameters(neural_parameters);
01194 
01195       // Calculate squared errors     
01196  
01197       squared_errors_backward = calculate_squared_errors();
01198  
01199       // Restart neural parameters
01200 
01201       neural_parameters[j] += actual_epsilon;
01202       multilayer_perceptron_pointer->set_neural_parameters(neural_parameters);
01203 
01204       // Calculate Jacobian elements
01205 
01206       for(int i = 0; i < training_instances_number; i++)
01207       {
01208          Jacobian[i][j] = (squared_errors_forward[i] - squared_errors_backward[i])/(2.0*actual_epsilon);
01209       }
01210    }
01211 
01212    return(Jacobian);
01213 }
01214 
01215 
01216 // Vector<double> calculate_squared_errors(void) method
01217 
01219 
01220 Vector<double> SumSquaredError::calculate_squared_errors(void)
01221 {
01222    // Control sentence (if debug)
01223 
01224    #ifdef _DEBUG 
01225 
01226    if(multilayer_perceptron_pointer == NULL)
01227    {
01228       std::cerr << "Flood Error: SumSquaredError class." << std::endl
01229                 << "double calculate_Jacobian_numerical_differentiation(void) method." << std::endl
01230                 << "Pointer to multilayer perceptron object cannot be NULL." << std::endl;
01231 
01232         exit(1);
01233    }
01234    else if(input_target_data_set_pointer == NULL)
01235    {
01236       std::cerr << "Flood Error: SumSquaredError class." << std::endl
01237                 << "double calculate_Jacobian_numerical_differentiation(void) method." << std::endl
01238                 << "Pointer to input-target data set object cannot be NULL." << std::endl;
01239 
01240       exit(1);
01241    }
01242 
01243    #endif
01244 
01245    int inputs_number = multilayer_perceptron_pointer->get_inputs_number();
01246    int outputs_number = multilayer_perceptron_pointer->get_outputs_number();
01247 
01248    int training_instances_number = input_target_data_set_pointer->get_training_instances_number();
01249 
01250    Vector<double> squared_errors(training_instances_number);
01251 
01252    Vector<double> input(inputs_number);
01253    Vector<double> output(outputs_number);
01254    Vector<double> target(outputs_number);
01255    Vector<double> instance_error(outputs_number);
01256 
01257    for(int i = 0; i < training_instances_number; i++)
01258    {
01259       // Input vector
01260 
01261           input = input_target_data_set_pointer->get_training_input_instance(i);
01262 
01263       // Output vector
01264 
01265       output = multilayer_perceptron_pointer->calculate_output(input);
01266 
01267       // Target vector
01268 
01269       target = input_target_data_set_pointer->get_training_target_instance(i);
01270 
01271       // Error
01272 
01273           squared_errors[i] = (output - target).dot(output - target);
01274    }
01275 
01276    return(squared_errors);
01277 }
01278 
01279 }
01280 
01281 // Flood: An Open Source Neural Networks C++ Library.
01282 // Copyright (C) 2005-2010 Roberto Lopez 
01283 //
01284 // This library is free software; you can redistribute it and/or
01285 // modify it under the s of the GNU Lesser General Public
01286 // License as published by the Free Software Foundation; either
01287 // version 2.1 of the License, or any later version.
01288 //
01289 // This library is distributed in the hope that it will be useful,
01290 // but WITHOUT ANY WARRANTY; without even the implied warranty of
01291 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
01292 // Lesser General Public License for more details.
01293 // You should have received a copy of the GNU Lesser General Public
01294 // License along with this library; if not, write to the Free Software
01295 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
01296