Club Amiga de Montreal - CAM/CAM_CD_1.iso/files/624a.lha/Neural_Network_v2.0/neural_network.cc

C/C++ Source or Header | 1992-03-28 | 23.9 KB | 816 lines

#include <iostream.h>
#include <fstream.h>
#include <new.h>
#include <math.h>
#include "Neural_network.h"

void Neural_network::allocate_matrices ()
{

  // Time to allocate the entire neural_net structure
  // Activation matrices
  hidden1_act = new double [num_hidden1];
  hidden2_act = new double [num_hidden2];
  output_act = new double [num_outputs];

  // Weight matrices
  input_weights = new double [num_hidden1 * num_inputs];
  hidden1_weights = new double [num_hidden2 * num_hidden1];
  hidden2_weights = new double [num_outputs * num_hidden2];

  // Learning rate matrices for each weight's learning rate.
  // Needed for delta-bar-delta algorithm
  input_learning_rate = new double [num_hidden1 * num_inputs];
  hidden1_learning_rate = new double [num_hidden2 * num_hidden1];
  hidden2_learning_rate = new double [num_outputs * num_hidden2];

  // Learning rate deltas for each weight's learning rate.
  // Needed for delta-bar-delta algorithm.
  input_learning_delta = new double [num_hidden1 * num_inputs];
  hidden1_learning_delta = new double [num_hidden2 * num_hidden1];
  hidden2_learning_delta = new double [num_outputs * num_hidden2];

  // Delta bar matrices for each weight's delta bar.
  // Needed for delta-bar-delta algorithm.
  input_learning_delta_bar = new double [num_hidden1 * num_inputs];
  hidden1_learning_delta_bar = new double [num_hidden2 * num_hidden1];
  hidden2_learning_delta_bar = new double [num_outputs * num_hidden2];

  // Weight delta matrices for each weights delta.
  // Needed for BackPropagation algorithm.
  input_weights_sum_delta = new double [num_hidden1 * num_inputs];
  hidden1_weights_sum_delta = new double [num_hidden2 * num_hidden1];
  hidden2_weights_sum_delta = new double [num_outputs * num_hidden2];

  // Sum of delta * weight matrices for each weight.
  // Needed for BackPropagation algorithm.
  hidden1_sum_delta_weight = new double [num_hidden2 * num_hidden1];
  hidden2_sum_delta_weight = new double [num_outputs * num_hidden2];

  /* Done neural net allocation */
}

void Neural_network::initialize_matrices (double range)
{
  int    x,y;
  double rand_max;

  training_examples = 0;
  examples_since_update = 0;

  rand_max = RAND_MAX;
  /* Initialize all weights from -range to +range randomly */
  for (x = 0; x < num_hidden1; ++x)
    {
      hidden1_sum_delta_weight [x] = 0.0;

      for (y = 0; y < num_inputs; ++y)
        {
          input_weights [x * num_inputs + y] = rand () / rand_max * range;
          if ( rand () < (RAND_MAX / 2) )
              input_weights [x * num_inputs + y] *= -1.0;

          input_weights_sum_delta [x * num_inputs + y] = 0.0;
          input_learning_rate [x * num_inputs + y] = learning_rate;
          input_learning_delta [x * num_inputs + y] = 0.0;
          input_learning_delta_bar [x * num_inputs + y] = 0.0;
        }
    }

  for (x = 0; x < num_hidden2; ++x)
    {
      hidden2_sum_delta_weight [x] = 0.0;

      for (y = 0; y < num_hidden1; ++y)
        {
          hidden1_weights [x * num_hidden1 + y] = rand () / rand_max * range;
          if ( rand () < (RAND_MAX / 2) )
              hidden1_weights [x * num_hidden1 + y] *= -1.0;

          hidden1_weights_sum_delta [x * num_hidden1 + y] = 0.0;
          hidden1_learning_rate [x * num_hidden1 + y] = learning_rate;
          hidden1_learning_delta [x * num_hidden1 + y] = 0.0;
          hidden1_learning_delta_bar [x * num_hidden1 + y] = 0.0;
        }
    }

  for (x = 0; x < num_outputs; ++x)
    {
      for (y = 0; y < num_hidden2; ++y)
        {
          hidden2_weights [x * num_hidden2 + y] = rand () / rand_max * range;
          if ( rand () < (RAND_MAX / 2) )
              hidden2_weights [x * num_hidden2 + y] *= -1.0;

          hidden2_weights_sum_delta [x * num_hidden2 + y] = 0.0;
          hidden2_learning_rate [x * num_hidden2 + y] = learning_rate;
          hidden2_learning_delta [x * num_hidden2 + y] = 0.0;
          hidden2_learning_delta_bar [x * num_hidden2 + y] = 0.0;
        }
    }

}

void Neural_network::deallocate_matrices ()
{

  // Time to destroy the entire neural_net structure
  // Activation matrices
  delete hidden1_act;
  delete hidden2_act;
  delete output_act;

  // Weight matrices
  delete input_weights;
  delete hidden1_weights;
  delete hidden2_weights;

  // Learning rate matrices for each weight's learning rate.
  // Needed for delta-bar-delta algorithm
  delete input_learning_rate;
  delete hidden1_learning_rate;
  delete hidden2_learning_rate;

  // Learning rate deltas for each weight's learning rate.
  // Needed for delta-bar-delta algorithm.
  delete input_learning_delta;
  delete hidden1_learning_delta;
  delete hidden2_learning_delta;

  // Delta bar matrices for each weight's delta bar.
  // Needed for delta-bar-delta algorithm.
  delete input_learning_delta_bar;
  delete hidden1_learning_delta_bar;
  delete hidden2_learning_delta_bar;

  // Weight delta matrices for each weights delta.
  // Needed for BackPropagation algorithm.
  delete input_weights_sum_delta;
  delete hidden1_weights_sum_delta;
  delete hidden2_weights_sum_delta;

  // Sum of delta * weight matrices for each weight.
  // Needed for BackPropagation algorithm.
  delete hidden1_sum_delta_weight;
  delete hidden2_sum_delta_weight;

  /* Done neural net deallocation */
}

Neural_network::Neural_network (int number_inputs, int number_hidden1,
                                int number_hidden2, int number_outputs,
                                double t_epsilon, double t_skip_epsilon,
                                double t_learning_rate, double t_theta,
                                double t_phi, double t_K, double range) :
                num_inputs (number_inputs), num_hidden1 (number_hidden1),
                num_hidden2 (number_hidden2), num_outputs (number_outputs),
                epsilon (t_epsilon), skip_epsilon (t_skip_epsilon),
                learning_rate (t_learning_rate), theta (t_theta), phi (t_phi),
                K (t_K), training_examples (0), examples_since_update (0)
{
  allocate_matrices ();
  initialize_matrices (range);
}


Neural_network::Neural_network (String& filename, int& file_error,
                                double t_epsilon, double t_skip_epsilon,
                                double t_learning_rate, double t_theta,
                                double t_phi, double t_K) :
                epsilon (t_epsilon), skip_epsilon (t_skip_epsilon),
                learning_rate (t_learning_rate), theta (t_theta), phi (t_phi),
                K (t_K), examples_since_update (0),
                hidden1_act (0), hidden2_act (0), output_act (0),
                input_weights (0), hidden1_weights (0), hidden2_weights (0),
                input_learning_rate (0), hidden1_learning_rate (0),
                hidden2_learning_rate (0), input_learning_delta (0),
                hidden1_learning_delta (0), hidden2_learning_delta (0),
                input_learning_delta_bar (0), hidden1_learning_delta_bar (0),
                hidden2_learning_delta_bar (0), input_weights_sum_delta (0),
                hidden1_weights_sum_delta (0), hidden2_weights_sum_delta (0),
                hidden1_sum_delta_weight (0), hidden2_sum_delta_weight (0)
{
  file_error = read_weights (filename);
}

int Neural_network::read_weights (String& filename)
{
  ifstream fp;
  int      x;
  long     iter;

  fp.open (filename);
  if ( fp.fail() != 0 )
    {
      cout << "Could not read weights from file " << filename << "\n";
      return (-1);
    }

  /* First read in how many iterations have been performed so far */
  fp >> iter;
  cout << "Iterations = " << iter << "\n";

  /* Next read in how many input nodes, hidden1 nodes, hidden2 nodes */
  /* and output nodes. */
  fp >> num_inputs >> num_hidden1 >> num_hidden2 >> num_outputs;

  // Deallocate previous matrices
  deallocate_matrices ();

  /* Allocate new matrices with new size */
  allocate_matrices ();

  // Initialize all matrices and variables
  initialize_matrices (1.0);
  training_examples = iter;

  /* Read input->hidden1 weights from file. */
  for (x = 0; x < num_inputs * num_hidden1; ++x)
    {
      fp >> input_weights [x];
    }

  /* Read hidden1->hidden2 weights from file. */
  for (x = 0; x < num_hidden1 * num_hidden2; ++x)
    {
      fp >> hidden1_weights [x];
    }

  /* Read hidden2->output weights from file. */
  for (x = 0; x < (num_hidden2 * num_outputs); ++x)
    {
      fp >> hidden2_weights [x];
    }

  fp.close ();

  /* Now all the weights have been loaded */
 return (0);
}



int Neural_network::save_weights (String& filename)
{
  ofstream fp;
  int      x;

  fp.open (filename);
  if ( fp.fail() != 0 )
    {
      cout << "Could not save weights to file " << filename << "\n";
      return (-1);
    }

  /* First write out how many iterations have been performed so far */
  fp << training_examples << "\n";

  /* Next write out how many input nodes, hidden1 nodes, hidden2 nodes */
  /* and output nodes. */
  fp << num_inputs << " " << num_hidden1 << " " << num_hidden2 << " ";
  fp << num_outputs << "\n";

  fp.precision (6);
  /* Write input->hidden1 weights to output. */
  for (x = 0; x < num_inputs * num_hidden1; ++x)
    {
      fp.width (10);
      fp << input_weights [x] << " ";
      if ( (x % 5) == 4 )
          fp << "\n";
    }
  fp << "\n\n";

  /* Write hidden1->hidden2 weights to output. */
  for (x = 0; x < num_hidden1 * num_hidden2; ++x)
    {
      fp.width (10);
      fp << hidden1_weights [x] << " ";
      if ( (x % 5) == 4 )
          fp << "\n";
    }
  fp << "\n\n";

  /* Write hidden2->output weights to output. */
  for (x = 0; x < (num_hidden2 * num_outputs); ++x)
    {
      fp.width (10);
      fp << hidden2_weights [x] << " ";
      if ( (x % 5) == 4 )
          fp << "\n";
    }
  fp << "\n\n";

  fp.close ();
  cout << "Closed file\n";

  /* Now all the weights have been saved */
  return (0);
}



void Neural_network::set_size_parameters (int number_inputs,
                                 int number_hidden1, int number_hidden2,
                                 int number_outputs, double range)
{
  double *new_input_weights,*new_hidden1_weights,*new_hidden2_weights;
  double rand_max;
  int    x;

  rand_max = RAND_MAX;

  // Allocate new weight matrices with new size
  new_input_weights = new double [number_hidden1 * number_inputs];
  new_hidden1_weights = new double [number_hidden2 * number_hidden1];
  new_hidden2_weights = new double [number_outputs * number_hidden2];

  // Copy over all weights
  // Input weights
  for (x = 0; x < number_hidden1 * number_inputs; ++x)
    {
      // IF the new size is larger than the old size, THEN make new connections
      // a random weight between +-range.
      if ( x >= (num_hidden1 * num_inputs) )
        {
          new_input_weights [x] = rand () / rand_max * range;
          if ( rand () < (RAND_MAX / 2) )
              new_input_weights [x] *= -1.0;
        }
      else
          new_input_weights [x] = input_weights [x];
    }

  // Hidden1 weights
  for (x = 0; x < number_hidden2 * number_hidden1; ++x)
    {
      // IF the new size is larger than the old size, THEN make new connections
      // a random weight between +-range.
      if ( x >= (num_hidden2 * num_hidden1) )
        {
          new_hidden1_weights [x] = rand () / rand_max * range;
          if ( rand () < (RAND_MAX / 2) )
              new_hidden1_weights [x] *= -1.0;
        }
      else
          new_hidden1_weights [x] = hidden1_weights [x];
    }

  // Hidden2 weights
  for (x = 0; x < number_outputs * number_hidden2; ++x)
    {
      // IF the new size is larger than the old size, THEN make new connections
      // a random weight between +-range.
      if ( x >= (num_outputs * num_hidden2) )
        {
          new_hidden2_weights [x] = rand () / rand_max * range;
          if ( rand () < (RAND_MAX / 2) )
              new_hidden2_weights [x] *= -1.0;
        }
      else
          new_hidden2_weights [x] = hidden2_weights [x];
    }

  // All weights have been copied.

  // Change size paramters
  num_inputs = number_inputs;
  num_hidden1 = number_hidden1;
  num_hidden2 = number_hidden2;
  num_outputs = number_outputs;

  // Deallocate all matrices
  deallocate_matrices ();

  // Allocate new nerual network matrices with the correct size and initialize
  allocate_matrices ();
  initialize_matrices (1.0);

  // Now deallocate new randomly initialized weight matrices and assign them
  // to the new weight matrices that have the correct weight values.
  delete input_weights;
  delete hidden1_weights;
  delete hidden2_weights;

  input_weights = new_input_weights;
  hidden1_weights = new_hidden1_weights;
  hidden2_weights = new_hidden2_weights;

}


void Neural_network::back_propagation (double input [],
                                       double desired_output [],
                                       int& done)
{
  int     x,y;
  int     size;
  double  delta,sum_delta,*weight,*p_sum_delta,*p_learning_delta;

  /* First check if training complete. */
  for (x = num_outputs - 1; x >= 0; --x)
    {
      if ( fabs (desired_output [x] - output_act [x]) > epsilon )
        {
          done = 0;
        }
    }

  /* Go backward through list for speed */
  size = num_hidden2;
  /* First calculate deltas of weights from output to hidden layer 2. */
  for (x = num_outputs - 1; x >= 0; --x)
    {
      weight = &hidden2_weights [x * size];
      p_sum_delta = &hidden2_weights_sum_delta [x * size];
      p_learning_delta = &hidden2_learning_delta [x * size];
      for (y = num_hidden2 - 1; y >= 0; --y)
        {
          /* Formula delta = (desired - actual) * derivative
             derivative = S(1 - S)
             Also calculate sum of deltas * weight for next layer.
          */
          delta = (desired_output [x] - output_act [x])
                   * SLOPE * output_act [x] * (1.0 - output_act [x]);

          hidden2_sum_delta_weight [y] += delta * weight [y];

          p_learning_delta [y] += delta;

          /* Now multiply by activation and sum in weights sum delta */
          p_sum_delta [y] += delta * hidden2_act [y];
        }
    }


  /* Next calculate deltas of weights between hidden layer2 and hidden
     layer 1 */
  size = num_hidden1;
  for (x = num_hidden2 - 1; x >= 0; --x)
    {
      sum_delta = hidden2_sum_delta_weight [x];
      weight = &hidden1_weights [x * size];
      p_sum_delta = &hidden1_weights_sum_delta [x * size];
      p_learning_delta = &hidden1_learning_delta [x * size];
      for (y = num_hidden1 - 1; y >= 0; --y)
        {
          /* Formula delta = SUM (previous deltas*weight)
                             * derivative
             previous deltas already muliplied by weight.
             derivative = S(1 - S)

             Also calculate sum of deltas * weight to save from doing
             it for next layer.
          */

          delta = sum_delta * hidden2_act [x] *
                  (1.0 - hidden2_act [x]) * SLOPE;

          hidden1_sum_delta_weight [y] += delta * weight [y];

          p_learning_delta [y] += delta;

          /* Now multiply by activation and sum in weights_sum_delta */
          p_sum_delta [y] += delta * hidden1_act [y];
        }
      hidden2_sum_delta_weight [x] = 0.0;
    }

  /* Finally calculate deltas of weights between hidden layer 1 and input
     layer */
  size = num_inputs;
  for (x = num_hidden1 - 1; x >= 0; --x)
    {
      sum_delta = hidden1_sum_delta_weight [x];
      p_sum_delta = &input_weights_sum_delta [x * size];
      p_learning_delta = &input_learning_delta [x * size];
      for (y = num_inputs - 1; y >= 0; --y)
        {
          /* Formula delta = SUM (previous deltas*weight)
                             * derivative * activation of input
             previous deltas already muliplied by weight
             derivative = S(1 - S)
          */
          delta = sum_delta * hidden1_act [x] *
                  (1.0 - hidden1_act [x]) * SLOPE;

          p_learning_delta [y] += delta;

          p_sum_delta [y] += (delta * input [y]);
        }
      hidden1_sum_delta_weight [x] = 0.0;
    }

  /* Now all deltas have been calculated and added into their appropriate
     neuron connection. */
  ++examples_since_update;

}


double Neural_network::calc_forward (double input [], double desired_output [],
                                     int& num_wrong, int& skip, int print_it,
                                     int& actual_printed)
{
  int     x,y,wrong;
  int     size;
  double  *weight,error,abs_error;

  skip = 1;
  wrong = 0;
  error = 0.0;

  /* Go backward for faster processing */
  /* Calculate hidden layer 1's activation */
  size = num_inputs;
  for (x = num_hidden1 - 1; x >= 0;  --x)
    {
      hidden1_act [x] = 0.0;
      weight = &input_weights [x * size];
      for (y = num_inputs - 1; y >= 0; --y)
        {
          hidden1_act [x] += (input [y] * weight [y]);
        }
      hidden1_act [x] = S(hidden1_act [x]);
    }

  /* Calculate hidden layer 2's activation */
  size = num_hidden1;
  for (x = num_hidden2 - 1; x >= 0; --x)
    {
      hidden2_act [x] = 0.0;
      weight = &hidden1_weights [x * size];
      for (y = num_hidden1 - 1; y >= 0; --y)
        {
          hidden2_act [x] += (hidden1_act [y] * weight [y]);
        }
      hidden2_act [x] = S(hidden2_act [x]);
    }

  /* Calculate output layer's activation */
  size = num_hidden2;
  for (x = num_outputs - 1; x >= 0; --x)
    {
      output_act [x] = 0.0;
      weight = &hidden2_weights [x * size];
      for (y = num_hidden2 - 1; y >= 0; --y)
        {
          output_act [x] += hidden2_act [y] * weight [y];
        }
      output_act [x] = S(output_act [x]);
      abs_error = fabs (output_act [x] - desired_output [x]);
      error += abs_error;
      if ( abs_error > epsilon )
          wrong = 1;
      if ( abs_error > skip_epsilon )
          skip = 0;
    }

  if ( wrong )
      ++num_wrong;

  cout.precision (3);
  if ( print_it == 2 )
    {
      for (x = 0; x < num_outputs; ++x)
        {
          cout.width (6);
          cout << output_act [x] << " ";
        }
      ++actual_printed;
    }
  else if ( print_it && wrong )
    {
      for (x = 0; x < num_outputs; ++x)
        {
          cout.width (6);
          cout << fabs (desired_output [x] - output_act [x]) << " ";
        }
      ++actual_printed;
    }

  return (error);

}


void Neural_network::update_weights ()
{
  int     x,y;
  int     size;
  double  rate,*p_ldelta,*p_ldelta_bar,*weight,*p_lrate,*p_sum_delta;

  // Check to see if any changes have been calculated.
  if ( examples_since_update == 0 )
    {
      return;
    }

  /* Go backward for slightly faster processing */
  /* First add deltas of weights from output to hidden layer 2. */
  size = num_hidden2;
  for (x = num_outputs - 1; x >= 0; --x)
    {
      p_ldelta = &hidden2_learning_delta [x * size];
      p_ldelta_bar = &hidden2_learning_delta_bar [x * size];
      weight = &hidden2_weights [x * size];
      p_lrate = &hidden2_learning_rate [x * size];
      p_sum_delta = &hidden2_weights_sum_delta [x * size];

      for (y = num_hidden2 - 1; y >= 0; --y)
        {
          rate = p_ldelta [y] * p_ldelta_bar [y];
          if ( rate < 0.0 )
            {
              p_lrate [y] -= (phi * p_lrate [y]);
            }
          else if ( rate > 0.0 )
            {
              p_lrate [y] += K;
            }

          weight [y] += (p_lrate [y] * p_sum_delta [y]);
          p_sum_delta [y] = 0.0;
          p_ldelta_bar [y] *= theta;
          p_ldelta_bar [y] += ((1.0 - theta) * p_ldelta [y]);
          p_ldelta [y] = 0.0;
        }
    }

  /* Next add deltas of weights between hidden layer2 and hidden
     layer 1 */
  size = num_hidden1;
  for (x = num_hidden2 - 1; x >= 0; --x)
    {
      p_ldelta = &hidden1_learning_delta [x * size];
      p_ldelta_bar = &hidden1_learning_delta_bar [x * size];
      weight = &hidden1_weights [x * size];
      p_lrate = &hidden1_learning_rate [x * size];
      p_sum_delta = &hidden1_weights_sum_delta [x * size];

      for (y = num_hidden1 - 1; y >= 0; --y)
        {
          rate = p_ldelta [y] * p_ldelta_bar [y];
          if ( rate < 0.0 )
            {
              p_lrate [y] -= (phi * p_lrate [y]);
            }
          else if ( rate > 0.0 )
            {
              p_lrate [y] += K;
            }

          weight [y] += (p_lrate [y] * p_sum_delta [y]);
          p_sum_delta [y] = 0.0;
          p_ldelta_bar [y] *= theta;
          p_ldelta_bar [y] += ((1.0 - theta) * p_ldelta [y]);
          p_ldelta [y] = 0.0;
        }
    }

  /* Finally add deltas of weights between hidden layer 1 and input
     layer */
  size = num_inputs;
  for (x = num_hidden1 - 1; x >= 0; --x)
    {
      p_ldelta = &input_learning_delta [x * size];
      p_ldelta_bar = &input_learning_delta_bar [x * size];
      weight = &input_weights [x * size];
      p_lrate = &input_learning_rate [x * size];
      p_sum_delta = &input_weights_sum_delta [x * size];

      for (y = num_inputs - 1; y >= 0; --y)
        {
          rate = p_ldelta [y] * p_ldelta_bar [y];
          if ( rate < 0.0 )
            {
              p_lrate [y] -= (phi * p_lrate [y]);
            }
          else if ( rate > 0.0 )
            {
              p_lrate [y] += K;
            }

          weight [y] += (p_lrate [y] * p_sum_delta [y]);
          p_sum_delta [y] = 0.0;
          p_ldelta_bar [y] *= theta;
          p_ldelta_bar [y] += ((1.0 - theta) * p_ldelta [y]);
          p_ldelta [y] = 0.0;
        }
    }

  /* Now all deltas have been added into their appropriate neuron
     connection. */
  training_examples += examples_since_update;
  examples_since_update = 0;

}


int Neural_network::calc_forward_test (double input [],
                                       double desired_output [],
                                       int print_it, double correct_eps,
                                       double good_eps)
{
  int x,y,wrong,good;

  wrong = 0;
  good = 0;

  /* Go backward for faster processing */
  /* Calculate hidden layer 1's activation */
  for (x = num_hidden1 - 1; x >= 0;  --x)
    {
      hidden1_act [x] = 0.0;
      for (y = num_inputs - 1; y >= 0; --y)
        {
          hidden1_act [x] += (input [y] *
                              input_weights [x * num_inputs + y]);
        }
      hidden1_act [x] = S(hidden1_act [x]);
    }

  /* Calculate hidden layer 2's activation */
  for (x = num_hidden2 - 1; x >= 0; --x)
    {
      hidden2_act [x] = 0.0;
      for (y = num_hidden1 - 1; y >= 0; --y)
        {
          hidden2_act [x] += (hidden1_act [y] *
                                 hidden1_weights [x*num_hidden1 + y]);
        }
      hidden2_act [x] = S(hidden2_act [x]);
    }

  /* Calculate output layer's activation */
  for (x = num_outputs - 1; x >= 0; --x)
    {
      output_act [x] = 0.0;
      for (y = num_hidden2 - 1; y >= 0; --y)
        {
          output_act [x] += hidden2_act [y] *
                                hidden2_weights [x * num_hidden2 + y];
        }
      output_act [x] = S(output_act [x]);

      if ( fabs (output_act [x] - desired_output [x]) > good_eps )
          wrong = 1;
      else if ( fabs (output_act [x] - desired_output [x]) > correct_eps )
          good = 1;
    }

  cout.precision(3);
  if ( print_it )
    {
      for (x = 0; x < num_outputs; ++x)
        {
          cout.width (6);
          cout << output_act [x] << " ";
        }
    }

  if ( wrong )
      return (WRONG);
  else if ( good )
      return (GOOD);
  else
      return (CORRECT);
}


void Neural_network::kick_weights (double range)
{
  int    x;
  double rand_max;
  double variation;

  rand_max = RAND_MAX;
  /* Add from -range to +range to all weights randomly */
  for (x = 0; x < (num_hidden1 * num_inputs); ++x)
    {
      variation = rand () / rand_max * range;
      if ( rand () < (RAND_MAX / 2) )
          variation *= -1.0;
      input_weights [x] += variation;
    }

  for (x = 0; x < (num_hidden2 * num_hidden1); ++x)
    {
      variation = rand () / rand_max * range;
      if ( rand () < (RAND_MAX / 2) )
          variation *= -1.0;
      hidden1_weights [x] += variation;
    }

  for (x = 0; x < (num_outputs * num_hidden2); ++x)
    {
      variation = rand () / rand_max * range;
      if ( rand () < (RAND_MAX / 2) )
          variation *= -1.0;
      hidden2_weights [x] += variation;
    }

}