Technically, a deep neural network is one that has two or more hidden layers of processing nodes. But deep neural networks for image processing or natural language processing typically have dozens or hundreds of hidden layers.

A special case is a neural network regressor that has exactly two hidden layers. My colleagues and I call such a system a neural network bistratum regressor. Bistratum means two layers in Latin.

In theory, a neural network with a single hidden layer can compute anything a neural network with two ot more layers can, but in practice, two hidden layers often works better.

I put together a demo to illustrate the neural network bistratum regressor input-output mechanism. The output of the demo is:

Neural network bistratum regression IO using C#

Creating 2-3-3-1 tanh() identity() neural network regressor
Done

Setting weights and biases to 0.01 to 0.25
Done

Computing output for x = [1.0, 2.0]
y = 0.4418

Displaying hidden node values:

A: 0.1586 0.1974 0.2355

B: 0.2629 0.2776 0.2923

End demo

The demo network has two input nodes, two hidden layers with three nodes each, and a single output node. The network uses tanh() activation on the hidden layer nodes, and identity() activation on the output node. The network is initialized with weights and biases values from 0.01 to 0.25. The network is fed an input of [1.0, 2.0]. The output is y = 0.4418.

This diagram explains the calculations:

Click to enlarge

The main challenge when using a neural network in finding good values for the weights and biases so that computed outputs match known correct outputs in a set of training data. The most common way to do this is by using one of many variations of stochastic gradient descent.

I’m a big fan of old science fiction movies from the 1950s. Jets had been developed only a few years earlier. Jet bombers made a couple of notable (to me anyway) appearances in two of my favorite movies of the decade.

Bottom Row: In “Beginning of the End” (1957), experiments with radiation to grow giant food result in giant locusts/grasshoppers. Hundreds of thousands of the menaces threaten to destroy Chicago and the move on. As a last resort, the military plans to use a B-36 to drop a nuclear bomb on Chicago. Scientists come up with a clever plan to use a synthetic mating call to lure the insects into Lake Michigan, and humanity is saved.

Demo program. Replace “lt” (less than), “gt”, “lte”, “gte” with Boolean operator symbols (my blog editor often chokes on symbols).

using System;
using System.IO;
using System.Collections.Generic;

namespace NeuralNetworkBistratumRegression
{
  internal class NeuralNetworkBistratumRegressionProgram
  {
    static void Main(string[] args)
    {
      Console.WriteLine("\nNeural network bistratum " +
        "regression IO using C# ");

      Console.WriteLine("\nCreating 2-3-3-1 tanh()" +
        " identity() neural network regressor ");
      
      NeuralNetworkBistratumRegressor nn =
         new NeuralNetworkBistratumRegressor(2, 3, 3, 1);
      Console.WriteLine("Done ");

      Console.WriteLine("\nSetting weights and biases to" +
        " 0.01 to 0.25 ");
      double[] wts = new double[] {.01, .02, .03, .04, .05,
        .06, .07, .08, .09, .10, .11, .12, .13, .14, .15,
        .16, .17, .18, .19, .20, .21, .22, .23, .24, .25 };
      nn.SetWeights(wts);
      Console.WriteLine("Done ");

      Console.WriteLine("\nComputing output" +
        " for x = [1.0, 2.0] ");
      double y = nn.Predict(new double[] { 1, 2 });
      Console.WriteLine("y = " + y.ToString("F4"));

      Console.WriteLine("\nDisplaying hidden node values: ");
      Console.WriteLine("\nA: " + 
        nn.aNodes[0].ToString("F4") + " " +
        nn.aNodes[1].ToString("F4") + " " +
        nn.aNodes[2].ToString("F4"));
      Console.WriteLine("\nB: " + 
        nn.bNodes[0].ToString("F4") + " " +
        nn.bNodes[1].ToString("F4") + " " +
        nn.bNodes[2].ToString("F4"));

      Console.WriteLine("\nEnd demo ");
      Console.ReadLine();

    } // Main

  } // class Program

  // ========================================================

  public class NeuralNetworkBistratumRegressor
  {
    // two hidden layers
    public int numInput;
    public int numHiddenA;
    public int numHiddenB;
    public int numOutput;

    public double[] iNodes;  // input nodes
    public double[] aNodes;
    public double[] bNodes;
    public double[] oNodes;  // output nodes

    public double[][] iaWeights; // input to hidden A
    public double[][] abWeights; // hidden A to hidden B
    public double[][] boWeights; // hidden B to output

    public double[] aBiases;
    public double[] bBiases;
    public double[] oBiases;

    private Random rnd;

    // ------------------------------------------------------

    public NeuralNetworkBistratumRegressor(int numInput,
      int numHiddenA, int numHiddenB, int numOutput,
      int seed = 0)
    {
      this.numInput = numInput;
      this.numHiddenA = numHiddenA;
      this.numHiddenB = numHiddenB;
      this.numOutput = numOutput; // 1 for regression

      this.iNodes = new double[numInput];
      this.iaWeights = MatMake(numInput, numHiddenA);
      this.aBiases = new double[numHiddenA];
      this.aNodes = new double[numHiddenA];

      this.abWeights = MatMake(numHiddenA, numHiddenB);
      this.bBiases = new double[numHiddenB];
      this.bNodes = new double[numHiddenB];

      this.boWeights = MatMake(numHiddenB, numOutput);
      this.oBiases = new double[numOutput];
      this.oNodes = new double[numOutput];

      this.rnd = new Random(seed);
    }

    // ------------------------------------------------------

    private static double[][] MatMake(int nRows, int nCols)
    {
      double[][] result = new double[nRows][];
      for (int i = 0; i "lt" nRows; ++i)
        result[i] = new double[nCols];
      return result;
    }

    // ------------------------------------------------------

    public double Predict(double[] x)
    {
      // copy input into iNodes
      for (int i = 0; i "lt" this.numInput; ++i)
        this.iNodes[i] = x[i];

      // compute hidden layer A
      for (int j = 0; j "lt" numHiddenA; ++j)
      {
        double sum = 0.0;
        for (int i = 0; i "lt" this.numInput; ++i)
          sum += this.iNodes[i] * this.iaWeights[i][j];
        sum += this.aBiases[j];
        this.aNodes[j] = HyperTan(sum);
      }

      // compute hidden layer B
      for (int j = 0; j "lt" this.numHiddenB; ++j)
      {
        double sum = 0.0;
        for (int i = 0; i "lt" this.numHiddenA; ++i)
          sum += this.aNodes[i] * this.abWeights[i][j];
        sum += this.bBiases[j];
        this.bNodes[j] = HyperTan(sum);
      }

      // compute output layer node
      for (int j = 0; j "lt" this.numOutput; ++j)
      {
        double sum = 0.0;
        for (int i = 0; i "lt" this.numHiddenB; ++i)
          sum += this.bNodes[i] * this.boWeights[i][j];
        sum += this.oBiases[j];
        this.oNodes[j] = Identity(sum);
      }

      return this.oNodes[0];  // a single value
    }

    // ------------------------------------------------------

    private static double HyperTan(double x)
    {
      if (x "lt" -6.0) return -1.0;
      else if (x "gt" 6.0) return 1.0;
      else return Math.Tanh(x);
    }

    // ------------------------------------------------------

    private static double Identity(double x)
    {
      return x;
    }


    // ------------------------------------------------------

    public void SetWeights(double[] weights)
    {
      int ptr = 0;
      for (int i = 0; i "lt" this.numInput; ++i)
        for (int j = 0; j "lt" this.numHiddenA; ++j)
          this.iaWeights[i][j] = weights[ptr++];

      for (int i = 0; i "lt" numHiddenA; ++i)
        this.aBiases[i] = weights[ptr++];

      for (int i = 0; i "lt" this.numHiddenA; ++i)
        for (int j = 0; j "lt" this.numHiddenB; ++j)
          this.abWeights[i][j] = weights[ptr++];

      for (int i = 0; i "lt" this.numHiddenB; ++i)
        this.bBiases[i] = weights[ptr++];

      for (int i = 0; i "lt" this.numHiddenB; ++i)
        for (int j = 0; j "lt" this.numOutput; ++j)
          this.boWeights[i][j] = weights[ptr++];

      for (int i = 0; i "lt" this.numOutput; ++i)
        this.oBiases[i] = weights[ptr++];
    }

    // ------------------------------------------------------

  } // class NeuralNetworkBistratumRegressor

  // ========================================================

} // ns