From 6fc4dacb22e9bc82cfc25a3fc719c2fee22217b3 Mon Sep 17 00:00:00 2001 From: Kory Becker Date: Tue, 28 Jan 2014 14:59:37 -0500 Subject: [PATCH 1/5] Added hidden layer. Unsupervised pre-training no longer includes the output layer. --- DeepLearning/Program.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DeepLearning/Program.cs b/DeepLearning/Program.cs index 5e40965..38646a6 100644 --- a/DeepLearning/Program.cs +++ b/DeepLearning/Program.cs @@ -34,7 +34,7 @@ static void Main(string[] args) }; // Setup the deep belief network and initialize with random weights. - DeepBeliefNetwork network = new DeepBeliefNetwork(2, 2); + DeepBeliefNetwork network = new DeepBeliefNetwork(inputs.First().Length, 1, 2); new GaussianWeights(network, 0.1).Randomize(); network.UpdateVisibleWeights(); @@ -57,8 +57,8 @@ static void Main(string[] args) // Learning data for the specified layer. double[][][] layerData; - // Unsupervised learning on each layer. - for (int layerIndex = 0; layerIndex < network.Machines.Count; layerIndex++) + // Unsupervised learning on each hidden layer, except for the output. + for (int layerIndex = 0; layerIndex < network.Machines.Count - 1; layerIndex++) { teacher.LayerIndex = layerIndex; layerData = teacher.GetLayerInput(batches); From e75a8b85fac0014726d1d5131f391240b7e03d29 Mon Sep 17 00:00:00 2001 From: Kory Becker Date: Tue, 28 Jan 2014 15:24:08 -0500 Subject: [PATCH 2/5] Set OR to XOR and 1 output node. --- DeepLearning/Program.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/DeepLearning/Program.cs b/DeepLearning/Program.cs index 38646a6..1740c38 100644 --- a/DeepLearning/Program.cs +++ b/DeepLearning/Program.cs @@ -27,14 +27,14 @@ static void Main(string[] args) // XOR output, cooresponding to the input. double[][] outputs = new double[][] { - new double[] { 1, 0 }, - new double[] { 0, 1 }, - new double[] { 0, 1 }, - new double[] { 0, 1 } + new double[] { 0 }, + new double[] { 1 }, + new double[] { 1 }, + new double[] { 0 } }; // Setup the deep belief network and initialize with random weights. - DeepBeliefNetwork network = new DeepBeliefNetwork(inputs.First().Length, 1, 2); + DeepBeliefNetwork network = new DeepBeliefNetwork(inputs.First().Length, 1, 1); new GaussianWeights(network, 0.1).Randomize(); network.UpdateVisibleWeights(); From 5243bbba40c86c83894fd14c10ef3cae6386a00f Mon Sep 17 00:00:00 2001 From: Kory Becker Date: Tue, 28 Jan 2014 15:27:07 -0500 Subject: [PATCH 3/5] Shortened training iterations. --- DeepLearning/Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DeepLearning/Program.cs b/DeepLearning/Program.cs index 1740c38..d05e0d5 100644 --- a/DeepLearning/Program.cs +++ b/DeepLearning/Program.cs @@ -62,7 +62,7 @@ static void Main(string[] args) { teacher.LayerIndex = layerIndex; layerData = teacher.GetLayerInput(batches); - for (int i = 0; i < 500; i++) + for (int i = 0; i < 100; i++) { double error = teacher.RunEpoch(layerData) / inputs.Length; if (i % 10 == 0) @@ -80,7 +80,7 @@ static void Main(string[] args) }; // Run supervised learning. - for (int i = 0; i < 5000; i++) + for (int i = 0; i < 100; i++) { double error = teacher2.RunEpoch(inputs, outputs) / inputs.Length; if (i % 10 == 0) From febee5c3a09c657cf04639f9d008368e629fb2a0 Mon Sep 17 00:00:00 2001 From: Kory Becker Date: Tue, 28 Jan 2014 15:43:06 -0500 Subject: [PATCH 4/5] Fixed bug with formatting output. Removed unused utility methods. --- DeepLearning/Program.cs | 48 ++++++----------------------------------- 1 file changed, 6 insertions(+), 42 deletions(-) diff --git a/DeepLearning/Program.cs b/DeepLearning/Program.cs index d05e0d5..b418b2e 100644 --- a/DeepLearning/Program.cs +++ b/DeepLearning/Program.cs @@ -34,7 +34,7 @@ static void Main(string[] args) }; // Setup the deep belief network and initialize with random weights. - DeepBeliefNetwork network = new DeepBeliefNetwork(inputs.First().Length, 1, 1); + DeepBeliefNetwork network = new DeepBeliefNetwork(inputs.First().Length, 3, 1); new GaussianWeights(network, 0.1).Randomize(); network.UpdateVisibleWeights(); @@ -62,7 +62,7 @@ static void Main(string[] args) { teacher.LayerIndex = layerIndex; layerData = teacher.GetLayerInput(batches); - for (int i = 0; i < 100; i++) + for (int i = 0; i < 5000; i++) { double error = teacher.RunEpoch(layerData) / inputs.Length; if (i % 10 == 0) @@ -80,7 +80,7 @@ static void Main(string[] args) }; // Run supervised learning. - for (int i = 0; i < 100; i++) + for (int i = 0; i < 5000; i++) { double error = teacher2.RunEpoch(inputs, outputs) / inputs.Length; if (i % 10 == 0) @@ -94,7 +94,9 @@ static void Main(string[] args) for (int i = 0; i < inputs.Length; i++) { double[] outputValues = network.Compute(inputs[i]); - if (FormatOutputResult(outputValues) == FormatOutputResult(outputs[i])) + double outputResult = outputValues.First() >= 0.5 ? 1 : 0; + + if (outputResult == outputs[i].First()) { correct++; } @@ -104,43 +106,5 @@ static void Main(string[] args) Console.Write("Press any key to quit .."); Console.ReadKey(); } - - #region Utility Methods - - /// - /// Converts a numeric output label (0, 1, 2, 3, etc) to its cooresponding array of doubles, where all values are 0 except for the index matching the label (ie., if the label is 2, the output is [0, 0, 1, 0, 0, ...]). - /// - /// double - /// double[] - private static double[] FormatOutputVector(double label) - { - double[] output = new double[10]; - - for (int i = 0; i < output.Length; i++) - { - if (i == label) - { - output[i] = 1; - } - else - { - output[i] = 0; - } - } - - return output; - } - - /// - /// Finds the largest output value in an array and returns its index. This allows for sequential classification from the outputs of a neural network (ie., if output at index 2 is the largest, the classification is class "3" (zero-based)). - /// - /// double[] - /// double - private static double FormatOutputResult(double[] output) - { - return output.ToList().IndexOf(output.Max()); - } - - #endregion } } From 913f696d03e47ab48ff04607138e2f681722e3ad Mon Sep 17 00:00:00 2001 From: Kory Becker Date: Tue, 28 Jan 2014 15:49:03 -0500 Subject: [PATCH 5/5] Added comments. --- DeepLearning/Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DeepLearning/Program.cs b/DeepLearning/Program.cs index b418b2e..a253c0b 100644 --- a/DeepLearning/Program.cs +++ b/DeepLearning/Program.cs @@ -33,8 +33,8 @@ static void Main(string[] args) new double[] { 0 } }; - // Setup the deep belief network and initialize with random weights. - DeepBeliefNetwork network = new DeepBeliefNetwork(inputs.First().Length, 3, 1); + // Setup the deep belief network (2 inputs, 3 hidden, 1 output) and initialize with random weights. + DeepBeliefNetwork network = new DeepBeliefNetwork(2, 3, 1); new GaussianWeights(network, 0.1).Randomize(); network.UpdateVisibleWeights();