From 15839c4eb6b1d2213c33bea11091ace4880cee75 Mon Sep 17 00:00:00 2001
From: Kory Becker <kbecker@primaryobjects.com>
Date: Thu, 30 Jan 2014 17:03:22 -0500
Subject: [PATCH 1/8] Re-factored neural network code into helper method, for
 easier training and cross validation.

---
 DeepLearning/Accuracy.cs         |  78 +++++++++++
 DeepLearning/DeepLearning.csproj |   8 ++
 DeepLearning/Program.cs          | 224 ++++++++++++++++++++++---------
 DeepLearning/Utility.cs          |  27 ++++
 4 files changed, 271 insertions(+), 66 deletions(-)
 create mode 100644 DeepLearning/Accuracy.cs
 create mode 100644 DeepLearning/Utility.cs

diff --git a/DeepLearning/Accuracy.cs b/DeepLearning/Accuracy.cs
new file mode 100644
index 0000000..d6b5a97
--- /dev/null
+++ b/DeepLearning/Accuracy.cs
@@ -0,0 +1,78 @@
+using Accord.MachineLearning.VectorMachines;
+using Accord.Neuro.Networks;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace DeepLearning
+{
+    public static class Accuracy
+    {
+        /// <summary>
+        /// Calculates the accuracy for a trainined SVM against the data.
+        /// </summary>
+        /// <param name="machine">MulticlassSupportVectorMachine</param>
+        /// <param name="data">List of DigitData</param>
+        /// <returns>double</returns>
+        public static double CalculateAccuracy(MulticlassSupportVectorMachine machine, double[][] inputs, int[] outputs)
+        {
+            double correct = 0;
+
+            for (int i = 0; i < inputs.Length; i++)
+            {
+                int output = machine.Compute(inputs[i]);
+                if (output == outputs[i])
+                {
+                    correct++;
+                }
+            }
+
+            return (correct / (double)inputs.Length);
+        }
+
+        /// <summary>
+        /// Calculates the accuracy for a trainined DNN against the data.
+        /// </summary>
+        /// <param name="network">DeepBeliefNetwork</param>
+        /// <param name="data">List of DigitData</param>
+        /// <returns>double</returns>
+        public static double CalculateAccuracy(DeepBeliefNetwork network, double[][] inputs, double[][] outputs)
+        {
+            double correct = 0;
+
+            for (int i = 0; i < inputs.Length; i++)
+            {
+                double[] outputValues = network.Compute(inputs[i]);
+                if (DataManager.FormatOutputResult(outputValues) == DataManager.FormatOutputResult(outputs[i]))
+                {
+                    correct++;
+                }
+            }
+
+            return (correct / (double)inputs.Length);
+        }
+
+        /// <summary>
+        /// Calculates the output for the svm and saves each result to a text file, one per line.
+        /// </summary>
+        /// <param name="machine">MulticlassSupportVectorMachine</param>
+        /// <param name="inputs">double[][]</param>
+        /// <param name="path">string</param>
+        /// <returns>int - number of rows processed</returns>
+        public static int SaveOutput(MulticlassSupportVectorMachine machine, double[][] inputs, string path)
+        {
+            File.AppendAllText(path, "ImageId,Label\r\n");
+
+            for (int i = 0; i < inputs.Length; i++)
+            {
+                int output = machine.Compute(inputs[i]);
+                File.AppendAllText(path, (i + 1) + "," + output.ToString() + "\r\n");
+            }
+
+            return inputs.Length;
+        }
+    }
+}
diff --git a/DeepLearning/DeepLearning.csproj b/DeepLearning/DeepLearning.csproj
index 61875a1..b44f2af 100644
--- a/DeepLearning/DeepLearning.csproj
+++ b/DeepLearning/DeepLearning.csproj
@@ -70,14 +70,22 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="Accuracy.cs" />
     <Compile Include="DataManager.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Utility.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config" />
     <None Include="packages.config" />
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\MLParser\MLParser.csproj">
+      <Project>{65c6935b-c55f-4208-88c8-61574dad1d7c}</Project>
+      <Name>MLParser</Name>
+    </ProjectReference>
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
diff --git a/DeepLearning/Program.cs b/DeepLearning/Program.cs
index 968f300..ca78bec 100644
--- a/DeepLearning/Program.cs
+++ b/DeepLearning/Program.cs
@@ -10,97 +10,189 @@
 using System.Threading.Tasks;
 using AForge.Neuro.Learning;
 using System.IO;
+using MLParser;
+using MLParser.Parsers;
+using MLParser.Types;
+using Accord.MachineLearning.VectorMachines;
+using Accord.MachineLearning.VectorMachines.Learning;
+using Accord.Statistics.Kernels;
+using MLParser.Interface;
+using System.Diagnostics;
 
 namespace DeepLearning
 {
     class Program
     {
         static void Main(string[] args)
+        {
+            Console.WriteLine("-= Training =-");
+            var network = RunDNN(@"../../../data/catsdogs-train.csv", 100, 10);
+            Console.WriteLine("-= Cross Validation =-");
+            RunDNN(@"../../../data/catsdogs-cv.csv", 100, 10, network);
+
+            /*for (int count = 200; count < 4000; count += 200)
+            {
+                Console.WriteLine("-= Training =-");
+                var machine = RunSvm(@"../../../data/catsdogs-train.csv", count);
+
+                Console.WriteLine("-= Cross Validation =-");
+                RunSvm(@"../../../data/catsdogs-cv.csv", count, machine);
+            }*/
+
+            Console.Write("Press any key to quit ..");
+            Console.ReadKey();
+        }
+
+        /// <summary>
+        /// Core machine learning method for parsing csv data, training the network, and calculating the accuracy.
+        /// </summary>
+        /// <param name="path">string - path to csv file (training, csv, test).</param>
+        /// <param name="count">int - max number of rows to process. This is useful for preparing learning curves, by using gradually increasing values. Use Int32.MaxValue to read all rows.</param>
+        /// <param name="epochs">int - max number of epochs per layer.</param>
+        /// <param name="network">DeepBeliefNetwork - Leave null for initial training.</param>
+        /// <returns>DeepBeliefNetwork</returns>
+        private static DeepBeliefNetwork RunDNN(string path, int count, int epochs, DeepBeliefNetwork network = null)
         {
             double[][] inputs;
             double[][] outputs;
-            double[][] testInputs;
-            double[][] testOutputs;
-
-            // Load ascii digits dataset.
-            inputs = DataManager.Load(@"../../../data/data.txt", out outputs);
-
-            // The first 500 data rows will be for training. The rest will be for testing.
-            testInputs = inputs.Skip(500).ToArray();
-            testOutputs = outputs.Skip(500).ToArray();
-            inputs = inputs.Take(500).ToArray();
-            outputs = outputs.Take(500).ToArray();
-
-            // Setup the deep belief network and initialize with random weights.
-            DeepBeliefNetwork network = new DeepBeliefNetwork(inputs.First().Length, 10, 10);
-            new GaussianWeights(network, 0.1).Randomize();
-            network.UpdateVisibleWeights();
+            int[] intOutputs;
+
+            // Parse the csv file to get inputs and outputs.
+            ReadData(path, count, out inputs, out intOutputs, new EndStringEndLabelParser());
             
-            // Setup the learning algorithm.
-            DeepBeliefNetworkLearning teacher = new DeepBeliefNetworkLearning(network)
+            // Format output as double[][].
+            outputs = intOutputs.Select(o => DataManager.FormatOutputVector((double)o)).ToArray();
+
+            if (network == null)
             {
-                Algorithm = (h, v, i) => new ContrastiveDivergenceLearning(h, v)
+                // Training.
+                network = DeepBeliefNetwork.Load(@"../../../data/network.dat");/* new DeepBeliefNetwork(inputs.First().Length, 100, 100, 100, 100, 2);
+                new NguyenWidrow(network).Randomize();
+                network.UpdateVisibleWeights();
+                network.Save(@"../../../data/network.dat");*/
+
+                // Setup the learning algorithm.
+                DeepBeliefNetworkLearning teacher = new DeepBeliefNetworkLearning(network)
                 {
-                    LearningRate = 0.1,
-                    Momentum = 0.5,
-                    Decay = 0.001,
-                }
-            };
-
-            // Setup batches of input for learning.
-            int batchCount = Math.Max(1, inputs.Length / 100);
-            // Create mini-batches to speed learning.
-            int[] groups = Accord.Statistics.Tools.RandomGroups(inputs.Length, batchCount);
-            double[][][] batches = inputs.Subgroups(groups);
-            // Learning data for the specified layer.
-            double[][][] layerData;
-
-            // Unsupervised learning on each hidden layer, except for the output layer.
-            for (int layerIndex = 0; layerIndex < network.Machines.Count - 1; layerIndex++)
-            {
-                teacher.LayerIndex = layerIndex;
-                layerData = teacher.GetLayerInput(batches);
-                for (int i = 0; i < 200; i++)
+                    Algorithm = (h, v, i) => new ContrastiveDivergenceLearning(h, v)
+                    {
+                        LearningRate = 0.1,
+                        Momentum = 0.5,
+                        Decay = 0.001,
+                    }
+                };
+
+                // Setup batches of input for learning.
+                int batchCount = Math.Max(1, inputs.Length / 100);
+                // Create mini-batches to speed learning.
+                int[] groups = Accord.Statistics.Tools.RandomGroups(inputs.Length, batchCount);
+                double[][][] batches = inputs.Subgroups(groups);
+                // Learning data for the specified layer.
+                double[][][] layerData;
+
+                DateTime startTime = DateTime.Now;
+                DateTime epochStart = DateTime.Now;
+
+                // Unsupervised learning on each hidden layer, except for the output layer.
+                for (int layerIndex = 0; layerIndex < network.Machines.Count - 1; layerIndex++)
                 {
-                    double error = teacher.RunEpoch(layerData) / inputs.Length;
-                    if (i % 10 == 0)
+                    teacher.LayerIndex = layerIndex;
+                    layerData = teacher.GetLayerInput(batches);
+                    for (int i = 0; i < epochs; i++)
                     {
-                        Console.WriteLine(i + ", Error = " + error);
+                        double error = teacher.RunEpoch(layerData) / inputs.Length;
+                        if (i % 10 == 0)
+                        {
+                            TimeSpan timeSpan = DateTime.Now - epochStart;
+                            Console.WriteLine(i + ", Error = " + error + ", " + Math.Round(timeSpan.TotalMinutes) + "m (" + Math.Round(timeSpan.TotalSeconds) + "s)");
+                            epochStart = DateTime.Now;
+                        }
                     }
                 }
-            }
 
-            // Supervised learning on entire network, to provide output classification.
-            var teacher2 = new BackPropagationLearning(network)
-            {
-                LearningRate = 0.1,
-                Momentum = 0.5
-            };
+                // Supervised learning on entire network, to provide output classification.
+                var teacher2 = new BackPropagationLearning(network)
+                {
+                    LearningRate = 0.1,
+                    Momentum = 0.5
+                };
 
-            // Run supervised learning.
-            for (int i = 0; i < 500; i++)
-            {
-                double error = teacher2.RunEpoch(inputs, outputs) / inputs.Length;
-                if (i % 10 == 0)
+                epochStart = DateTime.Now;
+
+                // Run supervised learning.
+                for (int i = 0; i < epochs; i++)
                 {
-                    Console.WriteLine(i + ", Error = " + error);
+                    double error = teacher2.RunEpoch(inputs, outputs) / inputs.Length;
+                    if (i % 10 == 0)
+                    {
+                        TimeSpan timeSpan = DateTime.Now - epochStart;
+                        Console.WriteLine(i + ", Error = " + error + ", " + Math.Round(timeSpan.TotalMinutes) + "m (" + Math.Round(timeSpan.TotalSeconds) + "s)");
+                        epochStart = DateTime.Now;
+                    }
                 }
+
+                TimeSpan runTime = DateTime.Now - startTime;
+                Console.WriteLine("Training completed after " + runTime.TotalMinutes + "m.");
+                startTime = DateTime.Now;
             }
 
-            // Test the resulting accuracy.
-            int correct = 0;
-            for (int i = 0; i < inputs.Length; i++)
+            // Calculate accuracy.
+            double accuracy = Utility.ShowProgressFor<double>(() => Accuracy.CalculateAccuracy(network, inputs, outputs), "Calculating Accuracy");
+            Console.WriteLine("Accuracy: " + Math.Round(accuracy * 100, 2) + "%");
+
+            return network;
+        }
+
+        /// <summary>
+        /// Core machine learning method for parsing csv data, training the svm, and calculating the accuracy.
+        /// </summary>
+        /// <param name="path">string - path to csv file (training, csv, test).</param>
+        /// <param name="count">int - max number of rows to process. This is useful for preparing learning curves, by using gradually increasing values. Use Int32.MaxValue to read all rows.</param>
+        /// <param name="machine">MulticlassSupportVectorMachine - Leave null for initial training.</param>
+        /// <returns>MulticlassSupportVectorMachine</returns>
+        private static MulticlassSupportVectorMachine RunSvm(string path, int count, MulticlassSupportVectorMachine machine = null)
+        {
+            double[][] inputs;
+            int[] outputs;
+
+            // Parse the csv file to get inputs and outputs.
+            ReadData(path, count, out inputs, out outputs, new EndStringEndLabelParser());
+
+            if (machine == null)
             {
-                double[] outputValues = network.Compute(testInputs[i]);
-                if (DataManager.FormatOutputResult(outputValues) == DataManager.FormatOutputResult(testOutputs[i]))
-                {
-                    correct++;
-                }
+                // Training.
+                MulticlassSupportVectorLearning teacher = null;
+
+                // Create the svm.
+                machine = new MulticlassSupportVectorMachine(1225, new Gaussian(4), 2);
+                teacher = new MulticlassSupportVectorLearning(machine, inputs, outputs);
+                teacher.Algorithm = (svm, classInputs, classOutputs, i, j) => new SequentialMinimalOptimization(svm, classInputs, classOutputs) { CacheSize = 0 };
+
+                // Train the svm.
+                Utility.ShowProgressFor(() => teacher.Run(), "Training");
             }
 
-            Console.WriteLine("Correct " + correct + "/" + inputs.Length + ", " + Math.Round(((double)correct / (double)inputs.Length * 100), 2) + "%");
-            Console.Write("Press any key to quit ..");
-            Console.ReadKey();
+            // Calculate accuracy.
+            double accuracy = Utility.ShowProgressFor<double>(() => Accuracy.CalculateAccuracy(machine, inputs, outputs), "Calculating Accuracy");
+            Console.WriteLine("Accuracy: " + Math.Round(accuracy * 100, 2) + "%");
+
+            return machine;
+        }
+
+        private static int ReadData(string path, int count, out double[][] inputs, out int[] outputs, IRowParser rowParser)
+        {
+            Parser parser = new Parser(rowParser);
+
+            // Read the training data CSV file and get a resulting array of doubles and output labels.
+            List<MLData> rows = Utility.ShowProgressFor<List<MLData>>(() => parser.Parse(path, count), "Reading data");
+
+            // Convert the rows into arrays for processing.
+            inputs = rows.Select(t => t.Data.ToArray()).ToArray();
+            outputs = rows.Select(t => t.Label).ToArray();
+
+            Console.WriteLine(rows.Count + " rows processed.");
+
+            return rows.Count;
         }
     }
 }
diff --git a/DeepLearning/Utility.cs b/DeepLearning/Utility.cs
new file mode 100644
index 0000000..7c843fc
--- /dev/null
+++ b/DeepLearning/Utility.cs
@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace DeepLearning
+{
+    public static class Utility
+    {
+        /// <summary>
+        /// Helper method for displaying progress text to the console for a specific operation.
+        /// </summary>
+        /// <param name="action">Action</param>
+        /// <param name="text">string</param>
+        public static T ShowProgressFor<T>(Func<T> action, string text)
+        {
+            T result;
+
+            Console.Write(text + " .. ");
+            result = action();
+            Console.WriteLine("Done!");
+
+            return result;
+        }
+    }
+}

From d5c49c1ed77ec82c8fb50ead77a2713668cc98d3 Mon Sep 17 00:00:00 2001
From: Kory Becker <kbecker@primaryobjects.com>
Date: Thu, 30 Jan 2014 17:17:18 -0500
Subject: [PATCH 2/8] Added app.config values.

---
 DeepLearning.sln                            |  6 ++
 DeepLearning/App.config                     | 13 ++++-
 DeepLearning/DeepLearning.csproj            |  1 +
 DeepLearning/Program.cs                     | 24 ++++++--
 MLParser/Interface/IRowParser.cs            | 26 +++++++++
 MLParser/MLParser.csproj                    | 62 +++++++++++++++++++++
 MLParser/Parser.cs                          | 60 ++++++++++++++++++++
 MLParser/Parsers/BaseParser.cs              | 52 +++++++++++++++++
 MLParser/Parsers/EndStringEndLabelParser.cs | 28 ++++++++++
 MLParser/Parsers/FrontLabelParser.cs        | 28 ++++++++++
 MLParser/Parsers/TestParser.cs              | 27 +++++++++
 MLParser/Properties/AssemblyInfo.cs         | 36 ++++++++++++
 MLParser/Types/MLData.cs                    | 28 ++++++++++
 readme.md                                   |  2 +-
 14 files changed, 386 insertions(+), 7 deletions(-)
 create mode 100644 MLParser/Interface/IRowParser.cs
 create mode 100644 MLParser/MLParser.csproj
 create mode 100644 MLParser/Parser.cs
 create mode 100644 MLParser/Parsers/BaseParser.cs
 create mode 100644 MLParser/Parsers/EndStringEndLabelParser.cs
 create mode 100644 MLParser/Parsers/FrontLabelParser.cs
 create mode 100644 MLParser/Parsers/TestParser.cs
 create mode 100644 MLParser/Properties/AssemblyInfo.cs
 create mode 100644 MLParser/Types/MLData.cs

diff --git a/DeepLearning.sln b/DeepLearning.sln
index 8dda2c5..5f84494 100644
--- a/DeepLearning.sln
+++ b/DeepLearning.sln
@@ -12,6 +12,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{A71E9C
 		.nuget\NuGet.targets = .nuget\NuGet.targets
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MLParser", "MLParser\MLParser.csproj", "{65C6935B-C55F-4208-88C8-61574DAD1D7C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -22,6 +24,10 @@ Global
 		{B06BDCBB-94F7-49F2-B7A6-D9D472F9179D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B06BDCBB-94F7-49F2-B7A6-D9D472F9179D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B06BDCBB-94F7-49F2-B7A6-D9D472F9179D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{65C6935B-C55F-4208-88C8-61574DAD1D7C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{65C6935B-C55F-4208-88C8-61574DAD1D7C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{65C6935B-C55F-4208-88C8-61574DAD1D7C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{65C6935B-C55F-4208-88C8-61574DAD1D7C}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
diff --git a/DeepLearning/App.config b/DeepLearning/App.config
index fad249e..9477b01 100644
--- a/DeepLearning/App.config
+++ b/DeepLearning/App.config
@@ -1,6 +1,17 @@
 <?xml version="1.0" encoding="utf-8" ?>
 <configuration>
-    <startup> 
+  <appSettings>
+    <add key="Width" value="35"/> <!-- Width of each training image -->
+    <add key="Height" value="35"/> <!-- Height of each training image -->
+    <add key="ClassCount" value="2"/> <!-- Number of unique classes -->
+    <add key="TrainCount" value="0"/> <!-- Max number of rows to read from the training set (useful for preparing learning curves). Set to 0 for all data. -->
+    <add key="EpochCount" value="600"/> <!-- Number of epochs for each neural network layer to train on. -->
+    <add key="Sigma" value="12"/> <!-- SVM sigma training value, smoothes the gaussian ie., fuzziness factor (lower values = lower bias and higher variance and maybe overfitting, higher values = higher bias and lower variance). -->
+    <add key="TrainPath" value="../../../data/catsdogs-train.csv"/> <!-- Path to training csv file -->
+    <add key="CvPath" value="../../../data/catsdogs-cv.csv"/> <!-- Path to CV csv file -->
+    <add key="TestPath" value="../../../data/catsdogs_test.csv"/> <!-- Path to test csv file -->
+  </appSettings>
+  <startup> 
         <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
     </startup>
 </configuration>
\ No newline at end of file
diff --git a/DeepLearning/DeepLearning.csproj b/DeepLearning/DeepLearning.csproj
index b44f2af..eb0a0a8 100644
--- a/DeepLearning/DeepLearning.csproj
+++ b/DeepLearning/DeepLearning.csproj
@@ -62,6 +62,7 @@
       <HintPath>..\packages\AForge.Neuro.2.2.5\lib\AForge.Neuro.dll</HintPath>
     </Reference>
     <Reference Include="System" />
+    <Reference Include="System.Configuration" />
     <Reference Include="System.Core" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
diff --git a/DeepLearning/Program.cs b/DeepLearning/Program.cs
index ca78bec..8d5143d 100644
--- a/DeepLearning/Program.cs
+++ b/DeepLearning/Program.cs
@@ -18,17 +18,31 @@
 using Accord.Statistics.Kernels;
 using MLParser.Interface;
 using System.Diagnostics;
+using System.Configuration;
 
 namespace DeepLearning
 {
     class Program
     {
+        #region App.Config Values
+
+        private static int _pixelCount = Int32.Parse(ConfigurationManager.AppSettings["Width"]) * Int32.Parse(ConfigurationManager.AppSettings["Height"]);
+        private static int _classCount = Int32.Parse(ConfigurationManager.AppSettings["ClassCount"]);
+        private static int _trainCount = Int32.Parse(ConfigurationManager.AppSettings["TrainCount"]);
+        private static int _epochCount = Int32.Parse(ConfigurationManager.AppSettings["EpochCount"]);
+        private static double _sigma = Double.Parse(ConfigurationManager.AppSettings["Sigma"]);
+        private static string _trainPath = ConfigurationManager.AppSettings["TrainPath"];
+        private static string _cvPath = ConfigurationManager.AppSettings["CvPath"];
+        private static string _testPath = ConfigurationManager.AppSettings["TestPath"];
+
+        #endregion
+
         static void Main(string[] args)
         {
             Console.WriteLine("-= Training =-");
-            var network = RunDNN(@"../../../data/catsdogs-train.csv", 100, 10);
+            var network = RunDNN(_trainPath, _trainCount, _epochCount);
             Console.WriteLine("-= Cross Validation =-");
-            RunDNN(@"../../../data/catsdogs-cv.csv", 100, 10, network);
+            RunDNN(_cvPath, _trainCount, _epochCount, network);
 
             /*for (int count = 200; count < 4000; count += 200)
             {
@@ -47,7 +61,7 @@ static void Main(string[] args)
         /// Core machine learning method for parsing csv data, training the network, and calculating the accuracy.
         /// </summary>
         /// <param name="path">string - path to csv file (training, csv, test).</param>
-        /// <param name="count">int - max number of rows to process. This is useful for preparing learning curves, by using gradually increasing values. Use Int32.MaxValue to read all rows.</param>
+        /// <param name="count">int - max number of rows to process. This is useful for preparing learning curves, by using gradually increasing values. Use 0 to read all rows.</param>
         /// <param name="epochs">int - max number of epochs per layer.</param>
         /// <param name="network">DeepBeliefNetwork - Leave null for initial training.</param>
         /// <returns>DeepBeliefNetwork</returns>
@@ -147,7 +161,7 @@ private static DeepBeliefNetwork RunDNN(string path, int count, int epochs, Deep
         /// Core machine learning method for parsing csv data, training the svm, and calculating the accuracy.
         /// </summary>
         /// <param name="path">string - path to csv file (training, csv, test).</param>
-        /// <param name="count">int - max number of rows to process. This is useful for preparing learning curves, by using gradually increasing values. Use Int32.MaxValue to read all rows.</param>
+        /// <param name="count">int - max number of rows to process. This is useful for preparing learning curves, by using gradually increasing values. Use 0 to read all rows.</param>
         /// <param name="machine">MulticlassSupportVectorMachine - Leave null for initial training.</param>
         /// <returns>MulticlassSupportVectorMachine</returns>
         private static MulticlassSupportVectorMachine RunSvm(string path, int count, MulticlassSupportVectorMachine machine = null)
@@ -164,7 +178,7 @@ private static MulticlassSupportVectorMachine RunSvm(string path, int count, Mul
                 MulticlassSupportVectorLearning teacher = null;
 
                 // Create the svm.
-                machine = new MulticlassSupportVectorMachine(1225, new Gaussian(4), 2);
+                machine = new MulticlassSupportVectorMachine(_pixelCount, new Gaussian(_sigma), _classCount);
                 teacher = new MulticlassSupportVectorLearning(machine, inputs, outputs);
                 teacher.Algorithm = (svm, classInputs, classOutputs, i, j) => new SequentialMinimalOptimization(svm, classInputs, classOutputs) { CacheSize = 0 };
 
diff --git a/MLParser/Interface/IRowParser.cs b/MLParser/Interface/IRowParser.cs
new file mode 100644
index 0000000..a240d00
--- /dev/null
+++ b/MLParser/Interface/IRowParser.cs
@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using CsvHelper;
+using MLParser.Types;
+
+namespace MLParser.Interface
+{
+    public interface IRowParser
+    {
+        /// <summary>
+        /// Parses a row from the csv file and returns the label (output).
+        /// </summary>
+        /// <param name="reader">CsvReader</param>
+        /// <returns>int</returns>
+        int ReadLabel(CsvReader reader);
+        /// <summary>
+        /// Parses a row from the csv file and returns the data (input) fields.
+        /// </summary>
+        /// <param name="reader">CsvReader</param>
+        /// <returns>List of double</returns>
+        List<double> ReadData(CsvReader reader);
+    }
+}
diff --git a/MLParser/MLParser.csproj b/MLParser/MLParser.csproj
new file mode 100644
index 0000000..1a13b06
--- /dev/null
+++ b/MLParser/MLParser.csproj
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{65C6935B-C55F-4208-88C8-61574DAD1D7C}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>MLParser</RootNamespace>
+    <AssemblyName>MLParser</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="CsvHelper">
+      <HintPath>..\packages\CsvHelper.2.3.0\lib\net40-client\CsvHelper.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Interface\IRowParser.cs" />
+    <Compile Include="Parser.cs" />
+    <Compile Include="Parsers\BaseParser.cs" />
+    <Compile Include="Parsers\EndStringEndLabelParser.cs" />
+    <Compile Include="Parsers\TestParser.cs" />
+    <Compile Include="Parsers\FrontLabelParser.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Types\MLData.cs" />
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
\ No newline at end of file
diff --git a/MLParser/Parser.cs b/MLParser/Parser.cs
new file mode 100644
index 0000000..079ea7a
--- /dev/null
+++ b/MLParser/Parser.cs
@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using CsvHelper;
+using MLParser.Interface;
+using MLParser.Types;
+
+namespace MLParser
+{
+    public class Parser
+    {
+        private IRowParser _rowParser = null;
+
+        public Parser(IRowParser rowParser)
+        {
+            _rowParser = rowParser;
+        }
+
+        /// <summary>
+        /// Parses a csv file containing inputs and an output label, returning a list of MLData.
+        /// </summary>
+        /// <param name="path">string</param>
+        /// <param name="maxRows">int - max number of rows to read</param>
+        /// <returns>List of MLData</returns>
+        public List<MLData> Parse(string path, int maxRows = 0)
+        {
+            List<MLData> dataList = new List<MLData>();
+
+            using (FileStream f = new FileStream(path, FileMode.Open))
+            {
+                using (StreamReader streamReader = new StreamReader(f, Encoding.GetEncoding(1252)))
+                {
+                    using (CsvReader csvReader = new CsvReader(streamReader))
+                    {
+                        csvReader.Configuration.HasHeaderRecord = false;
+
+                        while (csvReader.Read())
+                        {
+                            MLData row = new MLData()
+                            {
+                                Label = _rowParser.ReadLabel(csvReader),
+                                Data = _rowParser.ReadData(csvReader)
+                            };
+
+                            dataList.Add(row);
+
+                            if (maxRows > 0 && dataList.Count >= maxRows)
+                                break;
+                        }
+                    }
+                }
+            }
+
+            return dataList;
+        }
+    }
+}
diff --git a/MLParser/Parsers/BaseParser.cs b/MLParser/Parsers/BaseParser.cs
new file mode 100644
index 0000000..ff2b347
--- /dev/null
+++ b/MLParser/Parsers/BaseParser.cs
@@ -0,0 +1,52 @@
+using CsvHelper;
+using MLParser.Interface;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace MLParser.Parsers
+{
+    public abstract class BaseParser : IRowParser
+    {
+        public abstract int ReadLabel(CsvReader reader);
+        public abstract List<double> ReadData(CsvReader reader);
+
+        /// <summary>
+        /// Helper method for reading a row of data from a csv file. Reading starts at the startColumn and ends at the endColumn.
+        /// </summary>
+        /// <param name="reader">CsvReader</param>
+        /// <param name="startColumn">int - start index to begin reading fields from.</param>
+        /// <param name="endColumn">int - end index to stop reading fields at. Set to null to read until the end of the row.</param>
+        /// <returns>List of double</returns>
+        protected List<double> ReadData(CsvReader reader, int startColumn, int? endColumn = null)
+        {
+            List<double> data = new List<double>();
+
+            if (endColumn == null)
+            {
+                // Read until the end of the row.
+                endColumn = reader.Parser.FieldCount;
+            }
+
+            // Start at index to begin reading data from.
+            for (int i = startColumn; i < endColumn; i++)
+            {
+                // Read the value.
+                double value = Double.Parse(reader[i]);
+
+                // Store the normalized value in our data list.
+                data.Add(Normalize(value));
+            }
+
+            return data;
+        }
+
+        protected double Normalize(double value)
+        {
+            // Normalize the value (0 - 1): X = (X - min) / (max - min) => X = X / 255. Alternate method (-0.5 - 0.5): X = (X - avg) / max - min => X = (X - 127) / 255. http://en.wikipedia.org/wiki/Feature_scaling
+            return value / 255d;
+        }
+    }
+}
diff --git a/MLParser/Parsers/EndStringEndLabelParser.cs b/MLParser/Parsers/EndStringEndLabelParser.cs
new file mode 100644
index 0000000..ddc2ce1
--- /dev/null
+++ b/MLParser/Parsers/EndStringEndLabelParser.cs
@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using CsvHelper;
+using MLParser.Interface;
+using MLParser.Types;
+
+namespace MLParser.Parsers
+{
+    /// <summary>
+    /// Parses a csv file, assuming the last 2 columns consist of a string (filename) followed by the label, and the remaining columns contain the data.
+    /// </summary>
+    public class EndStringEndLabelParser : BaseParser
+    {
+        public override int ReadLabel(CsvReader reader)
+        {
+            return Int32.Parse(reader[reader.Parser.FieldCount - 1]);
+        }
+
+        public override List<double> ReadData(CsvReader reader)
+        {
+            // Start at index 0, and read up to the last 2 columns, which are the string (filename) and label.
+            return ReadData(reader, 0, reader.Parser.FieldCount - 2);
+        }
+    }
+}
diff --git a/MLParser/Parsers/FrontLabelParser.cs b/MLParser/Parsers/FrontLabelParser.cs
new file mode 100644
index 0000000..2f00849
--- /dev/null
+++ b/MLParser/Parsers/FrontLabelParser.cs
@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using CsvHelper;
+using MLParser.Interface;
+using MLParser.Types;
+
+namespace MLParser.Parsers
+{
+    /// <summary>
+    /// Parses a csv file, assuming column 0 contains the label and the remaining columns contain the data.
+    /// </summary>
+    public class FrontLabelParser : BaseParser
+    {
+        public override int ReadLabel(CsvReader reader)
+        {
+            return Int32.Parse(reader[0]);
+        }
+
+        public override List<double> ReadData(CsvReader reader)
+        {
+            // Start at index 1, as the index 0 contains the label.
+            return ReadData(reader, 1);
+        }
+    }
+}
diff --git a/MLParser/Parsers/TestParser.cs b/MLParser/Parsers/TestParser.cs
new file mode 100644
index 0000000..9f2fd13
--- /dev/null
+++ b/MLParser/Parsers/TestParser.cs
@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using CsvHelper;
+using MLParser.Interface;
+using MLParser.Types;
+
+namespace MLParser.Parsers
+{
+    /// <summary>
+    /// Parses a csv file in its entirety as data. Assumes no label is present and all columns will be data points. Useful for test.csv files (which usually do not contain labels).
+    /// </summary>
+    public class TestParser : BaseParser
+    {
+        public override int ReadLabel(CsvReader reader)
+        {
+            return 0;
+        }
+
+        public override List<double> ReadData(CsvReader reader)
+        {
+            return ReadData(reader, 0);
+        }
+    }
+}
diff --git a/MLParser/Properties/AssemblyInfo.cs b/MLParser/Properties/AssemblyInfo.cs
new file mode 100644
index 0000000..3ff3401
--- /dev/null
+++ b/MLParser/Properties/AssemblyInfo.cs
@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("MLParser")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("MLParser")]
+[assembly: AssemblyCopyright("Copyright ©  2014")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("667bf2c2-d7b8-479b-91a1-333ad738c8d4")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
diff --git a/MLParser/Types/MLData.cs b/MLParser/Types/MLData.cs
new file mode 100644
index 0000000..a06376a
--- /dev/null
+++ b/MLParser/Types/MLData.cs
@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace MLParser.Types
+{
+    /// <summary>
+    /// Data-type for holding machine learning data from a csv file, consisting of an array of doubles (input) and a label (output).
+    /// </summary>
+    public class MLData
+    {
+        /// <summary>
+        /// Input
+        /// </summary>
+        public List<double> Data { get; set; }
+        /// <summary>
+        /// Output
+        /// </summary>
+        public int Label { get; set; }
+
+        public MLData()
+        {
+            Data = new List<double>();
+        }
+    }
+}
diff --git a/readme.md b/readme.md
index 05f0620..3c6be49 100644
--- a/readme.md
+++ b/readme.md
@@ -10,7 +10,7 @@ Checkout the master branch for a slightly less-basic example of training on an A
 Deep-Learning Strategy
 ----------------------
 
-1. Start with a neural network with multiple RestrictedBoltzman machine layers.
+1. Start with a neural network with multiple RestrictedBoltzmann machine layers.
 2. Use unsupervised training on each layer in the network, one at a time, except for the output layer. This allows each layer to learn specific features about the input data.
 3. If you ran unsupervised training on the whole network, including the output layer, add an additional (untrained) layer to the network to serve as the output layer. Otherwise, skip this step.
 4. Run back-propagation on the entire network to fine-tune for classification.

From e93daea252581263868fefd51cfa596039706f70 Mon Sep 17 00:00:00 2001
From: Kory Becker <kbecker@primaryobjects.com>
Date: Thu, 30 Jan 2014 17:30:21 -0500
Subject: [PATCH 3/8] Added classCount.

---
 DeepLearning/App.config     | 4 ++--
 DeepLearning/DataManager.cs | 7 ++++---
 DeepLearning/Program.cs     | 4 ++--
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/DeepLearning/App.config b/DeepLearning/App.config
index 9477b01..c58fcfc 100644
--- a/DeepLearning/App.config
+++ b/DeepLearning/App.config
@@ -4,8 +4,8 @@
     <add key="Width" value="35"/> <!-- Width of each training image -->
     <add key="Height" value="35"/> <!-- Height of each training image -->
     <add key="ClassCount" value="2"/> <!-- Number of unique classes -->
-    <add key="TrainCount" value="0"/> <!-- Max number of rows to read from the training set (useful for preparing learning curves). Set to 0 for all data. -->
-    <add key="EpochCount" value="600"/> <!-- Number of epochs for each neural network layer to train on. -->
+    <add key="TrainCount" value="10"/> <!-- Max number of rows to read from the training set (useful for preparing learning curves). Set to 0 for all data. -->
+    <add key="EpochCount" value="10"/> <!-- Number of epochs for each neural network layer to train on. -->
     <add key="Sigma" value="12"/> <!-- SVM sigma training value, smoothes the gaussian ie., fuzziness factor (lower values = lower bias and higher variance and maybe overfitting, higher values = higher bias and lower variance). -->
     <add key="TrainPath" value="../../../data/catsdogs-train.csv"/> <!-- Path to training csv file -->
     <add key="CvPath" value="../../../data/catsdogs-cv.csv"/> <!-- Path to CV csv file -->
diff --git a/DeepLearning/DataManager.cs b/DeepLearning/DataManager.cs
index 7e308ea..1f61ceb 100644
--- a/DeepLearning/DataManager.cs
+++ b/DeepLearning/DataManager.cs
@@ -54,7 +54,7 @@ public static double[][] Load(string pathName, out double[][] outputs)
                                 else
                                 {
                                     // Read output label.
-                                    output.Add(FormatOutputVector(Double.Parse(ch.ToString())));
+                                    output.Add(FormatOutputVector(Double.Parse(ch.ToString()), 10));
 
                                     // Set flag to read inputs for next row.
                                     readOutput = false;
@@ -78,10 +78,11 @@ public static double[][] Load(string pathName, out double[][] outputs)
         /// Converts a numeric output label (0, 1, 2, 3, etc) to its cooresponding array of doubles, where all values are 0 except for the index matching the label (ie., if the label is 2, the output is [0, 0, 1, 0, 0, ...]).
         /// </summary>
         /// <param name="label">double</param>
+        /// <param name="classCount">int - number of unique classes (ie., 10 for digits 0-9, 2 for true or false, etc).</param>
         /// <returns>double[]</returns>
-        public static double[] FormatOutputVector(double label)
+        public static double[] FormatOutputVector(double label, int classCount)
         {
-            double[] output = new double[10];
+            double[] output = new double[classCount];
 
             for (int i = 0; i < output.Length; i++)
             {
diff --git a/DeepLearning/Program.cs b/DeepLearning/Program.cs
index 8d5143d..0a3cf2c 100644
--- a/DeepLearning/Program.cs
+++ b/DeepLearning/Program.cs
@@ -75,12 +75,12 @@ private static DeepBeliefNetwork RunDNN(string path, int count, int epochs, Deep
             ReadData(path, count, out inputs, out intOutputs, new EndStringEndLabelParser());
             
             // Format output as double[][].
-            outputs = intOutputs.Select(o => DataManager.FormatOutputVector((double)o)).ToArray();
+            outputs = intOutputs.Select(o => DataManager.FormatOutputVector((double)o, _classCount)).ToArray();
 
             if (network == null)
             {
                 // Training.
-                network = DeepBeliefNetwork.Load(@"../../../data/network.dat");/* new DeepBeliefNetwork(inputs.First().Length, 100, 100, 100, 100, 2);
+                network = DeepBeliefNetwork.Load("../../../data/network.dat"); /*new DeepBeliefNetwork(inputs.First().Length, 100, 100, 100, 100, outputs.First().Length);
                 new NguyenWidrow(network).Randomize();
                 network.UpdateVisibleWeights();
                 network.Save(@"../../../data/network.dat");*/

From 348f594ae4b43422dcc08ca10d4dff5259e11da4 Mon Sep 17 00:00:00 2001
From: Kory Becker <kbecker@primaryobjects.com>
Date: Thu, 30 Jan 2014 17:31:58 -0500
Subject: [PATCH 4/8] Set to load all training data.

---
 DeepLearning/App.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/DeepLearning/App.config b/DeepLearning/App.config
index c58fcfc..6c9f6ff 100644
--- a/DeepLearning/App.config
+++ b/DeepLearning/App.config
@@ -4,7 +4,7 @@
     <add key="Width" value="35"/> <!-- Width of each training image -->
     <add key="Height" value="35"/> <!-- Height of each training image -->
     <add key="ClassCount" value="2"/> <!-- Number of unique classes -->
-    <add key="TrainCount" value="10"/> <!-- Max number of rows to read from the training set (useful for preparing learning curves). Set to 0 for all data. -->
+    <add key="TrainCount" value="0"/> <!-- Max number of rows to read from the training set (useful for preparing learning curves). Set to 0 for all data. -->
     <add key="EpochCount" value="10"/> <!-- Number of epochs for each neural network layer to train on. -->
     <add key="Sigma" value="12"/> <!-- SVM sigma training value, smoothes the gaussian ie., fuzziness factor (lower values = lower bias and higher variance and maybe overfitting, higher values = higher bias and lower variance). -->
     <add key="TrainPath" value="../../../data/catsdogs-train.csv"/> <!-- Path to training csv file -->

From 2e5276091ae78500514dfd21aa10a4900bcc1658 Mon Sep 17 00:00:00 2001
From: Kory Becker <kbecker@primaryobjects.com>
Date: Thu, 30 Jan 2014 17:33:04 -0500
Subject: [PATCH 5/8] Set epoch count to 600.

---
 DeepLearning/App.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/DeepLearning/App.config b/DeepLearning/App.config
index 6c9f6ff..9477b01 100644
--- a/DeepLearning/App.config
+++ b/DeepLearning/App.config
@@ -5,7 +5,7 @@
     <add key="Height" value="35"/> <!-- Height of each training image -->
     <add key="ClassCount" value="2"/> <!-- Number of unique classes -->
     <add key="TrainCount" value="0"/> <!-- Max number of rows to read from the training set (useful for preparing learning curves). Set to 0 for all data. -->
-    <add key="EpochCount" value="10"/> <!-- Number of epochs for each neural network layer to train on. -->
+    <add key="EpochCount" value="600"/> <!-- Number of epochs for each neural network layer to train on. -->
     <add key="Sigma" value="12"/> <!-- SVM sigma training value, smoothes the gaussian ie., fuzziness factor (lower values = lower bias and higher variance and maybe overfitting, higher values = higher bias and lower variance). -->
     <add key="TrainPath" value="../../../data/catsdogs-train.csv"/> <!-- Path to training csv file -->
     <add key="CvPath" value="../../../data/catsdogs-cv.csv"/> <!-- Path to CV csv file -->

From 1cca6f35dab7be227969ae56f330726a0953f2a7 Mon Sep 17 00:00:00 2001
From: Kory Becker <kbecker@primaryobjects.com>
Date: Fri, 31 Jan 2014 12:04:26 -0500
Subject: [PATCH 6/8] Added estimated time to finish training per layer. Added
 HiddenNeuronCount app.config parameter.

---
 DeepLearning/App.config |  3 ++-
 DeepLearning/Program.cs | 11 ++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/DeepLearning/App.config b/DeepLearning/App.config
index 9477b01..2d30a6e 100644
--- a/DeepLearning/App.config
+++ b/DeepLearning/App.config
@@ -5,7 +5,8 @@
     <add key="Height" value="35"/> <!-- Height of each training image -->
     <add key="ClassCount" value="2"/> <!-- Number of unique classes -->
     <add key="TrainCount" value="0"/> <!-- Max number of rows to read from the training set (useful for preparing learning curves). Set to 0 for all data. -->
-    <add key="EpochCount" value="600"/> <!-- Number of epochs for each neural network layer to train on. -->
+    <add key="EpochCount" value="20"/> <!-- Number of epochs for each neural network layer to train on. -->
+    <add key="HiddenNeuronCount" value="100"/> <!-- Number of neurons to include in each hidden layer. -->
     <add key="Sigma" value="12"/> <!-- SVM sigma training value, smoothes the gaussian ie., fuzziness factor (lower values = lower bias and higher variance and maybe overfitting, higher values = higher bias and lower variance). -->
     <add key="TrainPath" value="../../../data/catsdogs-train.csv"/> <!-- Path to training csv file -->
     <add key="CvPath" value="../../../data/catsdogs-cv.csv"/> <!-- Path to CV csv file -->
diff --git a/DeepLearning/Program.cs b/DeepLearning/Program.cs
index 0a3cf2c..d7464b4 100644
--- a/DeepLearning/Program.cs
+++ b/DeepLearning/Program.cs
@@ -30,6 +30,7 @@ class Program
         private static int _classCount = Int32.Parse(ConfigurationManager.AppSettings["ClassCount"]);
         private static int _trainCount = Int32.Parse(ConfigurationManager.AppSettings["TrainCount"]);
         private static int _epochCount = Int32.Parse(ConfigurationManager.AppSettings["EpochCount"]);
+        private static int _hiddenNeuronCount = Int32.Parse(ConfigurationManager.AppSettings["HiddenNeuronCount"]);
         private static double _sigma = Double.Parse(ConfigurationManager.AppSettings["Sigma"]);
         private static string _trainPath = ConfigurationManager.AppSettings["TrainPath"];
         private static string _cvPath = ConfigurationManager.AppSettings["CvPath"];
@@ -80,10 +81,10 @@ private static DeepBeliefNetwork RunDNN(string path, int count, int epochs, Deep
             if (network == null)
             {
                 // Training.
-                network = DeepBeliefNetwork.Load("../../../data/network.dat"); /*new DeepBeliefNetwork(inputs.First().Length, 100, 100, 100, 100, outputs.First().Length);
+                network = new DeepBeliefNetwork(inputs.First().Length, _hiddenNeuronCount, _hiddenNeuronCount, _hiddenNeuronCount, _hiddenNeuronCount, outputs.First().Length);
                 new NguyenWidrow(network).Randomize();
                 network.UpdateVisibleWeights();
-                network.Save(@"../../../data/network.dat");*/
+                network.Save(@"../../../data/network.dat");
 
                 // Setup the learning algorithm.
                 DeepBeliefNetworkLearning teacher = new DeepBeliefNetworkLearning(network)
@@ -118,7 +119,11 @@ private static DeepBeliefNetwork RunDNN(string path, int count, int epochs, Deep
                         if (i % 10 == 0)
                         {
                             TimeSpan timeSpan = DateTime.Now - epochStart;
-                            Console.WriteLine(i + ", Error = " + error + ", " + Math.Round(timeSpan.TotalMinutes) + "m (" + Math.Round(timeSpan.TotalSeconds) + "s)");
+							int epochsRemainingByTen = (int)Math.Round((double)(epochs - i) / (double)10);
+							double minutesRemaining = epochsRemainingByTen * timeSpan.TotalMinutes;
+							DateTime finishTime = DateTime.Now + TimeSpan.FromMinutes(minutesRemaining);
+							
+                            Console.WriteLine(i + ", Error = " + error + ", " + Math.Round(timeSpan.TotalMinutes) + "m (" + Math.Round(timeSpan.TotalSeconds) + "s), eta " + finishTime.ToShortTimeString());
                             epochStart = DateTime.Now;
                         }
                     }

From 0f388a332f7df20bc3297e2bfa6a2b79015ee390 Mon Sep 17 00:00:00 2001
From: Kory Becker <kbecker@primaryobjects.com>
Date: Fri, 31 Jan 2014 21:04:58 -0500
Subject: [PATCH 7/8] Added separate app.config parameters for hidden layer
 epochs and fine-tuning epochs. Use more epochs for unsupervised hidden
 training and less for the fine-tuning, just enough for classification.

---
 DeepLearning/App.config |  5 +++--
 DeepLearning/Program.cs | 41 ++++++++++++++++++++++++++++-------------
 2 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/DeepLearning/App.config b/DeepLearning/App.config
index 2d30a6e..352cb37 100644
--- a/DeepLearning/App.config
+++ b/DeepLearning/App.config
@@ -5,8 +5,9 @@
     <add key="Height" value="35"/> <!-- Height of each training image -->
     <add key="ClassCount" value="2"/> <!-- Number of unique classes -->
     <add key="TrainCount" value="0"/> <!-- Max number of rows to read from the training set (useful for preparing learning curves). Set to 0 for all data. -->
-    <add key="EpochCount" value="20"/> <!-- Number of epochs for each neural network layer to train on. -->
-    <add key="HiddenNeuronCount" value="100"/> <!-- Number of neurons to include in each hidden layer. -->
+    <add key="HiddenEpochCount" value="100"/> <!-- Number of epochs for each hidden neural network layer to train on (unsupervised). -->
+    <add key="FineTuneEpochCount" value="5"/> <!-- Number of epochs for fine-tuning entire neural network (supervised). -->
+    <add key="HiddenNeuronCount" value="200"/> <!-- Number of neurons to include in each hidden layer. -->
     <add key="Sigma" value="12"/> <!-- SVM sigma training value, smoothes the gaussian ie., fuzziness factor (lower values = lower bias and higher variance and maybe overfitting, higher values = higher bias and lower variance). -->
     <add key="TrainPath" value="../../../data/catsdogs-train.csv"/> <!-- Path to training csv file -->
     <add key="CvPath" value="../../../data/catsdogs-cv.csv"/> <!-- Path to CV csv file -->
diff --git a/DeepLearning/Program.cs b/DeepLearning/Program.cs
index d7464b4..ccccbd9 100644
--- a/DeepLearning/Program.cs
+++ b/DeepLearning/Program.cs
@@ -29,7 +29,8 @@ class Program
         private static int _pixelCount = Int32.Parse(ConfigurationManager.AppSettings["Width"]) * Int32.Parse(ConfigurationManager.AppSettings["Height"]);
         private static int _classCount = Int32.Parse(ConfigurationManager.AppSettings["ClassCount"]);
         private static int _trainCount = Int32.Parse(ConfigurationManager.AppSettings["TrainCount"]);
-        private static int _epochCount = Int32.Parse(ConfigurationManager.AppSettings["EpochCount"]);
+        private static int _hiddenEpochCount = Int32.Parse(ConfigurationManager.AppSettings["HiddenEpochCount"]);
+        private static int _fineTuneEpochCount = Int32.Parse(ConfigurationManager.AppSettings["FineTuneEpochCount"]);
         private static int _hiddenNeuronCount = Int32.Parse(ConfigurationManager.AppSettings["HiddenNeuronCount"]);
         private static double _sigma = Double.Parse(ConfigurationManager.AppSettings["Sigma"]);
         private static string _trainPath = ConfigurationManager.AppSettings["TrainPath"];
@@ -41,9 +42,9 @@ class Program
         static void Main(string[] args)
         {
             Console.WriteLine("-= Training =-");
-            var network = RunDNN(_trainPath, _trainCount, _epochCount);
+            var network = RunDNN(_trainPath, _trainCount, _hiddenEpochCount, _fineTuneEpochCount);
             Console.WriteLine("-= Cross Validation =-");
-            RunDNN(_cvPath, _trainCount, _epochCount, network);
+            RunDNN(_cvPath, _trainCount, _hiddenEpochCount, _fineTuneEpochCount, network);
 
             /*for (int count = 200; count < 4000; count += 200)
             {
@@ -63,15 +64,21 @@ static void Main(string[] args)
         /// </summary>
         /// <param name="path">string - path to csv file (training, csv, test).</param>
         /// <param name="count">int - max number of rows to process. This is useful for preparing learning curves, by using gradually increasing values. Use 0 to read all rows.</param>
-        /// <param name="epochs">int - max number of epochs per layer.</param>
+        /// <param name="hiddenEpochs">int - max number of epochs per hidden layer (unsupervised).</param>
+        /// <param name="fineTuneEpochs">int - max number of epochs over entire network during fine-tuning (supervised). Set to 0 to be the same as hiddenEpochs.</param>
         /// <param name="network">DeepBeliefNetwork - Leave null for initial training.</param>
         /// <returns>DeepBeliefNetwork</returns>
-        private static DeepBeliefNetwork RunDNN(string path, int count, int epochs, DeepBeliefNetwork network = null)
+        private static DeepBeliefNetwork RunDNN(string path, int count, int hiddenEpochs, int fineTuneEpochs = 0, DeepBeliefNetwork network = null)
         {
             double[][] inputs;
             double[][] outputs;
             int[] intOutputs;
 
+            if (fineTuneEpochs == 0)
+            {
+                fineTuneEpochs = hiddenEpochs;
+            }
+
             // Parse the csv file to get inputs and outputs.
             ReadData(path, count, out inputs, out intOutputs, new EndStringEndLabelParser());
             
@@ -84,7 +91,7 @@ private static DeepBeliefNetwork RunDNN(string path, int count, int epochs, Deep
                 network = new DeepBeliefNetwork(inputs.First().Length, _hiddenNeuronCount, _hiddenNeuronCount, _hiddenNeuronCount, _hiddenNeuronCount, outputs.First().Length);
                 new NguyenWidrow(network).Randomize();
                 network.UpdateVisibleWeights();
-                network.Save(@"../../../data/network.dat");
+                network.Save(@"../../../data/network1.dat");
 
                 // Setup the learning algorithm.
                 DeepBeliefNetworkLearning teacher = new DeepBeliefNetworkLearning(network)
@@ -113,14 +120,14 @@ private static DeepBeliefNetwork RunDNN(string path, int count, int epochs, Deep
                 {
                     teacher.LayerIndex = layerIndex;
                     layerData = teacher.GetLayerInput(batches);
-                    for (int i = 0; i < epochs; i++)
+                    for (int i = 0; i < hiddenEpochs; i++)
                     {
                         double error = teacher.RunEpoch(layerData) / inputs.Length;
-                        if (i % 10 == 0)
+                        if (i % 2 == 0)
                         {
                             TimeSpan timeSpan = DateTime.Now - epochStart;
-							int epochsRemainingByTen = (int)Math.Round((double)(epochs - i) / (double)10);
-							double minutesRemaining = epochsRemainingByTen * timeSpan.TotalMinutes;
+							int epochsRemainingByCount = (int)Math.Round((double)(hiddenEpochs - i) / (double)2);
+							double minutesRemaining = epochsRemainingByCount * timeSpan.TotalMinutes;
 							DateTime finishTime = DateTime.Now + TimeSpan.FromMinutes(minutesRemaining);
 							
                             Console.WriteLine(i + ", Error = " + error + ", " + Math.Round(timeSpan.TotalMinutes) + "m (" + Math.Round(timeSpan.TotalSeconds) + "s), eta " + finishTime.ToShortTimeString());
@@ -129,6 +136,8 @@ private static DeepBeliefNetwork RunDNN(string path, int count, int epochs, Deep
                     }
                 }
 
+                network.Save(@"../../../data/network2.dat");
+
                 // Supervised learning on entire network, to provide output classification.
                 var teacher2 = new BackPropagationLearning(network)
                 {
@@ -139,17 +148,23 @@ private static DeepBeliefNetwork RunDNN(string path, int count, int epochs, Deep
                 epochStart = DateTime.Now;
 
                 // Run supervised learning.
-                for (int i = 0; i < epochs; i++)
+                for (int i = 0; i < fineTuneEpochs; i++)
                 {
                     double error = teacher2.RunEpoch(inputs, outputs) / inputs.Length;
-                    if (i % 10 == 0)
+                    if (i % 2 == 0)
                     {
                         TimeSpan timeSpan = DateTime.Now - epochStart;
-                        Console.WriteLine(i + ", Error = " + error + ", " + Math.Round(timeSpan.TotalMinutes) + "m (" + Math.Round(timeSpan.TotalSeconds) + "s)");
+                        int epochsRemainingByCount = (int)Math.Round((double)(fineTuneEpochs - i) / (double)2);
+                        double minutesRemaining = epochsRemainingByCount * timeSpan.TotalMinutes;
+                        DateTime finishTime = DateTime.Now + TimeSpan.FromMinutes(minutesRemaining);
+
+                        Console.WriteLine(i + ", Error = " + error + ", " + Math.Round(timeSpan.TotalMinutes) + "m (" + Math.Round(timeSpan.TotalSeconds) + "s), eta " + finishTime.ToShortTimeString());
                         epochStart = DateTime.Now;
                     }
                 }
 
+                network.Save(@"../../../data/network3.dat");
+
                 TimeSpan runTime = DateTime.Now - startTime;
                 Console.WriteLine("Training completed after " + runTime.TotalMinutes + "m.");
                 startTime = DateTime.Now;

From bb0eeaf3c850ac27bc5363cdcb6012f1f9c9d21c Mon Sep 17 00:00:00 2001
From: Kory Becker <kbecker@primaryobjects.com>
Date: Sat, 1 Feb 2014 19:05:51 -0500
Subject: [PATCH 8/8] Saves network after unsupervised training, for
 continuation. Added SaveOutput method for neural network.

---
 DeepLearning/Accuracy.cs | 22 ++++++++++++++++++++++
 DeepLearning/App.config  |  6 +++---
 DeepLearning/Program.cs  |  6 ++++++
 3 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/DeepLearning/Accuracy.cs b/DeepLearning/Accuracy.cs
index d6b5a97..cc56c56 100644
--- a/DeepLearning/Accuracy.cs
+++ b/DeepLearning/Accuracy.cs
@@ -74,5 +74,27 @@ public static int SaveOutput(MulticlassSupportVectorMachine machine, double[][]
 
             return inputs.Length;
         }
+
+        /// <summary>
+        /// Calculates the output for the neural network and saves each result to a text file, one per line.
+        /// </summary>
+        /// <param name="machine">DeepBeliefNetwork</param>
+        /// <param name="inputs">double[][]</param>
+        /// <param name="path">string</param>
+        /// <returns>int - number of rows processed</returns>
+        public static int SaveOutput(DeepBeliefNetwork network, double[][] inputs, string path)
+        {
+            File.AppendAllText(path, "ImageId,Label\r\n");
+
+            for (int i = 0; i < inputs.Length; i++)
+            {
+                double[] outputValues = network.Compute(inputs[i]);
+                double output = DataManager.FormatOutputResult(outputValues);
+                
+                File.AppendAllText(path, (i + 1) + "," + output.ToString() + "\r\n");
+            }
+
+            return inputs.Length;
+        }
     }
 }
diff --git a/DeepLearning/App.config b/DeepLearning/App.config
index 352cb37..b30b34c 100644
--- a/DeepLearning/App.config
+++ b/DeepLearning/App.config
@@ -5,13 +5,13 @@
     <add key="Height" value="35"/> <!-- Height of each training image -->
     <add key="ClassCount" value="2"/> <!-- Number of unique classes -->
     <add key="TrainCount" value="0"/> <!-- Max number of rows to read from the training set (useful for preparing learning curves). Set to 0 for all data. -->
-    <add key="HiddenEpochCount" value="100"/> <!-- Number of epochs for each hidden neural network layer to train on (unsupervised). -->
-    <add key="FineTuneEpochCount" value="5"/> <!-- Number of epochs for fine-tuning entire neural network (supervised). -->
+    <add key="HiddenEpochCount" value="300"/> <!-- Number of epochs for each hidden neural network layer to train on (unsupervised). -->
+    <add key="FineTuneEpochCount" value="10"/> <!-- Number of epochs for fine-tuning entire neural network (supervised). -->
     <add key="HiddenNeuronCount" value="200"/> <!-- Number of neurons to include in each hidden layer. -->
     <add key="Sigma" value="12"/> <!-- SVM sigma training value, smoothes the gaussian ie., fuzziness factor (lower values = lower bias and higher variance and maybe overfitting, higher values = higher bias and lower variance). -->
     <add key="TrainPath" value="../../../data/catsdogs-train.csv"/> <!-- Path to training csv file -->
     <add key="CvPath" value="../../../data/catsdogs-cv.csv"/> <!-- Path to CV csv file -->
-    <add key="TestPath" value="../../../data/catsdogs_test.csv"/> <!-- Path to test csv file -->
+    <add key="TestPath" value="../../../data/catsdogs-test.csv"/> <!-- Path to test csv file -->
   </appSettings>
   <startup> 
         <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
diff --git a/DeepLearning/Program.cs b/DeepLearning/Program.cs
index ccccbd9..25e2b2a 100644
--- a/DeepLearning/Program.cs
+++ b/DeepLearning/Program.cs
@@ -134,6 +134,12 @@ private static DeepBeliefNetwork RunDNN(string path, int count, int hiddenEpochs
                             epochStart = DateTime.Now;
                         }
                     }
+
+                    if (layerIndex == 0)
+                    {
+                        // Save a copy of the first layer unsupervised trained, so we can continue unsupervised training if we want.
+                        network.Save(@"../../../data/network1a.dat");
+                    }
                 }
 
                 network.Save(@"../../../data/network2.dat");