Merge pull request #76 from Sergio0694/dev

Computation graph and new APIs

Author: Sergio0694

NeuralNetwork.NET/APIs/CuDnnNetworkLayers.cs

@@ -20,7 +20,6 @@ public static class CuDnnNetworkLayers
         /// </summary>
         public static bool IsCudaSupportAvailable
         {
-            [Pure]
             get
             {
                 try
@@ -63,6 +62,20 @@ public static LayerFactory Softmax(
             WeightsInitializationMode weightsMode = WeightsInitializationMode.GlorotUniform, BiasInitializationMode biasMode = BiasInitializationMode.Zero) 
             => input => new CuDnnSoftmaxLayer(input, outputs, weightsMode, biasMode);
 
+        /// <summary>
+        /// Creates a convolutional layer with the desired number of kernels
+        /// </summary>
+        /// <param name="kernel">The volume information of the kernels used in the layer</param>
+        /// <param name="kernels">The number of convolution kernels to apply to the input volume</param>
+        /// <param name="activation">The desired activation function to use in the network layer</param>
+        /// <param name="biasMode">Indicates the desired initialization mode to use for the layer bias values</param>
+        [PublicAPI]
+        [Pure, NotNull]
+        public static LayerFactory Convolutional(
+            (int X, int Y) kernel, int kernels, ActivationFunctionType activation,
+            BiasInitializationMode biasMode = BiasInitializationMode.Zero) 
+            => input => new CuDnnConvolutionalLayer(input, ConvolutionInfo.Default, kernel, kernels, activation, biasMode);
+
         /// <summary>
         /// Creates a convolutional layer with the desired number of kernels
         /// </summary>
@@ -81,6 +94,14 @@ public static LayerFactory Convolutional(
         /// <summary>
         /// Creates a pooling layer with a window of size 2 and a stride of 2
         /// </summary>
+        /// <param name="activation">The desired activation function to use in the network layer</param>
+        [PublicAPI]
+        [Pure, NotNull]
+        public static LayerFactory Pooling(ActivationFunctionType activation) => input => new CuDnnPoolingLayer(input, PoolingInfo.Default, activation);
+
+        /// <summary>
+        /// Creates a pooling layer with a custom mode, window size and stride
+        /// </summary>
         /// <param name="info">The info on the pooling operation to perform</param>
         /// <param name="activation">The desired activation function to use in the network layer</param>
         [PublicAPI]

NeuralNetwork.NET/APIs/Enums/ComputationGraphNodeType.cs

@@ -0,0 +1,33 @@
+namespace NeuralNetworkNET.APIs.Enums
+{
+    /// <summary>
+    /// An <see langword="enum"/> that indicates the type of a node in a graph network
+    /// </summary>
+    public enum ComputationGraphNodeType : byte
+    {
+        /// <summary>
+        /// The root node for a computation graph, that forwards the network inputs through the computation pipeline(s)
+        /// </summary>
+        Input,
+
+        /// <summary>
+        /// A computation graph node with an associated <see cref="Interfaces.INetworkLayer"/> that processes an input <see cref="Structs.Tensor"/>
+        /// </summary>
+        Processing,
+
+        /// <summary>
+        /// The root node for a training sub-graph, a secondary training branch used during backpropagation to inject partial gradients
+        /// </summary>
+        TrainingBranch,
+
+        /// <summary>
+        /// A computation graph node that merges a series of inputs by stacking them along the depth axis
+        /// </summary>
+        DepthConcatenation,
+
+        /// <summary>
+        /// A computation graph node that merges a series of inputs by summing their values together
+        /// </summary>
+        Sum
+    }
+}

NeuralNetwork.NET/APIs/Enums/ExecutionModePreference.cs

@@ -0,0 +1,20 @@
+namespace NeuralNetworkNET.APIs.Enums
+{
+    /// <summary>
+    /// Indicates the preferred mode of execution for all expensive operations in the library
+    /// </summary>
+    public enum ExecutionModePreference
+    {
+        /// <summary>
+        /// Only CPU-powered functions are executed. This means that network layers are deserialized through the <see cref="NetworkLayers"/> class,
+        /// and that all other computations will only be scheduled on the CPU.
+        /// </summary>
+        Cpu,
+        
+        /// <summary>
+        /// CUDA-powered functions are supported. When using this mode, network layers are deserialized through the <see cref="CuDnnNetworkLayers"/> class,
+        /// and all other available operations will be scheduled on the GPU.
+        /// </summary>
+        Cuda
+    }
+}

NeuralNetwork.NET/APIs/Enums/LayersLoadingPreference.cs

@@ -8,6 +8,11 @@ public enum NetworkType : byte
         /// <summary>
         /// The classic network model, consisting in a linear stack of connected layers
         /// </summary>
-        Sequential
+        Sequential,
+
+        /// <summary>
+        /// A network with a custom computation graph used to process its inputs
+        /// </summary>
+        ComputationGraph
     }
 }

NeuralNetwork.NET/APIs/Enums/NetworkType.cs

@@ -1,6 +1,8 @@
 using System;
 using JetBrains.Annotations;
 using NeuralNetworkNET.SupervisedLearning.Progress;
+using SixLabors.ImageSharp;
+using SixLabors.ImageSharp.PixelFormats;
 
 namespace NeuralNetworkNET.APIs.Interfaces.Data
 {
@@ -19,6 +21,39 @@ public interface ITrainingDataset : IDataset
         /// </summary>
         int BatchesCount { get; }
 
+        /// <summary>
+        /// Artificially expands the current dataset applying the input transformation to each sample to create new ones
+        /// </summary>
+        /// <param name="factories">The list of functions to use to generate new samples from each one in the dataset</param>
+        void Expand([NotNull, ItemNotNull] params Func<float[], float[]>[] factories);
+
+        /// <summary>
+        /// Artificially expands the current dataset by reinterpreting each sample as an image of the specified type and applying the input transformation to each sample to create new ones
+        /// </summary>
+        /// <typeparam name="TPixel">The type of image pixels. It must be either <see cref="Alpha8"/>, <see cref="Rgb24"/> or <see cref="Argb32"/></typeparam>
+        /// <param name="width">The width of each sample image</param>
+        /// <param name="height">The height of each sample image</param>
+        /// <param name="factories">The list of functions to use to process the images and generate new samples</param>
+        void Expand<TPixel>(int width, int height, [NotNull, ItemNotNull] params Action<IImageProcessingContext<TPixel>>[] factories) where TPixel : struct, IPixel<TPixel>;
+
+        /// <summary>
+        /// Removes a specified fraction of samples from the current instance and returns a new <see cref="ITestDataset"/>
+        /// </summary>
+        /// <param name="ratio">The ratio of samples to include in the returned <see cref="ITrainingDataset"/></param>
+        /// <param name="progress">The optional progress callback to use</param>
+        [Pure, NotNull]
+        ITestDataset ExtractTest(float ratio, [CanBeNull] Action<TrainingProgressEventArgs> progress = null);
+
+        /// <summary>
+        /// Returns a pair of new datasets, where the first is an <see cref="ITrainingDataset"/> with the specified fraction of samples and 
+        /// the second is an <see cref="IValidationDataset"/> with the remaining number of samples from the current dataset
+        /// </summary>
+        /// <param name="ratio">The ratio of samples to include in the returned <see cref="ITrainingDataset"/></param>
+        /// <param name="tolerance">The desired tolerance to test the network for convergence</param>
+        /// <param name="epochs">The epochs interval to consider when testing the network for convergence</param>
+        [Pure, NotNull]
+        IValidationDataset ExtractValidation(float ratio, float tolerance = 1e-2f, int epochs = 5);
+
         /// <summary>
         /// Returns a pair of new datasets, where the first is an <see cref="ITrainingDataset"/> with the specified fraction of samples and 
         /// the second is an <see cref="ITestDataset"/> with the remaining number of samples from the current dataset

NeuralNetwork.NET/APIs/Interfaces/Data/ITrainingDataset.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using JetBrains.Annotations;
+using NeuralNetworkNET.APIs.Enums;
+
+namespace NeuralNetworkNET.APIs.Interfaces
+{
+    /// <summary>
+    /// The base <see langword="interface"/> for the various types of nodes in a graph network
+    /// </summary>
+    public interface IComputationGraphNode : IEquatable<IComputationGraphNode>
+    {
+        /// <summary>
+        /// Indicates the type of the current node
+        /// </summary>
+        ComputationGraphNodeType Type { get; }
+
+        /// <summary>
+        /// Gets the collection of child nodes for the current computation graph node
+        /// </summary>
+        [NotNull, ItemNotNull]
+        IReadOnlyList<IComputationGraphNode> Children { get; }
+    }
+}

NeuralNetwork.NET/APIs/Interfaces/IComputationGraphNode.cs

@@ -3,6 +3,7 @@
 using System.IO;
 using JetBrains.Annotations;
 using NeuralNetworkNET.APIs.Enums;
+using NeuralNetworkNET.APIs.Interfaces.Data;
 using NeuralNetworkNET.APIs.Structs;
 
 namespace NeuralNetworkNET.APIs.Interfaces
@@ -35,11 +36,21 @@ public interface INeuralNetwork : IEquatable<INeuralNetwork>, IClonable<INeuralN
         [NotNull, ItemNotNull]
         IReadOnlyList<INetworkLayer> Layers { get; }
 
+        /// <summary>
+        /// Gets the number of units in the current network (layers or graph nodes)
+        /// </summary>
+        int Size { get; }
+
         /// <summary>
         /// Gets the total number of parameters in the current network layer
         /// </summary>
         int Parameters { get; }
 
+        /// <summary>
+        /// Gets whether or not a numeric overflow has occurred in the network
+        /// </summary>
+        bool IsInNumericOverflow { get; }
+
         #endregion
 
         #region Methods
@@ -96,6 +107,12 @@ public interface INeuralNetwork : IEquatable<INeuralNetwork>, IClonable<INeuralN
         [CollectionAccess(CollectionAccessType.Read)]
         float CalculateCost([NotNull] float[,] x, [NotNull] float[,] y);
 
+        /// <summary>
+        /// Calculates the current network performances with the given dataset
+        /// </summary>
+        /// <param name="dataset">The <see cref="IDataset"/> instance to evaluate</param>
+        (float Cost, int Classified, float Accuracy) Evaluate([NotNull] IDataset dataset);
+
         #endregion
 
         #region Serialization

NeuralNetwork.NET/APIs/Interfaces/INeuralNetwork.cs

@@ -29,10 +29,18 @@ public static class NetworkLoader
         /// <returns>The deserialized network, or <see langword="null"/> if the operation fails</returns>
         [PublicAPI]
         [Pure, CanBeNull]
-        public static INeuralNetwork TryLoad([NotNull] FileInfo file, LayersLoadingPreference preference)
+        public static INeuralNetwork TryLoad([NotNull] FileInfo file, ExecutionModePreference preference)
         {
-            using (FileStream stream = file.OpenRead())
-                return TryLoad(stream, preference);
+            try
+            {
+                using (FileStream stream = file.OpenRead())
+                    return TryLoad(stream, preference);
+            }
+            catch (FileNotFoundException)
+            {
+                // Just keep going
+                return null;
+            }
         }
 
         /// <summary>
@@ -43,7 +51,7 @@ public static INeuralNetwork TryLoad([NotNull] FileInfo file, LayersLoadingPrefe
         /// <returns>The deserialized network, or <see langword="null"/> if the operation fails</returns>
         [PublicAPI]
         [Pure, CanBeNull]
-        public static INeuralNetwork TryLoad([NotNull] Stream stream, LayersLoadingPreference preference)
+        public static INeuralNetwork TryLoad([NotNull] Stream stream, ExecutionModePreference preference)
         {
             try
             {
@@ -53,6 +61,7 @@ public static INeuralNetwork TryLoad([NotNull] Stream stream, LayersLoadingPrefe
                     switch (model)
                     {
                         case NetworkType.Sequential: return SequentialNetwork.Deserialize(gzip, preference);
+                        case NetworkType.ComputationGraph: return ComputationGraphNetwork.Deserialize(gzip, preference);
                         default: return null;
                     }
                 }

NeuralNetwork.NET/APIs/NetworkLoader.cs

@@ -10,6 +10,7 @@
 using NeuralNetworkNET.APIs.Results;
 using NeuralNetworkNET.APIs.Structs;
 using NeuralNetworkNET.Extensions;
+using NeuralNetworkNET.Networks.Graph;
 using NeuralNetworkNET.Networks.Implementations;
 using NeuralNetworkNET.SupervisedLearning.Data;
 using NeuralNetworkNET.SupervisedLearning.Optimization;
@@ -41,6 +42,21 @@ public static INeuralNetwork NewSequential(TensorInfo input, [NotNull, ItemNotNu
             }).ToArray());
         }
 
+        /// <summary>
+        /// Creates a computational graph network with a custom structure
+        /// </summary>
+        /// <param name="input">The input <see cref="TensorInfo"/> description</param>
+        /// <param name="builder">An <see cref="Action{T}"/> used to build the graph from the input <see cref="NodeBuilder"/> node</param>
+        [PublicAPI]
+        [Pure, NotNull]
+        public static INeuralNetwork NewGraph(TensorInfo input, [NotNull] Action<NodeBuilder> builder)
+        {
+            NodeBuilder root = NodeBuilder.Input();
+            builder(root);
+            ComputationGraph graph = ComputationGraph.New(input, root);
+            return new ComputationGraphNetwork(graph);
+        }
+
         #region Training APIs
 
         /// <summary>
@@ -126,7 +142,7 @@ private static TrainingSessionResult TrainNetworkCore(
 
             // Start the training
             return NetworkTrainer.TrainNetwork(
-                network as SequentialNetwork ?? throw new ArgumentException("The input network instance isn't valid", nameof(network)), 
+                network as NeuralNetworkBase ?? throw new ArgumentException("The input network instance isn't valid", nameof(network)), 
                 dataset as BatchesCollection ?? throw new ArgumentException("The input dataset instance isn't valid", nameof(dataset)),
                 epochs, dropout, algorithm, batchProgress, trainingProgress, 
                 validationDataset as ValidationDataset,