To First Real Release

Author: IrishFix

CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.0.0] - 2023-05-04
+
+### Added
+
+- Regularization to the Networking (Dropout, L1 and L2)
+- Newtonsoft.Json dll to allow Architecture gathering
+
 ## [0.8.0] - 2023-05-03
 
 ### Changed
@@ -122,7 +129,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - PointComputations.cs, to facilitate convex hull computation, point cloud simplification, and continuing on, any solely point-based methods.
 - Conversion.cs, to allow conversion of Vector types.
 
-[unreleased]: https://github.com/IrishFix/Computational-Geometry/compare/v0.8.0...HEAD
+[unreleased]: https://github.com/IrishFix/Computational-Geometry/compare/v1.0.0...HEAD
+[0.8.0]: https://github.com/IrishFix/Computational-Geometry/compare/v0.8.0...v1.0.0
 [0.8.0]: https://github.com/IrishFix/Computational-Geometry/compare/v0.7.0...v0.8.0
 [0.7.0]: https://github.com/IrishFix/Computational-Geometry/compare/v0.6.0...v0.7.0
 [0.6.0]: https://github.com/IrishFix/Computational-Geometry/compare/v0.5.0...v0.6.0

Runtime/Networking/Layers/Dense.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using UniversalNumerics.Networking.Activations;
+using UniversalNumerics.Networking.Regularizers;
 
 // ReSharper disable MemberCanBePrivate.Global
 // ReSharper disable once CheckNamespace
@@ -13,6 +14,7 @@ public class Dense : ILayer {
         public int OutputCount;
 
         public IActivationFunction Activation;
+        public IRegularizer Regularization;
 
         public double[][] Weights;
         public double[] Biases;
@@ -23,10 +25,11 @@ public class Dense : ILayer {
         public double[][] WeightsGradients;
         public double[] BiasesGradients;
 
-        public Dense(int NumNeurons, IActivationFunction ActivationFunc) {
+        public Dense(int NumNeurons, IActivationFunction ActivationFunc, IRegularizer Regularizer=null) {
             InputCount = 0;
             OutputCount = NumNeurons;
             Activation = ActivationFunc;
+            Regularization = Regularizer;
         }
 
         public double[][] Forward(double[][] Batch) {
@@ -49,6 +52,7 @@ public double[][] Forward(double[][] Batch) {
 
             Outputs = Activation.BatchActivate(preActivation);;
             Inputs = Batch;
+
             return Outputs;
         }
 

Runtime/Networking/MLP.cs

@@ -37,7 +37,7 @@ public MLP(MLP templateNetwork) {
                     }
                 }
 
-                Dense clonedLayer = new Dense(layer.OutputCount, layer.Activation) {
+                Dense clonedLayer = new Dense(layer.OutputCount, layer.Activation, layer.Regularization) {
                     InputCount = layer.InputCount,
                     Weights = clonedWeights,
                     Biases = layer.Biases.ToArray()
@@ -81,6 +81,10 @@ public MLP Fit(double[][][] X, double[][][] y, int Epochs, IOptimizer Optimizer)
             for (int Epoch = 0; Epoch < Epochs; Epoch++) {
                 double epochError = 0;
                 for (int i = 0; i < inputCount; i++) {
+                    foreach (Dense Layer in Layers) {
+                        Layer.Regularization?.Regularize(Layer);
+                    }
+                    
                     double[][] output = Forward(X[i]);
                     double error = Loss.MSE(y[i], output);
 

Runtime/Networking/Regularizers.meta

@@ -0,0 +1,25 @@
+using System;
+using UniversalNumerics.Networking.Layers;
+
+// ReSharper disable once CheckNamespace
+namespace UniversalNumerics.Networking.Regularizers {
+    public class Dropout : IRegularizer {
+        private readonly double DropoutRate;
+        private readonly Random Rand;
+
+        public Dropout(double dropoutRate) {
+            DropoutRate = dropoutRate;
+            Rand = new Random();
+        }
+
+        public void Regularize(Dense layer) {
+            for (int i = 0; i < layer.OutputCount; i++) {
+                for (int j = 0; j < layer.InputCount; j++) {
+                    if (Rand.NextDouble() < DropoutRate) {
+                        layer.Weights[i][j] = 0;
+                    }
+                }
+            }
+        }
+    }
+}

Runtime/Networking/Regularizers/Dropout.cs

@@ -0,0 +1,8 @@
+using UniversalNumerics.Networking.Layers;
+
+// ReSharper disable once CheckNamespace
+namespace UniversalNumerics.Networking.Regularizers {
+    public interface IRegularizer {
+        public void Regularize(Dense Layer);
+    }
+}

Runtime/Networking/Regularizers/Dropout.cs.meta

@@ -0,0 +1,22 @@
+using System;
+using UniversalNumerics.Networking.Layers;
+
+// ReSharper disable once CheckNamespace
+namespace UniversalNumerics.Networking.Regularizers {
+    public class L1: IRegularizer {
+        private readonly double Strength;
+
+        public L1(double strength) {
+            Strength = strength;
+        }
+
+        public void Regularize(Dense layer) {
+            for (int i = 0; i < layer.OutputCount; i++) {
+                for (int j = 0; j < layer.InputCount; j++) {
+                    double sign = Math.Sign(layer.Weights[i][j]);
+                    layer.Weights[i][j] -= Strength * sign;
+                }
+            }
+        }
+    }
+}