From 741cf1ced699f0c68cb9e341e3aecfc45cfd8b79 Mon Sep 17 00:00:00 2001 From: Adrien Burgun Date: Fri, 28 Apr 2023 12:56:10 +0200 Subject: [PATCH] :fire: :art: Remove deprecated traits (NeuraOldTrainableNetwork and NeuraGradientSolver*) --- src/gradient_solver/backprop.rs | 45 ------------ src/gradient_solver/forward_forward.rs | 81 ++------------------- src/gradient_solver/mod.rs | 33 +-------- src/network/mod.rs | 45 ------------ src/network/sequential/mod.rs | 97 +------------------------- src/network/traits.rs | 4 +- src/train.rs | 5 +- 7 files changed, 13 insertions(+), 297 deletions(-) diff --git a/src/gradient_solver/backprop.rs b/src/gradient_solver/backprop.rs index 6832e79..c88021b 100644 --- a/src/gradient_solver/backprop.rs +++ b/src/gradient_solver/backprop.rs @@ -45,51 +45,6 @@ where } } -impl NeuraGradientSolverBase for (&NeuraBackprop, &Target) { - type Output = (NetworkInput, NetworkGradient); // epsilon, gradient -} - -impl> - NeuraGradientSolverFinal for (&NeuraBackprop, &Target) -{ - fn eval_final(&self, output: LayerOutput) -> Self::Output { - (self.0.loss.nabla(self.1, &output), ()) - } -} - -impl< - Input, - Target, - Loss, - Layer: NeuraTrainableLayerBackprop + NeuraTrainableLayerSelf, - > NeuraGradientSolverTransient for (&NeuraBackprop, &Target) -{ - fn eval_layer( - &self, - layer: &Layer, - input: &Input, - _output: &Layer::Output, - intermediary: &Layer::IntermediaryRepr, - rec_opt_output: Self::Output, - combine_gradients: impl Fn(Layer::Gradient, RecGradient) -> NetworkGradient, - ) -> Self::Output { - let (epsilon_in, rec_gradient) = rec_opt_output; - - let epsilon_out = layer.backprop_layer(input, intermediary, &epsilon_in); - let layer_gradient = layer.get_gradient(input, intermediary, &epsilon_in); - - (epsilon_out, combine_gradients(layer_gradient, rec_gradient)) - } - - fn map_epsilon To>( - &self, - rec_opt_output: Self::Output, - callback: Cb, - ) -> Self::Output { - (callback(rec_opt_output.0), rec_opt_output.1) - } -} - trait BackpropRecurse { fn recurse(&self, network: &Network, input: &Input) -> (Input, Gradient); } diff --git a/src/gradient_solver/forward_forward.rs b/src/gradient_solver/forward_forward.rs index 0bb6bfd..dce173f 100644 --- a/src/gradient_solver/forward_forward.rs +++ b/src/gradient_solver/forward_forward.rs @@ -1,5 +1,5 @@ use nalgebra::{DVector, Scalar}; -use num::{traits::NumAssignOps, Float, ToPrimitive}; +use num::{traits::NumAssignOps, Float}; use crate::{ derivable::NeuraDerivable, @@ -65,16 +65,13 @@ struct NeuraForwardPair { activation: Act, } -impl, Input: Clone, Trainable: NeuraTrainableLayerBase> - NeuraGradientSolver for NeuraForwardForward +impl< + F: Float, + Act: Clone + NeuraDerivable, + Input: Clone, + Trainable: NeuraTrainableLayerBase + NeuraLayer>, + > NeuraGradientSolver for NeuraForwardForward where - F: ToPrimitive, - Trainable: NeuraOldTrainableNetwork< - Input, - NeuraForwardPair, - Output = DVector, - Gradient = ::Gradient, - >, NeuraForwardPair: ForwardForwardRecurse::Gradient>, { @@ -126,70 +123,6 @@ where } } -impl NeuraGradientSolverBase for NeuraForwardPair { - type Output = NetworkGradient; -} - -impl NeuraGradientSolverFinal for NeuraForwardPair { - fn eval_final(&self, _output: LayerOutput) -> Self::Output { - () - } -} - -impl< - F: Float + Scalar + NumAssignOps, - Act: NeuraDerivable, - Input, - Layer: NeuraTrainableLayerSelf>, - > NeuraGradientSolverTransient for NeuraForwardPair -{ - fn eval_layer( - &self, - layer: &Layer, - input: &Input, - output: &Layer::Output, - intermediary: &Layer::IntermediaryRepr, - rec_gradient: RecGradient, - combine_gradients: impl Fn(Layer::Gradient, RecGradient) -> NetworkGradient, - ) -> Self::Output { - // let output = layer.eval(input); - let goodness = output - .iter() - .copied() - .reduce(|acc, x| acc + x * x) - .unwrap_or(F::zero()); - let goodness = if self.maximize { - goodness - F::from(self.threshold).unwrap() - } else { - F::from(self.threshold).unwrap() - goodness - }; - // We skip self.activation.eval(goodness) - - let two = F::from(2.0).unwrap(); - - // The original formula does not have a 1/2 term, - // so we must multiply by 2 - let mut goodness_derivative = output * (two * self.activation.derivate(goodness)); - - if self.maximize { - goodness_derivative = -goodness_derivative; - } - - // TODO: split backprop_layer into eval_training, get_gradient and get_backprop - let layer_gradient = layer.get_gradient(input, intermediary, &goodness_derivative); - - combine_gradients(layer_gradient, rec_gradient) - } - - fn map_epsilon To>( - &self, - rec_opt_output: Self::Output, - _callback: Cb, - ) -> Self::Output { - rec_opt_output - } -} - trait ForwardForwardRecurse { fn recurse(&self, network: &Network, input: &Input) -> Gradient; } diff --git a/src/gradient_solver/mod.rs b/src/gradient_solver/mod.rs index 126bd2f..740e0a2 100644 --- a/src/gradient_solver/mod.rs +++ b/src/gradient_solver/mod.rs @@ -4,38 +4,7 @@ pub use backprop::NeuraBackprop; mod forward_forward; pub use forward_forward::NeuraForwardForward; -use crate::{ - layer::{NeuraTrainableLayerBase, NeuraTrainableLayerEval}, - network::{NeuraOldTrainableNetwork, NeuraOldTrainableNetworkBase}, -}; - -pub trait NeuraGradientSolverBase { - type Output; -} - -pub trait NeuraGradientSolverFinal: NeuraGradientSolverBase { - fn eval_final(&self, output: LayerOutput) -> Self::Output; -} - -pub trait NeuraGradientSolverTransient>: - NeuraGradientSolverBase -{ - fn eval_layer( - &self, - layer: &Layer, - input: &Input, - output: &Layer::Output, - layer_intermediary: &Layer::IntermediaryRepr, - rec_opt_output: Self::Output, - combine_gradients: impl Fn(Layer::Gradient, RecGradient) -> NetworkGradient, - ) -> Self::Output; - - fn map_epsilon To>( - &self, - rec_opt_output: Self::Output, - callback: Cb, - ) -> Self::Output; -} +use crate::layer::{NeuraTrainableLayerBase, NeuraTrainableLayerEval}; pub trait NeuraGradientSolver { fn get_gradient( diff --git a/src/network/mod.rs b/src/network/mod.rs index c6a89a2..c4b7962 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -1,50 +1,5 @@ -use crate::{ - algebra::NeuraVectorSpace, - gradient_solver::{NeuraGradientSolverBase, NeuraGradientSolverFinal}, - layer::NeuraLayer, -}; - // pub mod residual; pub mod sequential; mod traits; pub use traits::*; - -// TODO: extract regularize from this, so that we can drop the trait constraints on NeuraSequential's impl -pub trait NeuraOldTrainableNetworkBase: NeuraLayer { - type Gradient: NeuraVectorSpace; - type LayerOutput; - - fn default_gradient(&self) -> Self::Gradient; - - fn apply_gradient(&mut self, gradient: &Self::Gradient); - - /// Should return the regularization gradient - fn regularize(&self) -> Self::Gradient; - - /// Called before an iteration begins, to allow the network to set itself up for training or not. - fn prepare(&mut self, train_iteration: bool); -} - -pub trait NeuraOldTrainableNetwork: NeuraOldTrainableNetworkBase -where - Optimizer: NeuraGradientSolverBase, -{ - fn traverse( - &self, - input: &Input, - optimizer: &Optimizer, - ) -> Optimizer::Output; -} - -impl> - NeuraOldTrainableNetwork for () -{ - fn traverse( - &self, - input: &Input, - optimizer: &Optimizer, - ) -> Optimizer::Output { - optimizer.eval_final(input.clone()) - } -} diff --git a/src/network/sequential/mod.rs b/src/network/sequential/mod.rs index b34966c..deedf8c 100644 --- a/src/network/sequential/mod.rs +++ b/src/network/sequential/mod.rs @@ -1,12 +1,9 @@ use std::borrow::Cow; use super::*; -use crate::{ - gradient_solver::NeuraGradientSolverTransient, - layer::{ - NeuraLayer, NeuraPartialLayer, NeuraShape, NeuraTrainableLayerBase, - NeuraTrainableLayerEval, NeuraTrainableLayerSelf, - }, +use crate::layer::{ + NeuraLayer, NeuraPartialLayer, NeuraShape, NeuraTrainableLayerBase, NeuraTrainableLayerEval, + NeuraTrainableLayerSelf, }; mod construct; @@ -82,94 +79,6 @@ impl NeuraSequential { } } -impl< - Input, - Layer: NeuraTrainableLayerEval + NeuraTrainableLayerSelf, - ChildNetwork: NeuraOldTrainableNetworkBase, - > NeuraOldTrainableNetworkBase for NeuraSequential -{ - type Gradient = (Layer::Gradient, Box); - type LayerOutput = Layer::Output; - - fn default_gradient(&self) -> Self::Gradient { - ( - self.layer.default_gradient(), - Box::new(self.child_network.default_gradient()), - ) - } - - fn apply_gradient(&mut self, gradient: &Self::Gradient) { - self.layer.apply_gradient(&gradient.0); - self.child_network.apply_gradient(&gradient.1); - } - - fn regularize(&self) -> Self::Gradient { - ( - self.layer.regularize_layer(), - Box::new(self.child_network.regularize()), - ) - } - - fn prepare(&mut self, is_training: bool) { - self.layer.prepare_layer(is_training); - self.child_network.prepare(is_training); - } -} - -/// A dummy implementation of `NeuraTrainableNetwork`, which simply calls `loss.eval` in `backpropagate`. -impl NeuraOldTrainableNetworkBase for () { - type Gradient = (); - type LayerOutput = Input; - - #[inline(always)] - fn default_gradient(&self) -> () { - () - } - - #[inline(always)] - fn apply_gradient(&mut self, _gradient: &()) { - // Noop - } - - #[inline(always)] - fn regularize(&self) -> () { - () - } - - #[inline(always)] - fn prepare(&mut self, _is_training: bool) { - // Noop - } -} - -impl< - Input, - Layer: NeuraTrainableLayerEval + NeuraTrainableLayerSelf, - Optimizer: NeuraGradientSolverTransient, - ChildNetwork: NeuraOldTrainableNetworkBase, - > NeuraOldTrainableNetwork for NeuraSequential -where - ChildNetwork: NeuraOldTrainableNetwork, -{ - fn traverse( - &self, - input: &Input, - optimizer: &Optimizer, - ) -> Optimizer::Output { - let (next_activation, intermediary) = self.layer.eval_training(input); - let child_result = self.child_network.traverse(&next_activation, optimizer); - - optimizer.eval_layer( - &self.layer, - input, - &next_activation, - &intermediary, - child_result, - |layer_gradient, child_gradient| (layer_gradient, Box::new(child_gradient)), - ) - } -} - impl From for NeuraSequential { fn from(layer: Layer) -> Self { Self { diff --git a/src/network/traits.rs b/src/network/traits.rs index f7afd1a..c2f25f9 100644 --- a/src/network/traits.rs +++ b/src/network/traits.rs @@ -1,8 +1,6 @@ use std::borrow::Cow; -use crate::prelude::NeuraTrainableLayerBase; - -use super::*; +use crate::layer::*; /// This trait has to be non-generic, to ensure that no downstream crate can implement it for foreign types, /// as that would otherwise cause infinite recursion when dealing with `NeuraNetworkRec`. diff --git a/src/train.rs b/src/train.rs index 351de66..cd03dfc 100644 --- a/src/train.rs +++ b/src/train.rs @@ -1,7 +1,4 @@ -use crate::{ - algebra::NeuraVectorSpace, gradient_solver::NeuraGradientSolver, layer::*, - network::NeuraOldTrainableNetworkBase, -}; +use crate::{algebra::NeuraVectorSpace, gradient_solver::NeuraGradientSolver, layer::*}; #[non_exhaustive] pub struct NeuraBatchedTrainer {