diff --git a/src/gradient_solver/backprop.rs b/src/gradient_solver/backprop.rs index c88021b..c2c3d40 100644 --- a/src/gradient_solver/backprop.rs +++ b/src/gradient_solver/backprop.rs @@ -17,22 +17,22 @@ impl NeuraBackprop { impl< Input, Target, - Trainable: NeuraTrainableLayerBase + NeuraLayer + NeuraNetworkRec, + Trainable: NeuraLayer + NeuraNetworkRec, Loss: NeuraLoss + Clone, > NeuraGradientSolver for NeuraBackprop where >::Output: ToPrimitive, - // Trainable: NeuraOldTrainableNetworkBase::Gradient>, + // Trainable: NeuraOldTrainableNetworkBase::Gradient>, // Trainable: for<'a> NeuraOldTrainableNetwork, &'a Target)>, for<'a> (&'a NeuraBackprop, &'a Target): - BackpropRecurse::Gradient>, + BackpropRecurse::Gradient>, { fn get_gradient( &self, trainable: &Trainable, input: &Input, target: &Target, - ) -> ::Gradient { + ) -> ::Gradient { let (_, gradient) = (self, target).recurse(trainable, input); // let (_, gradient) = trainable.traverse(input, &(self, target)); @@ -59,7 +59,7 @@ impl> BackpropRecurse impl< Input: Clone, - Network: NeuraNetworkRec + NeuraNetwork + NeuraTrainableLayerEval, + Network: NeuraNetworkRec + NeuraNetwork + NeuraLayer, Loss, Target, > BackpropRecurse for (&NeuraBackprop, &Target) @@ -68,14 +68,13 @@ where for<'a> (&'a NeuraBackprop, &'a Target): BackpropRecurse< Network::NodeOutput, Network::NextNode, - ::Gradient, + ::Gradient, >, // Verify that the current layer implements the right traits - Network::Layer: NeuraTrainableLayerSelf - + NeuraTrainableLayerBackprop, + Network::Layer: NeuraLayer, // Verify that the layer output can be cloned >::Output: Clone, - Network::NextNode: NeuraTrainableLayerEval, + Network::NextNode: NeuraLayer, { fn recurse(&self, network: &Network, input: &Input) -> (Input, Network::Gradient) { let layer = network.get_layer(); diff --git a/src/gradient_solver/forward_forward.rs b/src/gradient_solver/forward_forward.rs index dce173f..a4d5be2 100644 --- a/src/gradient_solver/forward_forward.rs +++ b/src/gradient_solver/forward_forward.rs @@ -3,7 +3,7 @@ use num::{traits::NumAssignOps, Float}; use crate::{ derivable::NeuraDerivable, - layer::NeuraTrainableLayerSelf, + layer::*, network::{NeuraNetwork, NeuraNetworkRec}, prelude::NeuraLayer, }; @@ -69,18 +69,18 @@ impl< F: Float, Act: Clone + NeuraDerivable, Input: Clone, - Trainable: NeuraTrainableLayerBase + NeuraLayer>, + Trainable: NeuraLayer>, > NeuraGradientSolver for NeuraForwardForward where NeuraForwardPair: - ForwardForwardRecurse::Gradient>, + ForwardForwardRecurse::Gradient>, { fn get_gradient( &self, trainable: &Trainable, input: &Input, target: &bool, - ) -> ::Gradient { + ) -> ::Gradient { let target = *target; let pair = NeuraForwardPair { threshold: self.threshold, @@ -137,13 +137,13 @@ impl ForwardForwardRecurse for NeuraForwardPair impl + NeuraNetworkRec> ForwardForwardRecurse for NeuraForwardPair where - Network::Layer: NeuraTrainableLayerSelf, + Network::Layer: NeuraLayer, >::Output: Clone, Self: ForwardForwardDerivate<>::Output>, Self: ForwardForwardRecurse< Network::NodeOutput, Network::NextNode, - ::Gradient, + ::Gradient, >, { fn recurse(&self, network: &Network, input: &Input) -> Network::Gradient { diff --git a/src/gradient_solver/mod.rs b/src/gradient_solver/mod.rs index 740e0a2..68a7483 100644 --- a/src/gradient_solver/mod.rs +++ b/src/gradient_solver/mod.rs @@ -4,9 +4,9 @@ pub use backprop::NeuraBackprop; mod forward_forward; pub use forward_forward::NeuraForwardForward; -use crate::layer::{NeuraTrainableLayerBase, NeuraTrainableLayerEval}; +use crate::layer::NeuraLayerBase; -pub trait NeuraGradientSolver { +pub trait NeuraGradientSolver { fn get_gradient( &self, trainable: &Trainable, diff --git a/src/layer/dense.rs b/src/layer/dense.rs index 8710ef9..9090260 100644 --- a/src/layer/dense.rs +++ b/src/layer/dense.rs @@ -1,7 +1,7 @@ use std::marker::PhantomData; -use nalgebra::{DMatrix, DVector}; -use num::Float; +use nalgebra::{DMatrix, DVector, Scalar}; +use num::{traits::NumAssignOps, Float}; use rand::Rng; use crate::{derivable::NeuraDerivable, err::NeuraDimensionsMismatch}; @@ -126,18 +126,10 @@ impl NeuraDenseLayerPartial { } } -impl, Reg: NeuraDerivable> NeuraShapedLayer - for NeuraDenseLayer -{ - fn output_shape(&self) -> NeuraShape { - NeuraShape::Vector(self.weights.shape().0) - } -} - impl< - F: Float + std::fmt::Debug + 'static, - Act: NeuraDerivable, - Reg: NeuraDerivable, + F: Float + Scalar + Send + NumAssignOps, + Act: NeuraDerivable + Clone + std::fmt::Debug + 'static, + Reg: NeuraDerivable + Clone + std::fmt::Debug + 'static, R: Rng, > NeuraPartialLayer for NeuraDenseLayerPartial where @@ -158,8 +150,10 @@ where } } -impl, Reg: NeuraDerivable> NeuraPartialLayer - for NeuraDenseLayer +impl, Reg: NeuraDerivable> + NeuraPartialLayer for NeuraDenseLayer +where + Self: Clone + std::fmt::Debug + 'static, { type Constructed = Self; type Err = NeuraDimensionsMismatch; @@ -176,28 +170,10 @@ impl, Reg: NeuraDerivable> NeuraPartialLayer } } -impl< - F: Float + std::fmt::Debug + 'static + std::ops::AddAssign + std::ops::MulAssign, - Act: NeuraDerivable, - Reg: NeuraDerivable, - > NeuraLayer> for NeuraDenseLayer -{ - type Output = DVector; - - fn eval(&self, input: &DVector) -> Self::Output { - assert_eq!(input.shape().0, self.weights.shape().1); - - let evaluated = &self.weights * input + &self.bias; - - evaluated.map(|x| self.activation.eval(x)) - } -} - -impl< - F: Float + std::fmt::Debug + 'static + std::ops::AddAssign + std::ops::MulAssign, - Act: NeuraDerivable, - Reg: NeuraDerivable, - > NeuraTrainableLayerBase for NeuraDenseLayer +impl, Reg: NeuraDerivable> + NeuraLayerBase for NeuraDenseLayer +where + Self: Clone + std::fmt::Debug + 'static, { type Gradient = (DMatrix, DVector); @@ -212,14 +188,25 @@ impl< self.weights += &gradient.0; self.bias += &gradient.1; } + + fn output_shape(&self) -> NeuraShape { + NeuraShape::Vector(self.weights.shape().0) + } + + fn regularize_layer(&self) -> Self::Gradient { + ( + self.weights.map(|x| self.regularization.derivate(x)), + DVector::zeros(self.bias.shape().0), + ) + } } -impl< - F: Float + std::fmt::Debug + 'static + std::ops::AddAssign + std::ops::MulAssign, - Act: NeuraDerivable, - Reg: NeuraDerivable, - > NeuraTrainableLayerEval> for NeuraDenseLayer +impl, Reg: NeuraDerivable> + NeuraLayer> for NeuraDenseLayer +where + Self: Clone + std::fmt::Debug + 'static, { + type Output = DVector; type IntermediaryRepr = DVector; // pre-activation values fn eval_training(&self, input: &DVector) -> (Self::Output, Self::IntermediaryRepr) { @@ -228,20 +215,6 @@ impl< (output, evaluated) } -} - -impl< - F: Float + std::fmt::Debug + 'static + std::ops::AddAssign + std::ops::MulAssign, - Act: NeuraDerivable, - Reg: NeuraDerivable, - > NeuraTrainableLayerSelf> for NeuraDenseLayer -{ - fn regularize_layer(&self) -> Self::Gradient { - ( - self.weights.map(|x| self.regularization.derivate(x)), - DVector::zeros(self.bias.shape().0), - ) - } fn get_gradient( &self, @@ -266,14 +239,7 @@ impl< (weights_gradient, bias_gradient) } -} -impl< - F: Float + std::fmt::Debug + 'static + std::ops::AddAssign + std::ops::MulAssign, - Act: NeuraDerivable, - Reg: NeuraDerivable, - > NeuraTrainableLayerBackprop> for NeuraDenseLayer -{ fn backprop_layer( &self, _input: &DVector, diff --git a/src/layer/dropout.rs b/src/layer/dropout.rs index be56222..8989cba 100644 --- a/src/layer/dropout.rs +++ b/src/layer/dropout.rs @@ -35,13 +35,7 @@ impl NeuraDropoutLayer { } } -impl NeuraShapedLayer for NeuraDropoutLayer { - fn output_shape(&self) -> NeuraShape { - self.shape - } -} - -impl NeuraPartialLayer for NeuraDropoutLayer { +impl NeuraPartialLayer for NeuraDropoutLayer { type Constructed = NeuraDropoutLayer; type Err = (); @@ -53,25 +47,15 @@ impl NeuraPartialLayer for NeuraDropoutLayer { } } -impl NeuraLayer> for NeuraDropoutLayer { - type Output = DVector; - - fn eval(&self, input: &DVector) -> Self::Output { - let mut output = input.clone(); - self.apply_dropout(&mut output); - output - } -} - -impl NeuraTrainableLayerBase for NeuraDropoutLayer { +impl NeuraLayerBase for NeuraDropoutLayer { type Gradient = (); fn default_gradient(&self) -> Self::Gradient { () } - fn apply_gradient(&mut self, _gradient: &Self::Gradient) { - // Noop + fn output_shape(&self) -> NeuraShape { + self.shape } fn prepare_layer(&mut self, is_training: bool) { @@ -98,30 +82,19 @@ impl NeuraTrainableLayerBase for NeuraDropoutLayer { } } -impl NeuraTrainableLayerEval> for NeuraDropoutLayer { +impl NeuraLayer> + for NeuraDropoutLayer +{ + type Output = DVector; + type IntermediaryRepr = (); fn eval_training(&self, input: &DVector) -> (Self::Output, Self::IntermediaryRepr) { - (self.eval(input), ()) - } -} - -impl NeuraTrainableLayerSelf> for NeuraDropoutLayer { - fn regularize_layer(&self) -> Self::Gradient { - () - } - - fn get_gradient( - &self, - _input: &DVector, - _intermediary: &Self::IntermediaryRepr, - _epsilon: &Self::Output, - ) -> Self::Gradient { - () + let mut output = input.clone(); + self.apply_dropout(&mut output); + (output, ()) } -} -impl NeuraTrainableLayerBackprop> for NeuraDropoutLayer { fn backprop_layer( &self, _input: &DVector, diff --git a/src/layer/isolate.rs b/src/layer/isolate.rs index f45c4cf..ea553b4 100644 --- a/src/layer/isolate.rs +++ b/src/layer/isolate.rs @@ -24,12 +24,6 @@ impl NeuraIsolateLayer { } } -impl NeuraShapedLayer for NeuraIsolateLayer { - fn output_shape(&self) -> NeuraShape { - self.end.sub(self.start).unwrap_or_else(|| unreachable!()) - } -} - impl NeuraPartialLayer for NeuraIsolateLayer { type Constructed = NeuraIsolateLayer; type Err = NeuraIsolateLayerErr; @@ -70,19 +64,7 @@ impl NeuraPartialLayer for NeuraIsolateLayer { } } -impl NeuraLayer> for NeuraIsolateLayer { - type Output = DVector; - - fn eval(&self, input: &DVector) -> Self::Output { - let (NeuraShape::Vector(start), NeuraShape::Vector(end)) = (self.start, self.end) else { - panic!("NeuraIsolateLayer expected a value of dimension {}, got a vector", self.start.dims()); - }; - - DVector::from_iterator(end - start, input.iter().cloned().skip(start).take(end)) - } -} - -impl NeuraTrainableLayerBase for NeuraIsolateLayer { +impl NeuraLayerBase for NeuraIsolateLayer { type Gradient = (); #[inline(always)] @@ -90,41 +72,26 @@ impl NeuraTrainableLayerBase for NeuraIsolateLayer { () } - #[inline(always)] - fn apply_gradient(&mut self, _gradient: &Self::Gradient) { - // Noop + fn output_shape(&self) -> NeuraShape { + self.end.sub(self.start).unwrap_or_else(|| unreachable!()) } } -impl NeuraTrainableLayerEval> for NeuraIsolateLayer { +impl NeuraLayer> for NeuraIsolateLayer { + type Output = DVector; + type IntermediaryRepr = (); fn eval_training(&self, input: &DVector) -> (Self::Output, Self::IntermediaryRepr) { - (self.eval(input), ()) - } -} + let (NeuraShape::Vector(start), NeuraShape::Vector(end)) = (self.start, self.end) else { + panic!("NeuraIsolateLayer expected a value of dimension {}, got a vector", self.start.dims()); + }; -impl NeuraTrainableLayerSelf for NeuraIsolateLayer -where - Self: NeuraTrainableLayerEval, -{ - #[inline(always)] - fn regularize_layer(&self) -> Self::Gradient { - () - } + let res = DVector::from_iterator(end - start, input.iter().cloned().skip(start).take(end)); - #[inline(always)] - fn get_gradient( - &self, - _input: &Input, - _intermediary: &Self::IntermediaryRepr, - _epsilon: &Self::Output, - ) -> Self::Gradient { - () + (res, ()) } -} -impl NeuraTrainableLayerBackprop> for NeuraIsolateLayer { fn backprop_layer( &self, input: &DVector, diff --git a/src/layer/lock.rs b/src/layer/lock.rs index d8d8dc0..64bebe2 100644 --- a/src/layer/lock.rs +++ b/src/layer/lock.rs @@ -24,56 +24,35 @@ impl NeuraLockLayer { } } -impl> NeuraLayer for NeuraLockLayer { - type Output = Layer::Output; +impl NeuraLayerBase for NeuraLockLayer { + type Gradient = (); - fn eval(&self, input: &Input) -> Self::Output { - self.layer.eval(input) + fn output_shape(&self) -> NeuraShape { + self.layer.output_shape() } -} - -impl NeuraTrainableLayerBase for NeuraLockLayer { - type Gradient = (); fn default_gradient(&self) -> Self::Gradient { () } - fn apply_gradient(&mut self, _gradient: &Self::Gradient) { - // Noop + fn prepare_layer(&mut self, is_training: bool) { + self.layer.prepare_layer(is_training); } } -impl> NeuraTrainableLayerEval - for NeuraLockLayer -{ - type IntermediaryRepr = Layer::IntermediaryRepr; +impl> NeuraLayer for NeuraLockLayer { + type Output = Layer::Output; - fn eval_training(&self, input: &Input) -> (Self::Output, Self::IntermediaryRepr) { - self.layer.eval_training(input) + fn eval(&self, input: &Input) -> Self::Output { + self.layer.eval(input) } -} -impl> NeuraTrainableLayerSelf - for NeuraLockLayer -{ - fn regularize_layer(&self) -> Self::Gradient { - () - } + type IntermediaryRepr = Layer::IntermediaryRepr; - fn get_gradient( - &self, - _input: &Input, - _intermediary: &Self::IntermediaryRepr, - _epsilon: &Self::Output, - ) -> Self::Gradient { - () + fn eval_training(&self, input: &Input) -> (Self::Output, Self::IntermediaryRepr) { + self.layer.eval_training(input) } -} -impl> NeuraTrainableLayerBackprop - for NeuraLockLayer -{ fn backprop_layer( &self, input: &Input, diff --git a/src/layer/mod.rs b/src/layer/mod.rs index 9dd6547..3f46cda 100644 --- a/src/layer/mod.rs +++ b/src/layer/mod.rs @@ -75,11 +75,35 @@ impl From<(usize, usize, usize)> for NeuraShape { } } -pub trait NeuraLayer { - /// What type the layer outputs - type Output; +pub trait NeuraLayerBase: std::fmt::Debug + Clone + 'static { + /// What type the gradient of the layer is + type Gradient: NeuraVectorSpace + Send + 'static; + + /// What the desired output shape of the layer is + fn output_shape(&self) -> NeuraShape; + + /// Constructor for the gradient, should return the zero vector + fn default_gradient(&self) -> Self::Gradient; + + /// Applies `δW_l` to the weights of the layer, the default implementation is a noop + #[allow(unused_variables)] + #[inline(always)] + fn apply_gradient(&mut self, gradient: &Self::Gradient) { + // Noop + } + + /// Arbitrary computation that can be executed at the start of an epoch + #[allow(unused_variables)] + #[inline(always)] + fn prepare_layer(&mut self, is_training: bool) { + // Noop + } - fn eval(&self, input: &Input) -> Self::Output; + /// Computes the regularization terms of the layer's gradient, called once per batch + #[inline(always)] + fn regularize_layer(&self) -> Self::Gradient { + self.default_gradient() + } fn lock_layer(self) -> NeuraLockLayer where @@ -89,87 +113,34 @@ pub trait NeuraLayer { } } -impl NeuraLayer for () { - type Output = Input; +pub trait NeuraLayer: NeuraLayerBase { + /// What type the layer outputs, may depend on `Input`. + type Output; + + /// A type that can hold data between calls to `eval_training`, `backprop_layer` and `get_gradient` + type IntermediaryRepr: 'static; - #[inline(always)] fn eval(&self, input: &Input) -> Self::Output { - input.clone() + self.eval_training(input).0 } -} - -pub trait NeuraShapedLayer { - fn output_shape(&self) -> NeuraShape; -} - -pub trait NeuraPartialLayer { - type Constructed: NeuraShapedLayer; - type Err; - - fn construct(self, input_shape: NeuraShape) -> Result; -} - -pub trait NeuraTrainableLayerBase { - /// The representation of the layer gradient as a vector space - type Gradient: NeuraVectorSpace; - - fn default_gradient(&self) -> Self::Gradient; - - /// Applies `δW_l` to the weights of the layer - fn apply_gradient(&mut self, gradient: &Self::Gradient); - - /// Arbitrary computation that can be executed at the start of an epoch - #[allow(unused_variables)] - #[inline(always)] - fn prepare_layer(&mut self, is_training: bool) {} -} -pub trait NeuraTrainableLayerEval: NeuraTrainableLayerBase + NeuraLayer { - /// An intermediary object type to be passed to the various training methods - type IntermediaryRepr; - - // TODO: move this into another trait fn eval_training(&self, input: &Input) -> (Self::Output, Self::IntermediaryRepr); -} - -/// Contains methods relative to a layer's ability to compute its own weights gradients, -/// given the derivative of the output variables. -pub trait NeuraTrainableLayerSelf: NeuraTrainableLayerEval { - /// Computes the regularization - fn regularize_layer(&self) -> Self::Gradient; /// Computes the layer's gradient, /// /// `intermediary` is guaranteed to have been generated by a previous call to `eval_training`, /// without mutation of `self` in-between, and with the same `input`. + #[allow(unused_variables)] + #[inline(always)] fn get_gradient( &self, input: &Input, intermediary: &Self::IntermediaryRepr, epsilon: &Self::Output, - ) -> Self::Gradient; -} + ) -> Self::Gradient { + self.default_gradient() + } -// impl> NeuraTrainableLayerSelf -// for Layer -// { -// #[inline(always)] -// fn regularize_layer(&self) -> Self::Gradient { -// () -// } - -// #[inline(always)] -// fn get_gradient( -// &self, -// input: &Input, -// intermediary: &Self::IntermediaryRepr, -// epsilon: Self::Output, -// ) -> Self::Gradient { -// () -// } -// } - -pub trait NeuraTrainableLayerBackprop: NeuraTrainableLayerEval { /// Computes the backpropagation term and the derivative of the internal weights, /// using the `input` vector outputted by the previous layer and the backpropagation term `epsilon` of the next layer. /// @@ -189,7 +160,31 @@ pub trait NeuraTrainableLayerBackprop: NeuraTrainableLayerEval { ) -> Input; } -impl NeuraTrainableLayerBase for () { +#[deprecated] +pub trait NeuraShapedLayer: NeuraLayerBase {} + +pub trait NeuraPartialLayer { + type Constructed: NeuraLayerBase + 'static; + type Err; + + fn construct(self, input_shape: NeuraShape) -> Result; +} + +#[deprecated] +pub trait NeuraTrainableLayerBase: NeuraLayerBase {} + +#[deprecated] +pub trait NeuraTrainableLayerEval: NeuraLayer {} + +/// Contains methods relative to a layer's ability to compute its own weights gradients, +/// given the derivative of the output variables. +#[deprecated] +pub trait NeuraTrainableLayerSelf: NeuraLayer {} + +#[deprecated] +pub trait NeuraTrainableLayerBackprop: NeuraLayer {} + +impl NeuraLayerBase for () { type Gradient = (); #[inline(always)] @@ -201,35 +196,22 @@ impl NeuraTrainableLayerBase for () { fn apply_gradient(&mut self, _gradient: &Self::Gradient) { // Noop } + + fn output_shape(&self) -> NeuraShape { + panic!("() has no shape!"); + } } -impl NeuraTrainableLayerEval for () { +impl NeuraLayer for () { + type Output = Input; type IntermediaryRepr = (); #[inline(always)] fn eval_training(&self, input: &Input) -> (Self::Output, Self::IntermediaryRepr) { - (self.eval(input), ()) + (input.clone(), ()) } -} -impl NeuraTrainableLayerSelf for () { #[inline(always)] - fn regularize_layer(&self) -> Self::Gradient { - () - } - - #[inline(always)] - fn get_gradient( - &self, - _input: &Input, - _intermediary: &Self::IntermediaryRepr, - _epsilon: &Self::Output, - ) -> Self::Gradient { - () - } -} - -impl NeuraTrainableLayerBackprop for () { fn backprop_layer( &self, _input: &Input, diff --git a/src/layer/normalize.rs b/src/layer/normalize.rs index c12d95b..5e2d6af 100644 --- a/src/layer/normalize.rs +++ b/src/layer/normalize.rs @@ -23,12 +23,6 @@ impl NeuraNormalizeLayer { } } -impl NeuraShapedLayer for NeuraNormalizeLayer { - fn output_shape(&self) -> NeuraShape { - self.shape - } -} - impl NeuraPartialLayer for NeuraNormalizeLayer { type Constructed = NeuraNormalizeLayer; @@ -39,7 +33,22 @@ impl NeuraPartialLayer for NeuraNormalizeLayer { } } -impl NeuraLayer> for NeuraNormalizeLayer { +impl NeuraLayerBase for NeuraNormalizeLayer { + fn output_shape(&self) -> NeuraShape { + self.shape + } + + type Gradient = (); + + fn default_gradient(&self) -> Self::Gradient { + () + } +} + +impl NeuraLayer> for NeuraNormalizeLayer { + // TODO: store the kroenecker term in the jacobian matrix (might as well) + type IntermediaryRepr = (DMatrix, F); // Partial jacobian matrix (without the kroenecker term) and stddev + type Output = DVector; fn eval(&self, input: &DVector) -> Self::Output { @@ -54,22 +63,6 @@ impl NeuraLayer> for NeuraNormalizeLayer { output } -} - -impl NeuraTrainableLayerBase for NeuraNormalizeLayer { - type Gradient = (); - - fn default_gradient(&self) -> Self::Gradient { - () - } - - fn apply_gradient(&mut self, _gradient: &Self::Gradient) { - // Noop - } -} - -impl NeuraTrainableLayerEval> for NeuraNormalizeLayer { - type IntermediaryRepr = (DMatrix, F); // Partial jacobian matrix (without the kroenecker term) and stddev fn eval_training(&self, input: &DVector) -> (Self::Output, Self::IntermediaryRepr) { let (mean, variance, len) = mean_variance(input); @@ -85,26 +78,7 @@ impl NeuraTrainableLayerEval> for N (input_centered / stddev, (jacobian_partial, stddev)) } -} - -impl NeuraTrainableLayerSelf> for NeuraNormalizeLayer { - fn regularize_layer(&self) -> Self::Gradient { - () - } - - fn get_gradient( - &self, - _input: &DVector, - _intermediary: &Self::IntermediaryRepr, - _epsilon: &Self::Output, - ) -> Self::Gradient { - () - } -} -impl NeuraTrainableLayerBackprop> - for NeuraNormalizeLayer -{ fn backprop_layer( &self, _input: &DVector, diff --git a/src/layer/softmax.rs b/src/layer/softmax.rs index 8172a88..ebae80f 100644 --- a/src/layer/softmax.rs +++ b/src/layer/softmax.rs @@ -16,8 +16,30 @@ impl NeuraSoftmaxLayer { } } +impl NeuraPartialLayer for NeuraSoftmaxLayer { + type Constructed = Self; + type Err = (); + + fn construct(self, input_shape: NeuraShape) -> Result { + Ok(Self { shape: input_shape }) + } +} + +impl NeuraLayerBase for NeuraSoftmaxLayer { + type Gradient = (); + + fn output_shape(&self) -> NeuraShape { + self.shape + } + + fn default_gradient(&self) -> Self::Gradient { + () + } +} + impl NeuraLayer> for NeuraSoftmaxLayer { type Output = DVector; + type IntermediaryRepr = Self::Output; // Result of self.eval fn eval(&self, input: &DVector) -> Self::Output { let mut res = input.clone(); @@ -39,64 +61,12 @@ impl NeuraLayer> for NeuraSoftmaxLa res } -} - -impl NeuraShapedLayer for NeuraSoftmaxLayer { - fn output_shape(&self) -> NeuraShape { - self.shape - } -} - -impl NeuraPartialLayer for NeuraSoftmaxLayer { - type Constructed = Self; - type Err = (); - - fn construct(self, input_shape: NeuraShape) -> Result { - Ok(Self { shape: input_shape }) - } -} - -impl NeuraTrainableLayerBase for NeuraSoftmaxLayer { - type Gradient = (); - - fn default_gradient(&self) -> Self::Gradient { - () - } - - fn apply_gradient(&mut self, _gradient: &Self::Gradient) { - // Noop - } -} - -impl NeuraTrainableLayerEval> for NeuraSoftmaxLayer { - type IntermediaryRepr = Self::Output; // Result of self.eval fn eval_training(&self, input: &DVector) -> (Self::Output, Self::IntermediaryRepr) { let res = self.eval(input); (res.clone(), res) } -} - -impl NeuraTrainableLayerSelf> for NeuraSoftmaxLayer { - #[inline(always)] - fn regularize_layer(&self) -> Self::Gradient { - () - } - - #[inline(always)] - fn get_gradient( - &self, - _input: &DVector, - _intermediary: &Self::IntermediaryRepr, - _epsilon: &Self::Output, - ) -> Self::Gradient { - () - } -} -impl NeuraTrainableLayerBackprop> - for NeuraSoftmaxLayer -{ fn backprop_layer( &self, input: &DVector, diff --git a/src/lib.rs b/src/lib.rs index 270bd68..8b5885d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,10 +23,7 @@ pub mod prelude { // Structs and traits pub use crate::gradient_solver::NeuraBackprop; - pub use crate::layer::{ - NeuraLayer, NeuraPartialLayer, NeuraShape, NeuraTrainableLayerBackprop, - NeuraTrainableLayerBase, NeuraTrainableLayerSelf, - }; + pub use crate::layer::{NeuraLayer, NeuraLayerBase, NeuraPartialLayer, NeuraShape}; pub use crate::network::sequential::{ NeuraSequential, NeuraSequentialLock, NeuraSequentialTail, }; diff --git a/src/network/graph/from.rs b/src/network/graph/from.rs index 3336bae..47f1364 100644 --- a/src/network/graph/from.rs +++ b/src/network/graph/from.rs @@ -25,7 +25,7 @@ impl FromSequential<(), Data> for NeuraGraph { } } -impl, ChildNetwork> +impl, ChildNetwork> FromSequential, Data> for NeuraGraph where NeuraGraph: FromSequential, @@ -57,7 +57,7 @@ impl NeuraGraph { ) -> Self where NeuraGraph: FromSequential, Data>, - NeuraSequential: NeuraShapedLayer, + NeuraSequential: NeuraLayerBase, { Self::from_sequential_rec(&network, vec![], input_shape) } diff --git a/src/network/graph/mod.rs b/src/network/graph/mod.rs index 1156e2e..3713fd9 100644 --- a/src/network/graph/mod.rs +++ b/src/network/graph/mod.rs @@ -1,10 +1,7 @@ use std::any::Any; use crate::{ - algebra::NeuraDynVectorSpace, - derivable::NeuraLoss, - layer::{NeuraShapedLayer, NeuraTrainableLayerEval}, - prelude::*, + algebra::NeuraDynVectorSpace, derivable::NeuraLoss, layer::NeuraLayerBase, prelude::*, }; mod node; @@ -16,31 +13,10 @@ pub use partial::NeuraGraphPartial; mod from; pub use from::FromSequential; -pub trait NeuraTrainableLayerFull: - NeuraTrainableLayerEval - + NeuraTrainableLayerBackprop - + NeuraTrainableLayerSelf - + NeuraShapedLayer - + Clone - + std::fmt::Debug - + 'static -where - Self::IntermediaryRepr: 'static, -{ -} +#[deprecated] +pub trait NeuraTrainableLayerFull: NeuraLayer {} -impl NeuraTrainableLayerFull for T -where - T: NeuraTrainableLayerEval - + NeuraTrainableLayerBackprop - + NeuraTrainableLayerSelf - + NeuraShapedLayer - + Clone - + std::fmt::Debug - + 'static, - T::IntermediaryRepr: 'static, -{ -} +impl NeuraTrainableLayerFull for T where T: NeuraLayer {} #[derive(Debug)] pub struct NeuraGraphNodeConstructed { @@ -49,7 +25,17 @@ pub struct NeuraGraphNodeConstructed { output: usize, } -#[derive(Debug)] +impl Clone for NeuraGraphNodeConstructed { + fn clone(&self) -> Self { + Self { + node: dyn_clone::clone_box(&*self.node), + inputs: self.inputs.clone(), + output: self.output.clone(), + } + } +} + +#[derive(Clone, Debug)] pub struct NeuraGraph { /// ## Class invariants /// @@ -65,10 +51,16 @@ pub struct NeuraGraph { buffer_size: usize, } -impl NeuraShapedLayer for NeuraGraph { +impl NeuraLayerBase for NeuraGraph { + type Gradient = (); + fn output_shape(&self) -> NeuraShape { self.output_shape } + + fn default_gradient(&self) -> Self::Gradient { + unimplemented!("NeuraGraph cannot be used as a layer yet") + } } impl NeuraGraph { @@ -178,8 +170,9 @@ impl NeuraGraph { } } -impl NeuraLayer for NeuraGraph { +impl NeuraLayer for NeuraGraph { type Output = Data; + type IntermediaryRepr = (); fn eval(&self, input: &Data) -> Self::Output { let mut buffer = self.create_buffer(); @@ -190,6 +183,21 @@ impl NeuraLayer for NeuraGraph { .take() .expect("Unreachable: output was not set") } + + #[allow(unused)] + fn eval_training(&self, input: &Data) -> (Self::Output, Self::IntermediaryRepr) { + unimplemented!("NeuraGraph cannot be used as a trainable layer yet"); + } + + #[allow(unused)] + fn backprop_layer( + &self, + input: &Data, + intermediary: &Self::IntermediaryRepr, + epsilon: &Self::Output, + ) -> Data { + unimplemented!("NeuraGraph cannot be used as a trainable layer yet"); + } } #[cfg(test)] diff --git a/src/network/graph/node.rs b/src/network/graph/node.rs index 9d368f2..da7458f 100644 --- a/src/network/graph/node.rs +++ b/src/network/graph/node.rs @@ -4,7 +4,6 @@ use std::{any::Any, fmt::Debug}; use crate::{ algebra::NeuraDynVectorSpace, err::NeuraAxisErr, - layer::{NeuraShapedLayer, NeuraTrainableLayerEval}, network::residual::{NeuraAxisDefault, NeuraCombineInputs, NeuraSplitInputs}, prelude::{NeuraPartialLayer, NeuraShape}, }; @@ -81,9 +80,9 @@ impl NeuraGraphNode { + 'static, Layer: NeuraPartialLayer + Clone + Debug + 'static, Layer::Constructed: - NeuraTrainableLayerFull<>::Combined, Output = Data>, + NeuraLayer<>::Combined, Output = Data>, Layer::Err: Debug, - >::Combined, >>::IntermediaryRepr: 'static, >::Combined: 'static, @@ -97,7 +96,7 @@ impl NeuraGraphNode { ) -> &'a Intermediary where Axis: NeuraCombineInputs, - Layer: NeuraTrainableLayerFull, + Layer: NeuraLayer, Axis::Combined: 'static, { intermediary @@ -106,7 +105,7 @@ impl NeuraGraphNode { } } -struct Intermediary> +struct Intermediary> where Layer::IntermediaryRepr: 'static, { @@ -117,7 +116,7 @@ where impl< Data: Clone, Axis: NeuraSplitInputs + Clone + Debug, - Layer: NeuraTrainableLayerFull<>::Combined, Output = Data>, + Layer: NeuraLayer<>::Combined, Output = Data>, > NeuraGraphNodeEval for NeuraGraphNode where Layer::IntermediaryRepr: 'static, @@ -188,9 +187,9 @@ impl< Layer: NeuraPartialLayer + Clone + Debug, > NeuraGraphNodePartial for NeuraGraphNode where - Layer::Constructed: NeuraTrainableLayerFull<>::Combined, Output = Data>, + Layer::Constructed: NeuraLayer<>::Combined, Output = Data>, Layer::Err: Debug, - >::Combined>>::IntermediaryRepr: 'static, + >::Combined>>::IntermediaryRepr: 'static, >::Combined: 'static, { fn inputs<'a>(&'a self) -> &'a [String] { diff --git a/src/network/graph/partial.rs b/src/network/graph/partial.rs index 99ad1d8..cfaa118 100644 --- a/src/network/graph/partial.rs +++ b/src/network/graph/partial.rs @@ -122,7 +122,7 @@ impl NeuraGraphPartial { } } -impl NeuraPartialLayer for NeuraGraphPartial { +impl NeuraPartialLayer for NeuraGraphPartial { type Constructed = NeuraGraph; type Err = NeuraGraphErr; diff --git a/src/network/residual/construct.rs b/src/network/residual/construct.rs index b0aedfb..5c1159c 100644 --- a/src/network/residual/construct.rs +++ b/src/network/residual/construct.rs @@ -78,33 +78,10 @@ where } } -impl NeuraShapedLayer for NeuraResidualNode { - #[inline(always)] - fn output_shape(&self) -> NeuraShape { - self.output_shape.unwrap() - } -} - -impl NeuraShapedLayer - for NeuraResidualNode -{ - #[inline(always)] - fn output_shape(&self) -> NeuraShape { - self.child_network.output_shape() - } -} - -impl NeuraShapedLayer for NeuraResidual { - #[inline(always)] - fn output_shape(&self) -> NeuraShape { - self.layers.output_shape() - } -} - impl NeuraPartialLayer for NeuraResidual where // Should always be satisfied: - Layers::Constructed: NeuraShapedLayer, + Layers::Constructed: NeuraLayerBase, { type Constructed = NeuraResidual; type Err = Layers::Err; diff --git a/src/network/residual/last.rs b/src/network/residual/last.rs index d6c0e1f..85619a3 100644 --- a/src/network/residual/last.rs +++ b/src/network/residual/last.rs @@ -63,7 +63,13 @@ impl NeuraResidualConstruct for NeuraResidualLast { } } -impl NeuraShapedLayer for NeuraResidualLast { +impl NeuraLayerBase for NeuraResidualLast { + type Gradient = (); + + fn default_gradient(&self) -> Self::Gradient { + () + } + fn output_shape(&self) -> NeuraShape { self.output_shape .expect("Called NeuraResidualLast::output_shape before constructing it") @@ -90,7 +96,7 @@ impl NeuraNetworkRec for NeuraResidualLast { #[inline(always)] fn merge_gradient(&self, _rec_gradient: (), _layer_gradient: ()) -> Self::Gradient where - Self::Layer: NeuraTrainableLayerBase, + Self::Layer: NeuraLayerBase, { () } @@ -135,62 +141,24 @@ impl NeuraNetwork> for NeuraResidualLast { } } -impl NeuraTrainableLayerBase for NeuraResidualLast { - type Gradient = (); - - #[inline(always)] - fn default_gradient(&self) -> Self::Gradient { - () - } - - #[inline(always)] - fn apply_gradient(&mut self, _gradient: &Self::Gradient) { - // Noop - } -} - impl NeuraLayer> for NeuraResidualLast { type Output = Data; + type IntermediaryRepr = (); - fn eval(&self, input: &NeuraResidualInput) -> Self::Output { + fn eval_training(&self, input: &NeuraResidualInput) -> (Self::Output, ()) { let result: Rc = input.clone().get_first() .expect("Invalid NeuraResidual state: network returned no data, did you forget to link the last layer?") .into(); - unwrap_or_clone(result) + (unwrap_or_clone(result), ()) } -} -impl NeuraTrainableLayerEval> for NeuraResidualLast { - type IntermediaryRepr = (); - - #[inline(always)] - fn eval_training( + fn backprop_layer( &self, input: &NeuraResidualInput, - ) -> (Self::Output, Self::IntermediaryRepr) { - (self.eval(input), ()) - } -} - -impl NeuraTrainableLayerSelf> for NeuraResidualLast { - #[inline(always)] - fn regularize_layer(&self) -> Self::Gradient { - () - } - - #[inline(always)] - fn get_gradient( - &self, - _input: &NeuraResidualInput, _intermediary: &Self::IntermediaryRepr, - _epsilon: &Self::Output, - ) -> Self::Gradient { - () + epsilon: &Self::Output, + ) -> NeuraResidualInput { + Cow::into_owned(self.map_gradient_out(input, epsilon, epsilon)) } } - -// let epsilon = Rc::new(epsilon.clone()); -// let mut epsilon_residual = NeuraResidualInput::new(); - -// epsilon_residual.push(0, epsilon); diff --git a/src/network/residual/node.rs b/src/network/residual/node.rs index 36ec981..6b04e9c 100644 --- a/src/network/residual/node.rs +++ b/src/network/residual/node.rs @@ -1,5 +1,3 @@ -use nalgebra::{DVector, Scalar}; -use num::Float; use std::borrow::Cow; use crate::network::*; @@ -102,24 +100,6 @@ impl NeuraResidualNode { } } -impl NeuraLayer>> - for NeuraResidualNode -where - Axis: NeuraCombineInputs>, - Layer: NeuraLayer>, - ChildNetwork: NeuraLayer>>, -{ - type Output = >>>::Output; - - fn eval(&self, input: &NeuraResidualInput>) -> Self::Output { - let (layer_input, mut rest) = self.process_input(input); - - self.combine_outputs(self.layer.eval(&layer_input), &mut rest); - - self.child_network.eval(&rest) - } -} - #[allow(dead_code)] pub struct NeuraResidualIntermediary { layer_intermediary: LayerIntermediary, @@ -127,9 +107,18 @@ pub struct NeuraResidualIntermediary, } -impl - NeuraTrainableLayerBase for NeuraResidualNode +impl< + Layer: NeuraLayerBase, + ChildNetwork: NeuraLayerBase, + Axis: Clone + std::fmt::Debug + 'static, + > NeuraLayerBase for NeuraResidualNode { + #[inline(always)] + fn output_shape(&self) -> NeuraShape { + todo!("output_shape for NeuraResidualNode is not yet ready"); + self.child_network.output_shape() + } + type Gradient = (Layer::Gradient, Box); fn default_gradient(&self) -> Self::Gradient { @@ -148,25 +137,37 @@ impl Self::Gradient { + ( + self.layer.regularize_layer(), + Box::new(self.child_network.regularize_layer()), + ) + } } -impl< - Data, - Axis: NeuraCombineInputs, - Layer: NeuraTrainableLayerEval, - ChildNetwork: NeuraTrainableLayerEval>, - > NeuraTrainableLayerEval> - for NeuraResidualNode +impl + NeuraLayer> for NeuraResidualNode where - NeuraResidualNode: - NeuraLayer, Output = ChildNetwork::Output>, + Axis: NeuraCombineInputs, + Layer: NeuraLayer, + ChildNetwork: NeuraLayer>, { + type Output = >>::Output; type IntermediaryRepr = NeuraResidualIntermediary< Layer::IntermediaryRepr, Layer::Output, ChildNetwork::IntermediaryRepr, >; + fn eval(&self, input: &NeuraResidualInput) -> Self::Output { + let (layer_input, mut rest) = self.process_input(input); + + self.combine_outputs(self.layer.eval(&layer_input), &mut rest); + + self.child_network.eval(&rest) + } + fn eval_training( &self, input: &NeuraResidualInput, @@ -186,25 +187,6 @@ where (output, intermediary) } -} - -impl< - Data, - Axis: NeuraCombineInputs, - Layer: NeuraTrainableLayerSelf, - ChildNetwork: NeuraTrainableLayerSelf>, - > NeuraTrainableLayerSelf> - for NeuraResidualNode -where - NeuraResidualNode: - NeuraLayer, Output = ChildNetwork::Output>, -{ - fn regularize_layer(&self) -> Self::Gradient { - ( - self.layer.regularize_layer(), - Box::new(self.child_network.regularize_layer()), - ) - } #[allow(unused)] fn get_gradient( @@ -213,7 +195,17 @@ where intermediary: &Self::IntermediaryRepr, epsilon: &Self::Output, ) -> Self::Gradient { - unimplemented!("NeuraResidualNode::get_gradient is not yet implemented, sorry"); + unimplemented!("NeuraResidualNode::get_gradient is not yet implemented"); + } + + #[allow(unused)] + fn backprop_layer( + &self, + input: &NeuraResidualInput, + intermediary: &Self::IntermediaryRepr, + epsilon: &Self::Output, + ) -> NeuraResidualInput { + unimplemented!("NeuraResidualNode::backprop_layer is not yet implemented"); } } @@ -225,8 +217,11 @@ impl NeuraNetworkBase for NeuraResidualNode NeuraNetworkRec - for NeuraResidualNode +impl< + Axis: Clone + std::fmt::Debug + 'static, + Layer: NeuraLayerBase, + ChildNetwork: NeuraLayerBase, + > NeuraNetworkRec for NeuraResidualNode { type NextNode = ChildNetwork; @@ -236,8 +231,8 @@ impl::Gradient, - layer_gradient: ::Gradient, + rec_gradient: ::Gradient, + layer_gradient: ::Gradient, ) -> Self::Gradient { (layer_gradient, Box::new(rec_gradient)) } diff --git a/src/network/residual/wrapper.rs b/src/network/residual/wrapper.rs index f3cdc7e..5d3b1f4 100644 --- a/src/network/residual/wrapper.rs +++ b/src/network/residual/wrapper.rs @@ -1,6 +1,6 @@ use std::borrow::Cow; -use crate::network::*; +use crate::{network::*, utils::unwrap_or_clone}; use super::*; @@ -45,18 +45,48 @@ impl NeuraResidual { } } -impl NeuraLayer for NeuraResidual +impl NeuraLayer for NeuraResidual where - Layers: NeuraLayer>, + Layers: NeuraLayer>, { type Output = Layers::Output; + type IntermediaryRepr = Layers::IntermediaryRepr; - fn eval(&self, input: &Input) -> Self::Output { + fn eval(&self, input: &Data) -> Self::Output { self.layers.eval(&self.input_to_residual_input(input)) } + + fn eval_training(&self, input: &Data) -> (Self::Output, Self::IntermediaryRepr) { + self.layers + .eval_training(&self.input_to_residual_input(input)) + } + + fn get_gradient( + &self, + input: &Data, + intermediary: &Self::IntermediaryRepr, + epsilon: &Self::Output, + ) -> Self::Gradient { + self.layers + .get_gradient(&self.input_to_residual_input(input), intermediary, &epsilon) + } + + fn backprop_layer( + &self, + input: &Data, + intermediary: &Self::IntermediaryRepr, + epsilon: &Self::Output, + ) -> Data { + unwrap_or_clone( + self.layers + .backprop_layer(&self.input_to_residual_input(input), intermediary, &epsilon) + .get_first() + .unwrap(), + ) + } } -impl NeuraTrainableLayerBase for NeuraResidual { +impl NeuraLayerBase for NeuraResidual { type Gradient = Layers::Gradient; #[inline(always)] @@ -68,34 +98,13 @@ impl NeuraTrainableLayerBase for NeuraResidual< fn apply_gradient(&mut self, gradient: &Self::Gradient) { self.layers.apply_gradient(gradient); } -} - -impl>> - NeuraTrainableLayerEval for NeuraResidual -{ - type IntermediaryRepr = Layers::IntermediaryRepr; - - fn eval_training(&self, input: &Data) -> (Self::Output, Self::IntermediaryRepr) { - self.layers - .eval_training(&self.input_to_residual_input(input)) - } -} -impl>> - NeuraTrainableLayerSelf for NeuraResidual -{ fn regularize_layer(&self) -> Self::Gradient { self.layers.regularize_layer() } - fn get_gradient( - &self, - input: &Data, - intermediary: &Self::IntermediaryRepr, - epsilon: &Self::Output, - ) -> Self::Gradient { - self.layers - .get_gradient(&self.input_to_residual_input(input), intermediary, &epsilon) + fn output_shape(&self) -> NeuraShape { + self.layers.output_shape() } } @@ -108,7 +117,7 @@ impl NeuraNetworkBase for NeuraResidual { } } -impl NeuraNetworkRec for NeuraResidual { +impl NeuraNetworkRec for NeuraResidual { type NextNode = Layers; #[inline(always)] @@ -119,8 +128,8 @@ impl NeuraNetworkRec for NeuraResidual #[inline(always)] fn merge_gradient( &self, - rec_gradient: ::Gradient, - _layer_gradient: ::Gradient, + rec_gradient: ::Gradient, + _layer_gradient: ::Gradient, ) -> Self::Gradient { rec_gradient } diff --git a/src/network/sequential/construct.rs b/src/network/sequential/construct.rs index a8b47eb..f8a50f5 100644 --- a/src/network/sequential/construct.rs +++ b/src/network/sequential/construct.rs @@ -1,4 +1,4 @@ -use crate::{err::NeuraRecursiveErr, layer::NeuraShapedLayer}; +use crate::err::NeuraRecursiveErr; use super::*; @@ -39,19 +39,3 @@ impl NeuraPartialLaye }) } } - -impl NeuraShapedLayer for NeuraSequential { - #[inline(always)] - fn output_shape(&self) -> NeuraShape { - self.layer.output_shape() - } -} - -impl NeuraShapedLayer - for NeuraSequential -{ - #[inline(always)] - fn output_shape(&self) -> NeuraShape { - self.child_network.output_shape() - } -} diff --git a/src/network/sequential/layer_impl.rs b/src/network/sequential/layer_impl.rs index aed7ef9..1d5aa20 100644 --- a/src/network/sequential/layer_impl.rs +++ b/src/network/sequential/layer_impl.rs @@ -1,19 +1,22 @@ use super::*; -use crate::layer::{NeuraTrainableLayerBackprop, NeuraTrainableLayerEval}; +use crate::layer::{NeuraLayer, NeuraLayerBase}; -impl, ChildNetwork: NeuraLayer> NeuraLayer +// impl NeuraLayerBase for NeuraSequential { +// #[inline(always)] +// fn output_shape(&self) -> NeuraShape { +// self.layer.output_shape() +// } +// } + +impl NeuraLayerBase for NeuraSequential { - type Output = ChildNetwork::Output; - - fn eval(&self, input: &Input) -> Self::Output { - self.child_network.eval(&self.layer.eval(input)) + #[inline(always)] + fn output_shape(&self) -> NeuraShape { + todo!("Have output_shape return Option"); + self.child_network.output_shape() } -} -impl NeuraTrainableLayerBase - for NeuraSequential -{ type Gradient = (Layer::Gradient, Box); fn default_gradient(&self) -> Self::Gradient { @@ -32,16 +35,25 @@ impl Neur self.layer.apply_gradient(&gradient.0); self.child_network.apply_gradient(&gradient.1); } + + fn regularize_layer(&self) -> Self::Gradient { + ( + self.layer.regularize_layer(), + Box::new(self.child_network.regularize_layer()), + ) + } } -impl< - Input, - Layer: NeuraTrainableLayerEval, - ChildNetwork: NeuraTrainableLayerEval, - > NeuraTrainableLayerEval for NeuraSequential +impl, ChildNetwork: NeuraLayer> NeuraLayer + for NeuraSequential { + type Output = ChildNetwork::Output; type IntermediaryRepr = (Layer::IntermediaryRepr, Box); + fn eval(&self, input: &Input) -> Self::Output { + self.child_network.eval(&self.layer.eval(input)) + } + fn eval_training(&self, input: &Input) -> (Self::Output, Self::IntermediaryRepr) { let (layer_output, layer_intermediary) = self.layer.eval_training(input); let (child_output, child_intermediary) = self.child_network.eval_training(&layer_output); @@ -51,20 +63,6 @@ impl< (layer_intermediary, Box::new(child_intermediary)), ) } -} - -impl< - Input, - Layer: NeuraTrainableLayerSelf, - ChildNetwork: NeuraTrainableLayerSelf + NeuraTrainableLayerBackprop, - > NeuraTrainableLayerSelf for NeuraSequential -{ - fn regularize_layer(&self) -> Self::Gradient { - ( - self.layer.regularize_layer(), - Box::new(self.child_network.regularize_layer()), - ) - } fn get_gradient( &self, @@ -74,14 +72,7 @@ impl< ) -> Self::Gradient { unimplemented!("NeuraSequential::get_gradient is not yet implemented, sorry"); } -} -impl< - Input, - Layer: NeuraTrainableLayerBackprop, - ChildNetwork: NeuraTrainableLayerBackprop, - > NeuraTrainableLayerBackprop for NeuraSequential -{ fn backprop_layer( &self, input: &Input, diff --git a/src/network/sequential/mod.rs b/src/network/sequential/mod.rs index d6481a9..a4dca4e 100644 --- a/src/network/sequential/mod.rs +++ b/src/network/sequential/mod.rs @@ -1,10 +1,7 @@ use std::borrow::Cow; use super::*; -use crate::layer::{ - NeuraLayer, NeuraPartialLayer, NeuraShape, NeuraTrainableLayerBase, NeuraTrainableLayerEval, - NeuraTrainableLayerSelf, -}; +use crate::layer::{NeuraLayer, NeuraLayerBase, NeuraPartialLayer, NeuraShape}; mod construct; mod layer_impl; @@ -29,7 +26,7 @@ pub use tail::*; /// ## Notes on implemented traits /// /// The different implementations for `NeuraTrainableNetwork`, -/// `NeuraLayer` and `NeuraTrainableLayerBase` each require that `ChildNetwork` implements those respective traits, +/// `NeuraLayer` and `NeuraLayerBase` each require that `ChildNetwork` implements those respective traits, /// and that the output type of `Layer` matches the input type of `ChildNetwork`. /// /// If a method, like `eval`, is reported as missing, @@ -96,7 +93,7 @@ impl NeuraNetworkBase for NeuraSequential NeuraNetworkRec +impl NeuraNetworkRec for NeuraSequential { type NextNode = ChildNetwork; @@ -107,14 +104,14 @@ impl Neur fn merge_gradient( &self, - rec_gradient: ::Gradient, - layer_gradient: ::Gradient, + rec_gradient: ::Gradient, + layer_gradient: ::Gradient, ) -> Self::Gradient { (layer_gradient, Box::new(rec_gradient)) } } -impl, ChildNetwork> NeuraNetwork +impl, ChildNetwork> NeuraNetwork for NeuraSequential where Layer::Output: Clone, diff --git a/src/network/traits.rs b/src/network/traits.rs index c2f25f9..d9d0f79 100644 --- a/src/network/traits.rs +++ b/src/network/traits.rs @@ -46,18 +46,18 @@ where ) -> Cow<'a, NodeInput>; } -pub trait NeuraNetworkRec: NeuraNetworkBase + NeuraTrainableLayerBase { +pub trait NeuraNetworkRec: NeuraNetworkBase + NeuraLayerBase { /// The type of the children network, it does not need to implement `NeuraNetworkBase`, /// although many functions will expect it to be either `()` or an implementation of `NeuraNetworkRec`. - type NextNode: NeuraTrainableLayerBase; + type NextNode: NeuraLayerBase; fn get_next(&self) -> &Self::NextNode; fn merge_gradient( &self, - rec_gradient: ::Gradient, - layer_gradient: ::Gradient, + rec_gradient: ::Gradient, + layer_gradient: ::Gradient, ) -> Self::Gradient where - Self::Layer: NeuraTrainableLayerBase; + Self::Layer: NeuraLayerBase; } diff --git a/src/train.rs b/src/train.rs index cd03dfc..b64b4d2 100644 --- a/src/train.rs +++ b/src/train.rs @@ -70,7 +70,7 @@ impl NeuraBatchedTrainer { pub fn train< Input: Clone, Target: Clone, - Network: NeuraTrainableLayerBase + NeuraTrainableLayerSelf, + Network: NeuraLayer, GradientSolver: NeuraGradientSolver, Inputs: IntoIterator, >(