🎨 Clean up NeuraSequential

examples/xor.rs

@@ -2,17 +2,19 @@
 
 use nalgebra::dvector;
 
+use neuramethyst::cycle_shuffling;
 use neuramethyst::derivable::activation::Relu;
 use neuramethyst::derivable::loss::Euclidean;
 use neuramethyst::prelude::*;
-use neuramethyst::cycle_shuffling;
 
 fn main() {
     let mut network = neura_sequential![
         neura_layer!("dense", 4, Relu),
         neura_layer!("dense", 3, Relu),
         neura_layer!("dense", 1, Relu)
-    ].construct(NeuraShape::Vector(2)).unwrap();
+    ]
+    .construct(NeuraShape::Vector(2))
+    .unwrap();
 
     let inputs = [
         (dvector![0.0, 0.0], dvector![0.0]),

src/algebra/mod.rs

@@ -107,11 +107,12 @@ impl<const N: usize, T: NeuraVectorSpace + Clone> NeuraVectorSpace for [T; N] {
     }
 }
 
-impl<F: Float, R: nalgebra::Dim, C: nalgebra::Dim, S: nalgebra::RawStorage<F, R, C>> NeuraVectorSpace for Matrix<F, R, C, S>
+impl<F: Float, R: nalgebra::Dim, C: nalgebra::Dim, S: nalgebra::RawStorage<F, R, C>>
+    NeuraVectorSpace for Matrix<F, R, C, S>
 where
     Matrix<F, R, C, S>: std::ops::MulAssign<F>,
     for<'c> Matrix<F, R, C, S>: std::ops::AddAssign<&'c Matrix<F, R, C, S>>,
-    F: From<f64> + Into<f64>
+    F: From<f64> + Into<f64>,
 {
     fn add_assign(&mut self, other: &Self) {
         *self += other;
@@ -122,7 +123,11 @@ where
     }
 
     fn norm_squared(&self) -> f64 {
-        self.iter().map(|x| *x * *x).reduce(|sum, curr| sum + curr).unwrap_or(F::zero()).into()
+        self.iter()
+            .map(|x| *x * *x)
+            .reduce(|sum, curr| sum + curr)
+            .unwrap_or(F::zero())
+            .into()
     }
 }
 

src/derivable/loss.rs

@@ -24,11 +24,7 @@ impl NeuraLoss for Euclidean {
     }
 
     #[inline]
-    fn nabla(
+    fn nabla(&self, target: &DVector<f64>, actual: &DVector<f64>) -> DVector<f64> {
-        &self,
-        target: &DVector<f64>,
-        actual: &DVector<f64>,
-    ) -> DVector<f64> {
         let mut res = DVector::zeros(target.len());
 
         // ∂E(y)/∂yᵢ = yᵢ - yᵢ'

src/layer/dense.rs

@@ -17,12 +17,8 @@ pub struct NeuraDenseLayer<F: Float, Act: NeuraDerivable<F>, Reg: NeuraDerivable
 }
 
 #[derive(Clone, Debug)]
-pub struct NeuraDenseLayerPartial<
+pub struct NeuraDenseLayerPartial<F: Float, Act: NeuraDerivable<F>, Reg: NeuraDerivable<F>, R: Rng>
-    F: Float,
+{
-    Act: NeuraDerivable<F>,
-    Reg: NeuraDerivable<F>,
-    R: Rng,
-> {
     activation: Act,
     regularization: Reg,
     output_size: usize,
@@ -143,7 +139,13 @@ impl<
 }
 
 impl<
-        F: Float + From<f64> + Into<f64> + std::fmt::Debug + 'static + std::ops::AddAssign + std::ops::MulAssign,
+        F: Float
+            + From<f64>
+            + Into<f64>
+            + std::fmt::Debug
+            + 'static
+            + std::ops::AddAssign
+            + std::ops::MulAssign,
         Act: NeuraDerivable<F>,
         Reg: NeuraDerivable<F>,
     > NeuraTrainableLayer<DVector<F>> for NeuraDenseLayer<F, Act, Reg>
@@ -184,7 +186,10 @@ impl<
     }
 
     fn regularize_layer(&self) -> Self::Gradient {
-        (self.weights.map(|x| self.regularization.derivate(x)), DVector::zeros(self.bias.shape().0))
+        (
+            self.weights.map(|x| self.regularization.derivate(x)),
+            DVector::zeros(self.bias.shape().0),
+        )
     }
 
     fn apply_gradient(&mut self, gradient: &Self::Gradient) {

src/layer/mod.rs

@@ -1,5 +1,3 @@
-use num::Float;
-
 use crate::algebra::NeuraVectorSpace;
 
 pub mod dense;
@@ -7,8 +5,8 @@ pub use dense::NeuraDenseLayer;
 
 #[derive(Clone, Copy, PartialEq, Debug)]
 pub enum NeuraShape {
-    Vector(usize),        // entries
+    Vector(usize),               // entries
-    Matrix(usize, usize), // rows, columns
+    Matrix(usize, usize),        // rows, columns
     Tensor(usize, usize, usize), // rows, columns, channels
 }
 
@@ -17,7 +15,7 @@ impl NeuraShape {
         match self {
             NeuraShape::Vector(entries) => *entries,
             NeuraShape::Matrix(rows, columns) => rows * columns,
-            NeuraShape::Tensor(rows, columns, channels) => rows * columns * channels
+            NeuraShape::Tensor(rows, columns, channels) => rows * columns * channels,
         }
     }
 }
@@ -31,6 +29,7 @@ pub trait NeuraLayer<Input> {
 impl<Input: Clone> NeuraLayer<Input> for () {
     type Output = Input;
 
+    #[inline(always)]
     fn eval(&self, input: &Input) -> Self::Output {
         input.clone()
     }
@@ -62,11 +61,7 @@ pub trait NeuraTrainableLayer<Input>: NeuraLayer<Input> {
     /// The function should then return a pair `(epsilon_{l-1}, δW_l)`,
     /// with `epsilon_{l-1}` being multiplied by `f_{l-1}'(activation)` by the next layer to obtain `delta_{l-1}`.
     /// Using this intermediate value for `delta` allows us to isolate it computation to the respective layers.
-    fn backprop_layer(
+    fn backprop_layer(&self, input: &Input, epsilon: Self::Output) -> (Input, Self::Gradient);
-        &self,
-        input: &Input,
-        epsilon: Self::Output,
-    ) -> (Input, Self::Gradient);
 
     /// Computes the regularization
     fn regularize_layer(&self) -> Self::Gradient;
@@ -80,10 +75,39 @@ pub trait NeuraTrainableLayer<Input>: NeuraLayer<Input> {
     fn prepare_layer(&mut self, is_training: bool) {}
 }
 
+impl<Input: Clone> NeuraTrainableLayer<Input> for () {
+    type Gradient = ();
+
+    #[inline(always)]
+    fn default_gradient(&self) -> Self::Gradient {
+        ()
+    }
+
+    #[inline(always)]
+    fn backprop_layer(&self, _input: &Input, epsilon: Self::Output) -> (Input, Self::Gradient) {
+        (epsilon, ())
+    }
+
+    #[inline(always)]
+    fn regularize_layer(&self) -> Self::Gradient {
+        ()
+    }
+
+    #[inline(always)]
+    fn apply_gradient(&mut self, _gradient: &Self::Gradient) {
+        // Noop
+    }
+}
+
 /// Temporary implementation of neura_layer
 #[macro_export]
 macro_rules! neura_layer {
     ( "dense", $output:expr, $activation:expr ) => {
-        $crate::layer::dense::NeuraDenseLayer::new_partial($output, rand::thread_rng(), $activation, $crate::derivable::regularize::NeuraL0)
+        $crate::layer::dense::NeuraDenseLayer::new_partial(
-    }
+            $output,
+            rand::thread_rng(),
+            $activation,
+            $crate::derivable::regularize::NeuraL0,
+        )
+    };
 }

src/lib.rs

@@ -1,15 +1,12 @@
 #![feature(generic_arg_infer)]
-#![feature(generic_const_exprs)]
+// #![feature(generic_const_exprs)]
-#![feature(negative_impls)]
 
 pub mod algebra;
 pub mod derivable;
-// pub mod layer;
+pub mod layer;
 pub mod network;
 pub mod train;
 
-pub mod layer;
-
 mod utils;
 
 // TODO: move to a different file
@@ -21,6 +18,8 @@ pub mod prelude {
 
     // Structs and traits
     pub use crate::layer::*;
-    pub use crate::network::sequential::{NeuraSequential, NeuraSequentialTail, NeuraSequentialBuild};
+    pub use crate::network::sequential::{
+        NeuraSequential, NeuraSequentialConstruct, NeuraSequentialTail,
+    };
     pub use crate::train::{NeuraBackprop, NeuraBatchedTrainer};
 }

src/network/mod.rs

@@ -3,11 +3,11 @@ use crate::{algebra::NeuraVectorSpace, derivable::NeuraLoss, layer::NeuraLayer};
 pub mod sequential;
 
 pub trait NeuraTrainableNetwork<Input>: NeuraLayer<Input> {
-    type Delta: NeuraVectorSpace;
+    type Gradient: NeuraVectorSpace;
 
-    fn default_gradient(&self) -> Self::Delta;
+    fn default_gradient(&self) -> Self::Gradient;
 
-    fn apply_gradient(&mut self, gradient: &Self::Delta);
+    fn apply_gradient(&mut self, gradient: &Self::Gradient);
 
     /// Should implement the backpropagation algorithm, see `NeuraTrainableLayer::backpropagate` for more information.
     fn backpropagate<Loss: NeuraLoss<Input = Self::Output>>(
@@ -15,10 +15,10 @@ pub trait NeuraTrainableNetwork<Input>: NeuraLayer<Input> {
         input: &Input,
         target: &Loss::Target,
         loss: Loss,
-    ) -> (Input, Self::Delta);
+    ) -> (Input, Self::Gradient);
 
     /// Should return the regularization gradient
-    fn regularize(&self) -> Self::Delta;
+    fn regularize(&self) -> Self::Gradient;
 
     /// Called before an iteration begins, to allow the network to set itself up for training or not.
     fn prepare(&mut self, train_iteration: bool);

src/network/sequential.rs

@@ -1,298 +0,0 @@
-use num::Float;
-
-use crate::{
-    derivable::NeuraLoss,
-    layer::{NeuraLayer, NeuraTrainableLayer, NeuraShape, NeuraPartialLayer},
-};
-
-use super::NeuraTrainableNetwork;
-
-#[derive(Clone, Debug)]
-pub struct NeuraSequential<Layer, ChildNetwork> {
-    pub layer: Layer,
-    pub child_network: Box<ChildNetwork>,
-}
-
-/// Operations on the tail end of a sequential network
-pub trait NeuraSequentialTail {
-    type TailTrimmed;
-    type TailPushed<T>;
-
-    fn trim_tail(self) -> Self::TailTrimmed;
-    fn push_tail<T>(self, layer: T) -> Self::TailPushed<T>;
-}
-
-impl<Layer, ChildNetwork> NeuraSequential<Layer, ChildNetwork> {
-    pub fn new(layer: Layer, child_network: ChildNetwork) -> Self {
-        Self {
-            layer,
-            child_network: Box::new(child_network),
-        }
-    }
-
-    pub fn new_match_output<Input>(layer: Layer, child_network: ChildNetwork) -> Self
-    where
-        Layer: NeuraLayer<Input>,
-        ChildNetwork: NeuraLayer<Layer::Output>,
-    {
-        Self::new(layer, child_network)
-    }
-
-    pub fn trim_front(self) -> ChildNetwork {
-        *self.child_network
-    }
-
-    pub fn push_front<Input, Input2, T: NeuraLayer<Input2, Output=Input>>(self, layer: T) -> NeuraSequential<T, Self>
-    where
-        Layer: NeuraLayer<Input>
-    {
-        NeuraSequential {
-            layer: layer,
-            child_network: Box::new(self),
-        }
-    }
-}
-
-// Trimming the last layer returns an empty network
-impl<Layer> NeuraSequentialTail for NeuraSequential<Layer, ()> {
-    type TailTrimmed = ();
-    type TailPushed<T> = NeuraSequential<Layer, NeuraSequential<T, ()>>;
-
-    fn trim_tail(self) -> Self::TailTrimmed {
-        ()
-    }
-
-    fn push_tail<T>(self, layer: T) -> Self::TailPushed<T> {
-        NeuraSequential {
-            layer: self.layer,
-            child_network: Box::new(NeuraSequential {
-                layer,
-                child_network: Box::new(()),
-            }),
-        }
-    }
-}
-
-// Trimming another layer returns a network which calls trim recursively
-impl<Layer, ChildNetwork: NeuraSequentialTail> NeuraSequentialTail
-    for NeuraSequential<Layer, ChildNetwork>
-{
-    type TailTrimmed = NeuraSequential<Layer, <ChildNetwork as NeuraSequentialTail>::TailTrimmed>;
-    type TailPushed<T> =
-        NeuraSequential<Layer, <ChildNetwork as NeuraSequentialTail>::TailPushed<T>>;
-
-    fn trim_tail(self) -> Self::TailTrimmed {
-        NeuraSequential {
-            layer: self.layer,
-            child_network: Box::new(self.child_network.trim_tail()),
-        }
-    }
-
-    fn push_tail<T>(self, layer: T) -> Self::TailPushed<T> {
-        NeuraSequential {
-            layer: self.layer,
-            child_network: Box::new(self.child_network.push_tail(layer)),
-        }
-    }
-}
-
-impl<Input, Layer: NeuraLayer<Input>, ChildNetwork: NeuraLayer<Layer::Output>> NeuraLayer<Input>
-    for NeuraSequential<Layer, ChildNetwork>
-{
-    type Output = ChildNetwork::Output;
-
-    fn eval(&self, input: &Input) -> Self::Output {
-        self.child_network.eval(&self.layer.eval(input))
-    }
-}
-
-impl<Input: Clone> NeuraTrainableNetwork<Input> for () {
-    type Delta = ();
-
-    fn default_gradient(&self) -> () {
-        ()
-    }
-
-    fn apply_gradient(&mut self, _gradient: &()) {
-        // Noop
-    }
-
-    fn backpropagate<Loss: NeuraLoss<Input = Self::Output>>(
-        &self,
-        final_activation: &Input,
-        target: &Loss::Target,
-        loss: Loss,
-    ) -> (Input, Self::Delta) {
-        let backprop_epsilon = loss.nabla(target, &final_activation);
-
-        (backprop_epsilon, ())
-    }
-
-    fn regularize(&self) -> () {
-        ()
-    }
-
-    fn prepare(&mut self, _is_training: bool) {
-        // Noop
-    }
-}
-
-impl<Input, Layer: NeuraTrainableLayer<Input>, ChildNetwork: NeuraTrainableNetwork<Layer::Output>>
-    NeuraTrainableNetwork<Input> for NeuraSequential<Layer, ChildNetwork>
-{
-    type Delta = (Layer::Gradient, Box<ChildNetwork::Delta>);
-
-    fn default_gradient(&self) -> Self::Delta {
-        (self.layer.default_gradient(), Box::new(self.child_network.default_gradient()))
-    }
-
-    fn apply_gradient(&mut self, gradient: &Self::Delta) {
-        self.layer.apply_gradient(&gradient.0);
-        self.child_network.apply_gradient(&gradient.1);
-    }
-
-    fn backpropagate<Loss: NeuraLoss<Input = Self::Output>>(
-        &self,
-        input: &Input,
-        target: &Loss::Target,
-        loss: Loss,
-    ) -> (Input, Self::Delta) {
-        let next_activation = self.layer.eval(input);
-        let (backprop_gradient, weights_gradient) =
-            self.child_network
-                .backpropagate(&next_activation, target, loss);
-        let (backprop_gradient, layer_gradient) =
-            self.layer.backprop_layer(input, backprop_gradient);
-
-        (
-            backprop_gradient,
-            (layer_gradient, Box::new(weights_gradient)),
-        )
-    }
-
-    fn regularize(&self) -> Self::Delta {
-        (
-            self.layer.regularize_layer(),
-            Box::new(self.child_network.regularize()),
-        )
-    }
-
-    fn prepare(&mut self, is_training: bool) {
-        self.layer.prepare_layer(is_training);
-        self.child_network.prepare(is_training);
-    }
-}
-
-impl<Layer> From<Layer> for NeuraSequential<Layer, ()> {
-    fn from(layer: Layer) -> Self {
-        Self {
-            layer,
-            child_network: Box::new(()),
-        }
-    }
-}
-
-pub trait NeuraSequentialBuild {
-    type Constructed;
-    type Err;
-
-    fn construct(self, input_shape: NeuraShape) -> Result<Self::Constructed, Self::Err>;
-}
-
-#[derive(Debug, Clone)]
-pub enum NeuraSequentialBuildErr<Err, ChildErr> {
-    Current(Err),
-    Child(ChildErr),
-}
-
-impl<Layer: NeuraPartialLayer> NeuraSequentialBuild for NeuraSequential<Layer, ()> {
-    type Constructed = NeuraSequential<Layer::Constructed, ()>;
-    type Err = Layer::Err;
-
-    fn construct(self, input_shape: NeuraShape) -> Result<Self::Constructed, Self::Err> {
-        Ok(NeuraSequential {
-            layer: self.layer.construct(input_shape)?,
-            child_network: Box::new(())
-        })
-    }
-}
-
-impl<Layer: NeuraPartialLayer + , ChildNetwork: NeuraSequentialBuild> NeuraSequentialBuild for NeuraSequential<Layer, ChildNetwork> {
-    type Constructed = NeuraSequential<Layer::Constructed, ChildNetwork::Constructed>;
-    type Err = NeuraSequentialBuildErr<Layer::Err, ChildNetwork::Err>;
-
-    fn construct(self, input_shape: NeuraShape) -> Result<Self::Constructed, Self::Err> {
-        let layer = self.layer.construct(input_shape).map_err(|e| NeuraSequentialBuildErr::Current(e))?;
-
-        // TODO: ensure that this operation (and all recursive operations) are directly allocated on the heap
-        let child_network = self.child_network
-            .construct(Layer::output_shape(&layer))
-            .map_err(|e| NeuraSequentialBuildErr::Child(e))?;
-        let child_network = Box::new(child_network);
-
-        Ok(NeuraSequential {
-            layer,
-            child_network,
-        })
-    }
-
-
-}
-
-/// An utility to recursively create a NeuraSequential network, while writing it in a declarative and linear fashion.
-/// Note that this can quickly create big and unwieldly types.
-#[macro_export]
-macro_rules! neura_sequential {
-    [] => {
-        ()
-    };
-
-    [ $layer:expr $(,)? ] => {
-        $crate::network::sequential::NeuraSequential::from($layer)
-    };
-
-    [ $first:expr, $($rest:expr),+ $(,)? ] => {
-        $crate::network::sequential::NeuraSequential::new($first, neura_sequential![$($rest),+])
-    };
-}
-
-#[cfg(test)]
-mod test {
-    use nalgebra::dvector;
-
-    use crate::{
-        derivable::{activation::Relu, regularize::NeuraL0},
-        layer::{NeuraDenseLayer, NeuraShape, NeuraLayer},
-        neura_layer,
-    };
-
-    use super::NeuraSequentialBuild;
-
-    #[test]
-    fn test_neura_network_macro() {
-        let mut rng = rand::thread_rng();
-
-        let _ = neura_sequential![
-            NeuraDenseLayer::from_rng(8, 12, &mut rng, Relu, NeuraL0) as NeuraDenseLayer<f64, _, _>,
-            NeuraDenseLayer::from_rng(12, 16, &mut rng, Relu, NeuraL0) as NeuraDenseLayer<f64, _, _>,
-            NeuraDenseLayer::from_rng(16, 2, &mut rng, Relu, NeuraL0) as NeuraDenseLayer<f64, _, _>
-        ];
-
-        let _ = neura_sequential![
-            NeuraDenseLayer::from_rng(2, 2, &mut rng, Relu, NeuraL0) as NeuraDenseLayer<f64, _, _>,
-        ];
-
-        let _ = neura_sequential![
-            NeuraDenseLayer::from_rng(8, 16, &mut rng, Relu, NeuraL0) as NeuraDenseLayer<f64, _, _>,
-            NeuraDenseLayer::from_rng(16, 12, &mut rng, Relu, NeuraL0) as NeuraDenseLayer<f64, _, _>,
-        ];
-
-        let network = neura_sequential![
-            neura_layer!("dense", 16, Relu),
-            neura_layer!("dense", 12, Relu),
-            neura_layer!("dense", 2, Relu)
-        ].construct(NeuraShape::Vector(2)).unwrap();
-
-        network.eval(&dvector![0.0f64, 0.0]);
-    }
-}

src/network/sequential/construct.rs

@@ -0,0 +1,52 @@
+use super::*;
+
+pub trait NeuraSequentialConstruct {
+    type Constructed;
+    type Err;
+
+    fn construct(self, input_shape: NeuraShape) -> Result<Self::Constructed, Self::Err>;
+}
+
+#[derive(Debug, Clone)]
+pub enum NeuraSequentialConstructErr<Err, ChildErr> {
+    Current(Err),
+    Child(ChildErr),
+}
+
+impl<Layer: NeuraPartialLayer> NeuraSequentialConstruct for NeuraSequential<Layer, ()> {
+    type Constructed = NeuraSequential<Layer::Constructed, ()>;
+    type Err = Layer::Err;
+
+    fn construct(self, input_shape: NeuraShape) -> Result<Self::Constructed, Self::Err> {
+        Ok(NeuraSequential {
+            layer: self.layer.construct(input_shape)?,
+            child_network: Box::new(()),
+        })
+    }
+}
+
+impl<Layer: NeuraPartialLayer, ChildNetwork: NeuraSequentialConstruct> NeuraSequentialConstruct
+    for NeuraSequential<Layer, ChildNetwork>
+{
+    type Constructed = NeuraSequential<Layer::Constructed, ChildNetwork::Constructed>;
+    type Err = NeuraSequentialConstructErr<Layer::Err, ChildNetwork::Err>;
+
+    fn construct(self, input_shape: NeuraShape) -> Result<Self::Constructed, Self::Err> {
+        let layer = self
+            .layer
+            .construct(input_shape)
+            .map_err(|e| NeuraSequentialConstructErr::Current(e))?;
+
+        // TODO: ensure that this operation (and all recursive operations) are directly allocated on the heap
+        let child_network = self
+            .child_network
+            .construct(Layer::output_shape(&layer))
+            .map_err(|e| NeuraSequentialConstructErr::Child(e))?;
+        let child_network = Box::new(child_network);
+
+        Ok(NeuraSequential {
+            layer,
+            child_network,
+        })
+    }
+}

src/network/sequential/mod.rs

@@ -0,0 +1,287 @@
+use super::NeuraTrainableNetwork;
+use crate::{
+    derivable::NeuraLoss,
+    layer::{NeuraLayer, NeuraPartialLayer, NeuraShape, NeuraTrainableLayer},
+};
+
+mod construct;
+mod tail;
+
+pub use construct::*;
+pub use tail::*;
+
+/// Chains a layer with the rest of a neural network, in a fashion similar to a cartesian product,
+/// while preserving all type information.
+/// The type `Layer` represents the current layer of the neural network,
+/// and its output will be fed to the `ChildNetwork`, which will typically either be another `NeuraSequential`
+/// instance or `()`.
+///
+/// `ChildNetwork` is also free to be another implementation of `NeuraNetwork`,
+/// which allows `NeuraSequential` to be used together with other network structures.
+///
+/// `child_network` is stored in a `Box`, so as to avoid taking up too much space on the stack.
+///
+/// ## Notes on implemented traits
+///
+/// The different implementations for `NeuraTrainableNetwork`,
+/// `NeuraLayer` and `NeuraTrainableLayer` each require that `ChildNetwork` implements those respective traits,
+/// and that the output type of `Layer` matches the input type of `ChildNetwork`.
+///
+/// If a method, like `eval`, is reported as missing,
+/// then it likely means that the output type of `Layer` does not match the input type of `ChildNetwork`,
+/// or that a similar issue arose within `ChildNetwork`.
+///
+/// ## Trimming and appending layers
+///
+/// If you want to modify the network structure, you can do so by using the `trim_front`, `trim_tail`,
+/// `push_front` and `push_tail` methods.
+///
+/// The operations on the front are trivial, as it simply involves wrapping the current instance in a new `NeuraSequential`
+/// instance.
+///
+/// The operations on the tail end are more complex, and require recursively traversing the `NeuraSequential` structure,
+/// until an instance of `NeuraSequential<Layer, ()>` is found.
+/// If your network feeds into a type that does not implement `NeuraSequentialTail`, then you will not be able to use those operations.
+#[derive(Clone, Debug)]
+pub struct NeuraSequential<Layer, ChildNetwork> {
+    pub layer: Layer,
+    pub child_network: Box<ChildNetwork>,
+}
+
+impl<Layer, ChildNetwork> NeuraSequential<Layer, ChildNetwork> {
+    pub fn new(layer: Layer, child_network: ChildNetwork) -> Self {
+        Self {
+            layer,
+            child_network: Box::new(child_network),
+        }
+    }
+
+    pub fn trim_front(self) -> ChildNetwork {
+        *self.child_network
+    }
+
+    pub fn push_front<Input, Input2, T: NeuraLayer<Input2, Output = Input>>(
+        self,
+        layer: T,
+    ) -> NeuraSequential<T, Self>
+    where
+        Layer: NeuraLayer<Input>,
+    {
+        NeuraSequential {
+            layer: layer,
+            child_network: Box::new(self),
+        }
+    }
+}
+
+impl<Input, Layer: NeuraLayer<Input>, ChildNetwork: NeuraLayer<Layer::Output>> NeuraLayer<Input>
+    for NeuraSequential<Layer, ChildNetwork>
+{
+    type Output = ChildNetwork::Output;
+
+    fn eval(&self, input: &Input) -> Self::Output {
+        self.child_network.eval(&self.layer.eval(input))
+    }
+}
+
+impl<
+        Input,
+        Layer: NeuraTrainableLayer<Input>,
+        ChildNetwork: NeuraTrainableLayer<Layer::Output>,
+    > NeuraTrainableLayer<Input> for NeuraSequential<Layer, ChildNetwork>
+{
+    type Gradient = (Layer::Gradient, Box<ChildNetwork::Gradient>);
+
+    fn default_gradient(&self) -> Self::Gradient {
+        (
+            self.layer.default_gradient(),
+            Box::new(self.child_network.default_gradient()),
+        )
+    }
+
+    fn backprop_layer(
+        &self,
+        input: &Input,
+        incoming_epsilon: Self::Output,
+    ) -> (Input, Self::Gradient) {
+        let output = self.layer.eval(input);
+        let (transient_epsilon, child_gradient) =
+            self.child_network.backprop_layer(&output, incoming_epsilon);
+        let (outgoing_epsilon, layer_gradient) =
+            self.layer.backprop_layer(input, transient_epsilon);
+
+        (outgoing_epsilon, (layer_gradient, Box::new(child_gradient)))
+    }
+
+    fn regularize_layer(&self) -> Self::Gradient {
+        (
+            self.layer.regularize_layer(),
+            Box::new(self.child_network.regularize_layer()),
+        )
+    }
+
+    fn apply_gradient(&mut self, gradient: &Self::Gradient) {
+        self.layer.apply_gradient(&gradient.0);
+        self.child_network.apply_gradient(&gradient.1);
+    }
+}
+
+impl<
+        Input,
+        Layer: NeuraTrainableLayer<Input>,
+        ChildNetwork: NeuraTrainableNetwork<Layer::Output>,
+    > NeuraTrainableNetwork<Input> for NeuraSequential<Layer, ChildNetwork>
+{
+    type Gradient = (Layer::Gradient, Box<ChildNetwork::Gradient>);
+
+    fn default_gradient(&self) -> Self::Gradient {
+        (
+            self.layer.default_gradient(),
+            Box::new(self.child_network.default_gradient()),
+        )
+    }
+
+    fn apply_gradient(&mut self, gradient: &Self::Gradient) {
+        self.layer.apply_gradient(&gradient.0);
+        self.child_network.apply_gradient(&gradient.1);
+    }
+
+    fn backpropagate<Loss: NeuraLoss<Input = Self::Output>>(
+        &self,
+        input: &Input,
+        target: &Loss::Target,
+        loss: Loss,
+    ) -> (Input, Self::Gradient) {
+        let next_activation = self.layer.eval(input);
+        let (backprop_gradient, weights_gradient) =
+            self.child_network
+                .backpropagate(&next_activation, target, loss);
+        let (backprop_gradient, layer_gradient) =
+            self.layer.backprop_layer(input, backprop_gradient);
+
+        (
+            backprop_gradient,
+            (layer_gradient, Box::new(weights_gradient)),
+        )
+    }
+
+    fn regularize(&self) -> Self::Gradient {
+        (
+            self.layer.regularize_layer(),
+            Box::new(self.child_network.regularize()),
+        )
+    }
+
+    fn prepare(&mut self, is_training: bool) {
+        self.layer.prepare_layer(is_training);
+        self.child_network.prepare(is_training);
+    }
+}
+
+/// A dummy implementation of `NeuraTrainableNetwork`, which simply calls `loss.eval` in `backpropagate`.
+impl<Input: Clone> NeuraTrainableNetwork<Input> for () {
+    type Gradient = ();
+
+    #[inline(always)]
+    fn default_gradient(&self) -> () {
+        ()
+    }
+
+    #[inline(always)]
+    fn apply_gradient(&mut self, _gradient: &()) {
+        // Noop
+    }
+
+    #[inline(always)]
+    fn backpropagate<Loss: NeuraLoss<Input = Self::Output>>(
+        &self,
+        final_activation: &Input,
+        target: &Loss::Target,
+        loss: Loss,
+    ) -> (Input, Self::Gradient) {
+        let backprop_epsilon = loss.nabla(target, &final_activation);
+
+        (backprop_epsilon, ())
+    }
+
+    #[inline(always)]
+    fn regularize(&self) -> () {
+        ()
+    }
+
+    #[inline(always)]
+    fn prepare(&mut self, _is_training: bool) {
+        // Noop
+    }
+}
+
+impl<Layer> From<Layer> for NeuraSequential<Layer, ()> {
+    fn from(layer: Layer) -> Self {
+        Self {
+            layer,
+            child_network: Box::new(()),
+        }
+    }
+}
+
+/// An utility to recursively create a NeuraSequential network, while writing it in a declarative and linear fashion.
+/// Note that this can quickly create big and unwieldly types.
+#[macro_export]
+macro_rules! neura_sequential {
+    [] => {
+        ()
+    };
+
+    [ $layer:expr $(,)? ] => {
+        $crate::network::sequential::NeuraSequential::from($layer)
+    };
+
+    [ $first:expr, $($rest:expr),+ $(,)? ] => {
+        $crate::network::sequential::NeuraSequential::new($first, neura_sequential![$($rest),+])
+    };
+}
+
+#[cfg(test)]
+mod test {
+    use nalgebra::dvector;
+
+    use crate::{
+        derivable::{activation::Relu, regularize::NeuraL0},
+        layer::{NeuraDenseLayer, NeuraLayer, NeuraShape},
+        neura_layer,
+    };
+
+    use super::NeuraSequentialConstruct;
+
+    #[test]
+    fn test_neura_network_macro() {
+        let mut rng = rand::thread_rng();
+
+        let _ = neura_sequential![
+            NeuraDenseLayer::from_rng(8, 12, &mut rng, Relu, NeuraL0) as NeuraDenseLayer<f64, _, _>,
+            NeuraDenseLayer::from_rng(12, 16, &mut rng, Relu, NeuraL0)
+                as NeuraDenseLayer<f64, _, _>,
+            NeuraDenseLayer::from_rng(16, 2, &mut rng, Relu, NeuraL0) as NeuraDenseLayer<f64, _, _>
+        ];
+
+        let _ =
+            neura_sequential![NeuraDenseLayer::from_rng(2, 2, &mut rng, Relu, NeuraL0)
+                as NeuraDenseLayer<f64, _, _>,];
+
+        let _ = neura_sequential![
+            NeuraDenseLayer::from_rng(8, 16, &mut rng, Relu, NeuraL0) as NeuraDenseLayer<f64, _, _>,
+            NeuraDenseLayer::from_rng(16, 12, &mut rng, Relu, NeuraL0)
+                as NeuraDenseLayer<f64, _, _>,
+        ];
+
+        let network = neura_sequential![
+            neura_layer!("dense", 16, Relu),
+            neura_layer!("dense", 12, Relu),
+            neura_layer!("dense", 2, Relu)
+        ]
+        .construct(NeuraShape::Vector(2))
+        .unwrap();
+
+        network.eval(&dvector![0.0f64, 0.0]);
+    }
+}

src/network/sequential/tail.rs

@@ -0,0 +1,54 @@
+use super::*;
+
+/// Operations on the tail end of a sequential network
+pub trait NeuraSequentialTail {
+    type TailTrimmed;
+    type TailPushed<T>;
+
+    fn trim_tail(self) -> Self::TailTrimmed;
+    fn push_tail<T>(self, layer: T) -> Self::TailPushed<T>;
+}
+
+// Trimming the last layer returns an empty network
+impl<Layer> NeuraSequentialTail for NeuraSequential<Layer, ()> {
+    type TailTrimmed = ();
+    // GAT :3
+    type TailPushed<T> = NeuraSequential<Layer, NeuraSequential<T, ()>>;
+
+    fn trim_tail(self) -> Self::TailTrimmed {
+        ()
+    }
+
+    fn push_tail<T>(self, layer: T) -> Self::TailPushed<T> {
+        NeuraSequential {
+            layer: self.layer,
+            child_network: Box::new(NeuraSequential {
+                layer,
+                child_network: Box::new(()),
+            }),
+        }
+    }
+}
+
+// Trimming another layer returns a network which calls trim recursively
+impl<Layer, ChildNetwork: NeuraSequentialTail> NeuraSequentialTail
+    for NeuraSequential<Layer, ChildNetwork>
+{
+    type TailTrimmed = NeuraSequential<Layer, <ChildNetwork as NeuraSequentialTail>::TailTrimmed>;
+    type TailPushed<T> =
+        NeuraSequential<Layer, <ChildNetwork as NeuraSequentialTail>::TailPushed<T>>;
+
+    fn trim_tail(self) -> Self::TailTrimmed {
+        NeuraSequential {
+            layer: self.layer,
+            child_network: Box::new(self.child_network.trim_tail()),
+        }
+    }
+
+    fn push_tail<T>(self, layer: T) -> Self::TailPushed<T> {
+        NeuraSequential {
+            layer: self.layer,
+            child_network: Box::new(self.child_network.push_tail(layer)),
+        }
+    }
+}

src/train.rs

@@ -1,9 +1,4 @@
-use crate::{
+use crate::{algebra::NeuraVectorSpace, derivable::NeuraLoss, network::NeuraTrainableNetwork};
-    algebra::{NeuraVector, NeuraVectorSpace},
-    derivable::NeuraLoss,
-    layer::NeuraLayer,
-    network::{sequential::NeuraSequential, NeuraTrainableNetwork},
-};
 
 pub trait NeuraGradientSolver<Input, Target, Trainable: NeuraTrainableNetwork<Input>> {
     fn get_gradient(
@@ -11,14 +6,9 @@ pub trait NeuraGradientSolver<Input, Target, Trainable: NeuraTrainableNetwork<In
         trainable: &Trainable,
         input: &Input,
         target: &Target,
-    ) -> Trainable::Delta;
+    ) -> Trainable::Gradient;
 
-    fn score(
+    fn score(&self, trainable: &Trainable, input: &Input, target: &Target) -> f64;
-        &self,
-        trainable: &Trainable,
-        input: &Input,
-        target: &Target,
-    ) -> f64;
 }
 
 #[non_exhaustive]
@@ -32,24 +22,23 @@ impl<Loss: NeuraLoss + Clone> NeuraBackprop<Loss> {
     }
 }
 
-impl<Input, Target, Trainable: NeuraTrainableNetwork<Input>, Loss: NeuraLoss<Input = Trainable::Output, Target = Target> + Clone>
+impl<
-    NeuraGradientSolver<Input, Target, Trainable> for NeuraBackprop<Loss>
+        Input,
+        Target,
+        Trainable: NeuraTrainableNetwork<Input>,
+        Loss: NeuraLoss<Input = Trainable::Output, Target = Target> + Clone,
+    > NeuraGradientSolver<Input, Target, Trainable> for NeuraBackprop<Loss>
 {
     fn get_gradient(
         &self,
         trainable: &Trainable,
         input: &Input,
         target: &Target,
-    ) -> Trainable::Delta {
+    ) -> Trainable::Gradient {
         trainable.backpropagate(input, target, self.loss.clone()).1
     }
 
-    fn score(
+    fn score(&self, trainable: &Trainable, input: &Input, target: &Target) -> f64 {
-        &self,
-        trainable: &Trainable,
-        input: &Input,
-        target: &Target,
-    ) -> f64 {
         let output = trainable.eval(&input);
         self.loss.eval(target, &output)
     }
@@ -187,14 +176,14 @@ impl NeuraBatchedTrainer {
 
 #[cfg(test)]
 mod test {
-    use nalgebra::{DMatrix, dmatrix, dvector};
+    use nalgebra::{dmatrix, dvector};
 
     use super::*;
     use crate::{
         assert_approx,
         derivable::{activation::Linear, loss::Euclidean, regularize::NeuraL0},
-        layer::NeuraDenseLayer,
+        layer::{NeuraLayer, NeuraDenseLayer},
-        network::sequential::NeuraSequentialTail,
+        network::sequential::{NeuraSequentialTail, NeuraSequential},
         neura_sequential,
     };
 
@@ -242,18 +231,14 @@ mod test {
         assert_approx!(0.48, intermediary[1], EPSILON);
         assert_approx!(0.191, network.eval(&input)[0], EPSILON);
 
-        assert_approx!(
+        assert_approx!(0.327, Euclidean.eval(&target, &network.eval(&input)), 0.001);
-            0.327,
-            Euclidean.eval(&target, &network.eval(&input)),
-            0.001
-        );
 
         let delta = network.eval(&input)[0] - target[0];
 
         let (gradient_first, gradient_second) =
             NeuraBackprop::new(Euclidean).get_gradient(&network, &input, &target);
         let gradient_first = gradient_first.0;
-        let gradient_second = gradient_second.0.0;
+        let gradient_second = gradient_second.0 .0;
 
         assert_approx!(gradient_second[0], intermediary[0] * delta, EPSILON);
         assert_approx!(gradient_second[1], intermediary[1] * delta, EPSILON);