It was becoming almost impossible to manage the dimensions of the layers, especially with convolution layers. Generic consts are nice, but they are a bit too early to have right now for this use-case. We'll probably be expanding the implementations to accept const or dynamically-sized layers at some point, for performance-critical applications.main
parent
cc7686569a
commit
2edbff860c
@ -0,0 +1,180 @@
|
||||
use super::{NeuraLayer, NeuraTrainableLayer};
|
||||
use crate::{
|
||||
algebra::{NeuraMatrix, NeuraVector, NeuraVectorSpace},
|
||||
derivable::NeuraDerivable,
|
||||
};
|
||||
|
||||
use rand::Rng;
|
||||
use rand_distr::Distribution;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct NeuraDenseLayer<
|
||||
Act: NeuraDerivable<f64>,
|
||||
Reg: NeuraDerivable<f64>,
|
||||
const INPUT_LEN: usize,
|
||||
const OUTPUT_LEN: usize,
|
||||
> {
|
||||
weights: NeuraMatrix<INPUT_LEN, OUTPUT_LEN, f64>,
|
||||
bias: NeuraVector<OUTPUT_LEN, f64>,
|
||||
activation: Act,
|
||||
regularization: Reg,
|
||||
}
|
||||
|
||||
impl<
|
||||
Act: NeuraDerivable<f64>,
|
||||
Reg: NeuraDerivable<f64>,
|
||||
const INPUT_LEN: usize,
|
||||
const OUTPUT_LEN: usize,
|
||||
> NeuraDenseLayer<Act, Reg, INPUT_LEN, OUTPUT_LEN>
|
||||
{
|
||||
pub fn new(
|
||||
weights: NeuraMatrix<INPUT_LEN, OUTPUT_LEN, f64>,
|
||||
bias: NeuraVector<OUTPUT_LEN, f64>,
|
||||
activation: Act,
|
||||
regularization: Reg,
|
||||
) -> Self {
|
||||
Self {
|
||||
weights,
|
||||
bias,
|
||||
activation,
|
||||
regularization,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_rng(rng: &mut impl Rng, activation: Act, regularization: Reg) -> Self {
|
||||
let mut weights: NeuraMatrix<INPUT_LEN, OUTPUT_LEN, f64> = NeuraMatrix::from_value(0.0f64);
|
||||
|
||||
// Use Xavier (or He) initialisation, using the harmonic mean
|
||||
// Ref: https://www.deeplearning.ai/ai-notes/initialization/index.html
|
||||
let distribution = rand_distr::Normal::new(
|
||||
0.0,
|
||||
activation.variance_hint() * 2.0 / (INPUT_LEN as f64 + OUTPUT_LEN as f64),
|
||||
)
|
||||
.unwrap();
|
||||
// let distribution = rand_distr::Uniform::new(-0.5, 0.5);
|
||||
|
||||
for i in 0..OUTPUT_LEN {
|
||||
for j in 0..INPUT_LEN {
|
||||
weights[i][j] = distribution.sample(rng);
|
||||
}
|
||||
}
|
||||
|
||||
Self {
|
||||
weights,
|
||||
// Biases are initialized based on the activation's hint
|
||||
bias: NeuraVector::from_value(activation.bias_hint()),
|
||||
activation,
|
||||
regularization,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<
|
||||
Act: NeuraDerivable<f64>,
|
||||
Reg: NeuraDerivable<f64>,
|
||||
const INPUT_LEN: usize,
|
||||
const OUTPUT_LEN: usize,
|
||||
> NeuraLayer for NeuraDenseLayer<Act, Reg, INPUT_LEN, OUTPUT_LEN>
|
||||
{
|
||||
type Input = NeuraVector<INPUT_LEN, f64>;
|
||||
|
||||
type Output = NeuraVector<OUTPUT_LEN, f64>;
|
||||
|
||||
fn eval(&self, input: &Self::Input) -> Self::Output {
|
||||
let mut result = self.weights.multiply_vector(input);
|
||||
|
||||
for i in 0..OUTPUT_LEN {
|
||||
result[i] = self.activation.eval(result[i] + self.bias[i]);
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl<
|
||||
Act: NeuraDerivable<f64>,
|
||||
Reg: NeuraDerivable<f64>,
|
||||
const INPUT_LEN: usize,
|
||||
const OUTPUT_LEN: usize,
|
||||
> NeuraTrainableLayer for NeuraDenseLayer<Act, Reg, INPUT_LEN, OUTPUT_LEN>
|
||||
{
|
||||
type Delta = (
|
||||
NeuraMatrix<INPUT_LEN, OUTPUT_LEN, f64>,
|
||||
NeuraVector<OUTPUT_LEN, f64>,
|
||||
);
|
||||
|
||||
fn backpropagate(
|
||||
&self,
|
||||
input: &Self::Input,
|
||||
epsilon: Self::Output,
|
||||
) -> (Self::Input, Self::Delta) {
|
||||
let evaluated = self.weights.multiply_vector(input);
|
||||
// Compute delta (the input gradient of the neuron) from epsilon (the output gradient of the neuron),
|
||||
// with `self.activation'(input) ° epsilon = delta`
|
||||
let mut delta: NeuraVector<OUTPUT_LEN, f64> = epsilon.clone();
|
||||
for i in 0..OUTPUT_LEN {
|
||||
delta[i] *= self.activation.derivate(evaluated[i]);
|
||||
}
|
||||
|
||||
// Compute the weight gradient
|
||||
let weights_gradient = delta.reverse_dot(input);
|
||||
|
||||
let new_epsilon = self.weights.transpose_multiply_vector(&delta);
|
||||
|
||||
// According to https://datascience.stackexchange.com/questions/20139/gradients-for-bias-terms-in-backpropagation
|
||||
// The gradient of the bias is equal to the delta term of the backpropagation algorithm
|
||||
let bias_gradient = delta;
|
||||
|
||||
(new_epsilon, (weights_gradient, bias_gradient))
|
||||
}
|
||||
|
||||
fn apply_gradient(&mut self, gradient: &Self::Delta) {
|
||||
NeuraVectorSpace::add_assign(&mut self.weights, &gradient.0);
|
||||
NeuraVectorSpace::add_assign(&mut self.bias, &gradient.1);
|
||||
}
|
||||
|
||||
fn regularize(&self) -> Self::Delta {
|
||||
let mut res = Self::Delta::default();
|
||||
|
||||
for i in 0..OUTPUT_LEN {
|
||||
for j in 0..INPUT_LEN {
|
||||
res.0[i][j] = self.regularization.derivate(self.weights[i][j]);
|
||||
}
|
||||
}
|
||||
|
||||
// Note: biases aren't taken into account here, as per https://stats.stackexchange.com/questions/153605/no-regularisation-term-for-bias-unit-in-neural-network
|
||||
|
||||
res
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use crate::{
|
||||
derivable::{activation::Relu, regularize::NeuraL0},
|
||||
utils::uniform_vector,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn test_from_rng() {
|
||||
let mut rng = rand::thread_rng();
|
||||
let layer: NeuraDenseLayer<_, _, 64, 32> =
|
||||
NeuraDenseLayer::from_rng(&mut rng, Relu, NeuraL0);
|
||||
let mut input = [0.0; 64];
|
||||
for x in 0..64 {
|
||||
input[x] = rng.gen();
|
||||
}
|
||||
assert!(layer.eval(&input.into()).len() == 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stack_overflow_big_layer() {
|
||||
let layer = NeuraDenseLayer::from_rng(&mut rand::thread_rng(), Relu, NeuraL0)
|
||||
as NeuraDenseLayer<Relu, NeuraL0, 1000, 1000>;
|
||||
|
||||
layer.backpropagate(&uniform_vector(), uniform_vector());
|
||||
|
||||
<NeuraDenseLayer<Relu, NeuraL0, 1000, 1000> as NeuraTrainableLayer>::Delta::zero();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,170 @@
|
||||
mod dense;
|
||||
pub use dense::NeuraDenseLayer;
|
||||
|
||||
mod convolution;
|
||||
pub use convolution::{NeuraConv1DPadLayer, NeuraConv2DBlockLayer, NeuraConv2DPadLayer};
|
||||
|
||||
mod dropout;
|
||||
pub use dropout::NeuraDropoutLayer;
|
||||
|
||||
mod softmax;
|
||||
pub use softmax::NeuraSoftmaxLayer;
|
||||
|
||||
mod one_hot;
|
||||
pub use one_hot::NeuraOneHotLayer;
|
||||
|
||||
mod lock;
|
||||
pub use lock::NeuraLockLayer;
|
||||
|
||||
mod pool;
|
||||
pub use pool::{NeuraGlobalPoolLayer, NeuraPool1DLayer};
|
||||
|
||||
mod reshape;
|
||||
pub use reshape::{NeuraFlattenLayer, NeuraReshapeLayer};
|
||||
|
||||
use crate::algebra::NeuraVectorSpace;
|
||||
|
||||
pub trait NeuraLayer {
|
||||
type Input;
|
||||
type Output;
|
||||
|
||||
fn eval(&self, input: &Self::Input) -> Self::Output;
|
||||
}
|
||||
|
||||
pub trait NeuraTrainableLayer: NeuraLayer {
|
||||
/// The representation of the layer gradient as a vector space
|
||||
type Delta: NeuraVectorSpace;
|
||||
|
||||
/// Computes the backpropagation term and the derivative of the internal weights,
|
||||
/// using the `input` vector outputted by the previous layer and the backpropagation term `epsilon` of the next layer.
|
||||
///
|
||||
/// Note: we introduce the term `epsilon`, which together with the activation of the current function can be used to compute `delta_l`:
|
||||
/// ```no_rust
|
||||
/// f_l'(a_l) * epsilon_l = delta_l
|
||||
/// ```
|
||||
///
|
||||
/// The function should then return a pair `(epsilon_{l-1}, δW_l)`,
|
||||
/// with `epsilon_{l-1}` being multiplied by `f_{l-1}'(activation)` by the next layer to obtain `delta_{l-1}`.
|
||||
/// Using this intermediate value for `delta` allows us to isolate it computation to the respective layers.
|
||||
fn backpropagate(
|
||||
&self,
|
||||
input: &Self::Input,
|
||||
epsilon: Self::Output,
|
||||
) -> (Self::Input, Self::Delta);
|
||||
|
||||
/// Computes the regularization
|
||||
fn regularize(&self) -> Self::Delta;
|
||||
|
||||
/// Applies `δW_l` to the weights of the layer
|
||||
fn apply_gradient(&mut self, gradient: &Self::Delta);
|
||||
|
||||
/// Called before an iteration begins, to allow the layer to set itself up for training.
|
||||
#[inline(always)]
|
||||
fn prepare_epoch(&mut self) {}
|
||||
|
||||
/// Called at the end of training, to allow the layer to clean itself up
|
||||
#[inline(always)]
|
||||
fn cleanup(&mut self) {}
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! neura_layer {
|
||||
( "dense", $( $shape:expr ),*; $activation:expr ) => {
|
||||
$crate::layer::NeuraDenseLayer::from_rng(&mut rand::thread_rng(), $activation, $crate::derivable::regularize::NeuraL0)
|
||||
as neura_layer!("_dense_shape", $($shape),*)
|
||||
};
|
||||
|
||||
( "dense", $( $shape:expr ),*; $activation:expr, $regularization:expr ) => {
|
||||
$crate::layer::NeuraDenseLayer::from_rng(&mut rand::thread_rng(), $activation, $regularization)
|
||||
as neura_layer!("_dense_shape", $($shape),*)
|
||||
};
|
||||
|
||||
( "_dense_shape", $output:expr ) => {
|
||||
$crate::layer::NeuraDenseLayer<_, _, _, $output>
|
||||
};
|
||||
|
||||
( "_dense_shape", $input:expr, $output:expr ) => {
|
||||
$crate::layer::NeuraDenseLayer<_, _, $input, $output>
|
||||
};
|
||||
|
||||
( "dropout", $probability:expr ) => {
|
||||
$crate::layer::NeuraDropoutLayer::new($probability, rand::thread_rng())
|
||||
as $crate::layer::NeuraDropoutLayer<_, _>
|
||||
};
|
||||
|
||||
( "softmax" ) => {
|
||||
$crate::layer::NeuraSoftmaxLayer::new() as $crate::layer::NeuraSoftmaxLayer<_>
|
||||
};
|
||||
|
||||
( "softmax", $length:expr ) => {
|
||||
$crate::layer::NeuraSoftmaxLayer::new() as $crate::layer::NeuraSoftmaxLayer<$length>
|
||||
};
|
||||
|
||||
( "one_hot" ) => {
|
||||
$crate::layer::NeuraOneHotLayer as $crate::layer::NeuraOneHotLayer<2, _>
|
||||
};
|
||||
|
||||
( "lock", $layer:expr ) => {
|
||||
$crate::layer::NeuraLockLayer($layer)
|
||||
};
|
||||
|
||||
( "conv1d_pad", $length:expr, $feats:expr; $window:expr; $layer:expr ) => {
|
||||
$crate::layer::NeuraConv1DPadLayer::new($layer, Default::default()) as $crate::layer::NeuraConv1DPadLayer<$length, $feats, $window, _>
|
||||
};
|
||||
|
||||
( "conv1d_pad"; $window:expr; $layer:expr ) => {
|
||||
$crate::layer::NeuraConv1DPadLayer::new($layer, Default::default()) as $crate::layer::NeuraConv1DPadLayer<_, _, $window, _>
|
||||
};
|
||||
|
||||
( "conv2d_pad", $feats:expr, $length:expr; $width:expr, $window:expr; $layer:expr ) => {
|
||||
$crate::layer::NeuraConv2DPadLayer::new($layer, Default::default(), $width) as $crate::layer::NeuraConv2DPadLayer<$length, $feats, $window, _>
|
||||
};
|
||||
|
||||
( "conv2d_pad"; $width:expr, $window:expr; $layer:expr ) => {
|
||||
$crate::layer::NeuraConv2DPadLayer::new($layer, Default::default(), $width) as $crate::layer::NeuraConv2DPadLayer<_, _, $window, _>
|
||||
};
|
||||
|
||||
( "conv2d_block", $feats:expr, $width:expr, $height:expr; $block_size:expr; $layer:expr ) => {
|
||||
$crate::layer::NeuraConv2DBlockLayer::new($layer) as $crate::layer::NeuraConv2DBlockLayer<$width, $height, $feats, $block_size, _>
|
||||
};
|
||||
|
||||
( "conv2d_block", $width:expr, $height:expr; $block_size:expr; $layer:expr ) => {
|
||||
$crate::layer::NeuraConv2DBlockLayer::new($layer) as $crate::layer::NeuraConv2DBlockLayer<$width, $height, _, $block_size, _>
|
||||
};
|
||||
|
||||
( "pool_global"; $reduce:expr ) => {
|
||||
$crate::layer::NeuraGlobalPoolLayer::new($reduce) as $crate::layer::NeuraGlobalPoolLayer<_, _, _>
|
||||
};
|
||||
|
||||
( "pool_global", $feats:expr, $length:expr; $reduce:expr ) => {
|
||||
$crate::layer::NeuraGlobalPoolLayer::new($reduce) as $crate::layer::NeuraGlobalPoolLayer<$length, $feats, _>
|
||||
};
|
||||
|
||||
( "pool1d", $blocklength:expr; $reduce:expr ) => {
|
||||
$crate::layer::NeuraPool1DLayer::new($reduce) as $crate::layer::NeuraPool1DLayer<_, $blocklength, _, _>
|
||||
};
|
||||
|
||||
( "pool1d", $blocks:expr, $blocklength:expr; $reduce:expr ) => {
|
||||
$crate::layer::NeuraPool1DLayer::new($reduce) as $crate::layer::NeuraPool1DLayer<$blocks, $blocklength, _, _>
|
||||
};
|
||||
|
||||
( "pool1d", $feats:expr, $blocks:expr, $blocklength:expr; $reduce:expr ) => {
|
||||
$crate::layer::NeuraPool1DLayer::new($reduce) as $crate::layer::NeuraPool1DLayer<$blocks, $blocklength, $feats, _>
|
||||
};
|
||||
|
||||
( "unstable_flatten" ) => {
|
||||
$crate::layer::NeuraFlattenLayer::new() as $crate::layer::NeuraFlattenLayer<_, _, f64>
|
||||
};
|
||||
|
||||
( "unstable_flatten", $width:expr, $height:expr ) => {
|
||||
$crate::layer::NeuraFlattenLayer::new() as $crate::layer::NeuraFlattenLayer<$width, $height, f64>
|
||||
};
|
||||
|
||||
( "unstable_reshape", $height:expr ) => {
|
||||
$crate::layer::NeuraReshapeLayer::new() as $crate::layer::NeuraReshapeLayer<_, $height, f64>
|
||||
};
|
||||
|
||||
( "unstable_reshape", $width:expr, $height:expr ) => {
|
||||
$crate::layer::NeuraReshapeLayer::new() as $crate::layer::NeuraReshapeLayer<$width, $height, f64>
|
||||
};
|
||||
}
|
||||
Loading…
Reference in new issue