diff --git a/examples/bivariate.rs b/examples/bivariate.rs index 115925c..89fea50 100644 --- a/examples/bivariate.rs +++ b/examples/bivariate.rs @@ -1,11 +1,12 @@ use std::io::Write; +#[allow(unused_imports)] use nalgebra::{dvector, DVector}; #[allow(unused_imports)] use neuramethyst::derivable::activation::{LeakyRelu, Linear, Relu, Tanh}; use neuramethyst::derivable::loss::CrossEntropy; use neuramethyst::derivable::regularize::NeuraL1; -use neuramethyst::{plot_losses, prelude::*}; +use neuramethyst::{one_hot, plot_losses, prelude::*}; use rand::Rng; @@ -132,11 +133,3 @@ fn draw_neuron_activation Vec>(callback: F, scale: f64) viuer::print(&image::DynamicImage::ImageRgb8(image), &config).unwrap(); } - -fn one_hot(value: usize, categories: usize) -> DVector { - let mut res = DVector::from_element(categories, 0.0); - if value < categories { - res[value] = 1.0; - } - res -} diff --git a/examples/densenet.rs b/examples/densenet.rs index 3bcfdd8..85018dc 100644 --- a/examples/densenet.rs +++ b/examples/densenet.rs @@ -4,7 +4,7 @@ use nalgebra::{dvector, DVector}; use neuramethyst::derivable::activation::Linear; use neuramethyst::derivable::loss::CrossEntropy; use neuramethyst::derivable::regularize::NeuraL1; -use neuramethyst::{plot_losses, prelude::*}; +use neuramethyst::{one_hot, plot_losses, prelude::*}; use rand::Rng; @@ -95,11 +95,3 @@ fn main() { writeln!(&mut file, "{},{},{}", input[0], input[1], guess).unwrap(); } } - -fn one_hot(value: usize, categories: usize) -> DVector { - let mut res = DVector::from_element(categories, 0.0); - if value < categories { - res[value] = 1.0; - } - res -} diff --git a/examples/mnist-decoder.rs b/examples/mnist-decoder.rs index 2d59b5f..0df81f0 100644 --- a/examples/mnist-decoder.rs +++ b/examples/mnist-decoder.rs @@ -1,4 +1,5 @@ use nalgebra::DVector; +use rand::Rng; use rust_mnist::Mnist; use neuramethyst::{ @@ -7,7 +8,7 @@ use neuramethyst::{ activation::{Linear, Logistic, Relu, Swish, Tanh}, loss::{CrossEntropy, Euclidean}, }, - plot_losses, + one_hot, plot_losses, prelude::*, }; @@ -59,23 +60,27 @@ pub fn main() { neura_layer!("dense", 100).activation(Swish(Logistic)), neura_layer!("dense", 50).activation(Swish(Logistic)), neura_layer!("dense", LATENT_SIZE).activation(Tanh), - neura_layer!("dense", 50), - neura_layer!("dense", 100), + neura_layer!("dense", 100).activation(Swish(Logistic)), neura_layer!("dense", WIDTH * HEIGHT).activation(Relu), ] .construct(NeuraShape::Vector(WIDTH * HEIGHT)) .unwrap(); - let trainer = NeuraBatchedTrainer::with_epochs(0.03, 75, 512, TRAIN_SIZE); + let mut trainer = NeuraBatchedTrainer::with_epochs(0.03, 200, 512, TRAIN_SIZE); + trainer.learning_momentum = 0.002; // trainer.log_iterations = 1; + let mut rng = rand::thread_rng(); let losses = trainer.train( &NeuraBackprop::new(Euclidean), &mut network, - cycle_shuffling( - train_images.clone().zip(train_images.clone()), - rand::thread_rng(), - ), + cycle_shuffling(train_images.clone(), rand::thread_rng()).map(move |input| { + let dx = rng.gen_range(-4..4); + let dy = rng.gen_range(-4..4); + + let shifted = shift(&input, dx, dy); + (shifted.clone(), shifted) + }), &test_data, ); @@ -83,7 +88,7 @@ pub fn main() { // Then, train a small network to decode the encoded data into the categories - let trimmed_network = network.clone().trim_tail().trim_tail().trim_tail(); + let trimmed_network = network.clone().trim_tail().trim_tail(); let mut network = neura_sequential![ ..trimmed_network.lock(), @@ -102,11 +107,11 @@ pub fn main() { .zip(test_labels.clone()) .collect::>(); - let trainer = NeuraBatchedTrainer::with_epochs(0.03, 20, 128, TRAIN_SIZE); + let trainer = NeuraBatchedTrainer::with_epochs(0.03, 10, 128, TRAIN_SIZE); plot_losses( trainer.train( - &NeuraBackprop::new(Euclidean), + &NeuraBackprop::new(CrossEntropy), &mut network, cycle_shuffling(train_images.clone().zip(train_labels), rand::thread_rng()), &test_data, @@ -135,10 +140,21 @@ pub fn main() { ); } -fn one_hot(value: usize, categories: usize) -> DVector { - let mut res = DVector::from_element(categories, 0.0); - if value < categories { - res[value] = 1.0; +fn shift(image: &DVector, dx: i32, dy: i32) -> DVector { + let mut res = DVector::from_element(image.len(), 0.0); + let width = WIDTH as i32; + let height = HEIGHT as i32; + + for y in 0..height { + for x in 0..width { + let x2 = x + dx; + let y2 = y + dy; + if y2 < 0 || y2 >= height || x2 < 0 || x2 >= width { + continue; + } + res[(y2 * width + x2) as usize] = image[(y * width + x) as usize]; + } } + res } diff --git a/examples/mnist-diffusion.rs b/examples/mnist-diffusion.rs index 3b7b062..f9e18ce 100644 --- a/examples/mnist-diffusion.rs +++ b/examples/mnist-diffusion.rs @@ -10,7 +10,7 @@ use neuramethyst::{ loss::Euclidean, regularize::NeuraL2, }, - plot_losses, + one_hot, plot_losses, prelude::*, }; @@ -145,14 +145,6 @@ fn uniform_vector(length: usize) -> DVector { res } -fn one_hot(value: usize, categories: usize) -> DVector { - let mut res = DVector::from_element(categories, 0.0); - if value < categories { - res[value] = 1.0; - } - res -} - fn add_noise(mut image: DVector, rng: &mut impl Rng, amount: f32) -> DVector { if amount <= 0.0 { return image; diff --git a/src/derivable/mod.rs b/src/derivable/mod.rs index 2100224..48b0055 100644 --- a/src/derivable/mod.rs +++ b/src/derivable/mod.rs @@ -2,7 +2,7 @@ use crate::algebra::NeuraVector; pub mod activation; pub mod loss; -pub mod reduce; +// pub mod reduce; pub mod regularize; pub trait NeuraDerivable { diff --git a/src/err.rs b/src/err.rs new file mode 100644 index 0000000..7b742ef --- /dev/null +++ b/src/err.rs @@ -0,0 +1,93 @@ +//! Various error types +//! + +use std::fmt::{Debug, Formatter}; + +use crate::prelude::*; + +pub trait NeuraRecursiveErrDebug { + fn fmt_rec(&self, f: &mut Formatter<'_>, depth: usize) -> std::fmt::Result; +} + +impl NeuraRecursiveErrDebug for NeuraRecursiveErr { + fn fmt_rec(&self, f: &mut Formatter<'_>, depth: usize) -> std::fmt::Result { + match self { + Self::Current(err) => { + write!(f, "NeuraRecursiveErr(depth={}, ", depth)?; + err.fmt(f)?; + write!(f, ")") + } + Self::Child(_) => write!(f, "NeuraRecursiveErr(depth={}, ())", depth), + } + } +} + +impl NeuraRecursiveErrDebug + for NeuraRecursiveErr +{ + #[inline] + fn fmt_rec(&self, f: &mut Formatter<'_>, depth: usize) -> std::fmt::Result { + match self { + Self::Current(err) => { + write!(f, "NeuraRecursiveErr(depth={}, ", depth)?; + err.fmt(f)?; + write!(f, ")") + } + Self::Child(err) => err.fmt_rec(f, depth + 1), + } + } +} + +impl Debug for NeuraRecursiveErr +where + Self: NeuraRecursiveErrDebug, +{ + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.fmt_rec(f, 0) + } +} + +/// Error type returned by `NeuraIsolateLayer::construct` +#[derive(Clone, Debug)] +pub enum NeuraIsolateLayerErr { + Incompatible { + start: NeuraShape, + end: NeuraShape, + input_shape: NeuraShape, + }, + OutOfBound { + start: NeuraShape, + end: NeuraShape, + input_shape: NeuraShape, + }, + OutOfOrder { + start: NeuraShape, + end: NeuraShape, + }, +} + +#[derive(Clone, Copy, Debug)] +pub enum NeuraAxisErr { + NoInput, + ConflictingShape(NeuraShape, NeuraShape), +} + +#[derive(Clone, Debug)] +pub enum NeuraResidualConstructErr { + Layer(LayerErr), + WrongConnection(isize), + AxisErr(NeuraAxisErr), + NoOutput, +} + +#[derive(Clone)] +pub enum NeuraRecursiveErr { + Current(Err), + Child(ChildErr), +} + +pub struct NeuraDimensionsMismatch { + pub existing: usize, + pub new: NeuraShape, +} diff --git a/src/layer/dense.rs b/src/layer/dense.rs index f000802..8710ef9 100644 --- a/src/layer/dense.rs +++ b/src/layer/dense.rs @@ -4,7 +4,7 @@ use nalgebra::{DMatrix, DVector}; use num::Float; use rand::Rng; -use crate::derivable::NeuraDerivable; +use crate::{derivable::NeuraDerivable, err::NeuraDimensionsMismatch}; use super::*; @@ -158,6 +158,24 @@ where } } +impl, Reg: NeuraDerivable> NeuraPartialLayer + for NeuraDenseLayer +{ + type Constructed = Self; + type Err = NeuraDimensionsMismatch; + + fn construct(self, input_shape: NeuraShape) -> Result { + if input_shape.size() != self.weights.shape().1 { + return Err(NeuraDimensionsMismatch { + existing: self.weights.shape().1, + new: input_shape, + }); + } + + Ok(self) + } +} + impl< F: Float + std::fmt::Debug + 'static + std::ops::AddAssign + std::ops::MulAssign, Act: NeuraDerivable, diff --git a/src/layer/isolate.rs b/src/layer/isolate.rs index 049bec5..f45c4cf 100644 --- a/src/layer/isolate.rs +++ b/src/layer/isolate.rs @@ -1,5 +1,7 @@ use nalgebra::{DVector, Scalar}; +use crate::err::NeuraIsolateLayerErr; + use super::*; /// **Class invariant:** start and end are @@ -9,24 +11,6 @@ pub struct NeuraIsolateLayer { end: NeuraShape, } -#[derive(Clone, Debug)] -pub enum NeuraIsolateLayerErr { - Incompatible { - start: NeuraShape, - end: NeuraShape, - input_shape: NeuraShape, - }, - OutOfBound { - start: NeuraShape, - end: NeuraShape, - input_shape: NeuraShape, - }, - OutOfOrder { - start: NeuraShape, - end: NeuraShape, - }, -} - impl NeuraIsolateLayer { pub fn new>(start: T, end: T) -> Option { let start = start.into(); diff --git a/src/lib.rs b/src/lib.rs index 201c9c9..0f1e6d3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,7 @@ pub mod layer; pub mod network; pub mod train; +pub mod err; mod utils; // TODO: move to a different file diff --git a/src/network/residual/axis.rs b/src/network/residual/axis.rs index b9aa234..9baa3cf 100644 --- a/src/network/residual/axis.rs +++ b/src/network/residual/axis.rs @@ -2,17 +2,11 @@ use std::borrow::Borrow; use nalgebra::{Const, DVector, Dyn, Scalar, VecStorage}; -use crate::prelude::NeuraShape; +use crate::{err::NeuraAxisErr, prelude::NeuraShape}; #[derive(Clone, Copy, Debug)] pub struct NeuraAxisAppend; -#[derive(Clone, Copy, Debug)] -pub enum NeuraAxisErr { - NoInput, - ConflictingShape(NeuraShape, NeuraShape), -} - pub trait NeuraCombineInputs { type Combined; diff --git a/src/network/residual/construct.rs b/src/network/residual/construct.rs index cebe5e8..b0aedfb 100644 --- a/src/network/residual/construct.rs +++ b/src/network/residual/construct.rs @@ -1,4 +1,7 @@ +use crate::err::*; + use super::*; +use NeuraResidualConstructErr::*; pub trait NeuraResidualConstruct { type Constructed; @@ -12,64 +15,13 @@ pub trait NeuraResidualConstruct { ) -> Result; } -#[derive(Clone, Debug)] -pub enum NeuraResidualConstructErr { - LayerErr(LayerErr), - ChildErr(ChildErr), - WrongConnection(isize), - AxisErr(NeuraAxisErr), - NoOutput, -} - -use NeuraResidualConstructErr::*; - -// impl NeuraResidualConstruct for NeuraResidualNode -// where -// Axis: NeuraCombineInputs>, -// { -// type Constructed = NeuraResidualNode; -// type Err = NeuraResidualConstructErr::Err>; - -// fn construct_residual( -// self, -// inputs: NeuraResidualInput, -// indices: NeuraResidualInput, -// current_index: usize, -// ) -> Result { -// let (layer_input_shape, _rest) = inputs.shift(); -// let layer_input_shape = self -// .axis -// .combine(layer_input_shape) -// .map_err(|e| AxisErr(e))?; - -// let layer = self -// .layer -// .construct(layer_input_shape) -// .map_err(|e| LayerErr(e))?; -// let layer_shape = layer.output_shape(); - -// if let Some(oob_offset) = self.offsets.iter().copied().find(|o| *o > 0) { -// return Err(WrongConnection(oob_offset)); -// } -// // TODO: check rest for non-zero columns - -// Ok(NeuraResidualNode { -// layer, -// child_network: (), -// offsets: self.offsets, -// axis: self.axis, -// output_shape: Some(layer_shape), -// }) -// } -// } - impl NeuraResidualConstruct for NeuraResidualNode where Axis: NeuraCombineInputs>, { type Constructed = NeuraResidualNode; - type Err = NeuraResidualConstructErr; + type Err = NeuraRecursiveErr, ChildNetwork::Err>; fn construct_residual( self, @@ -82,16 +34,19 @@ where let self_input_shapes = input_shapes.iter().map(|x| **x).collect::>(); - let layer_input_shape = self.axis.combine(input_shapes).map_err(|e| AxisErr(e))?; + let layer_input_shape = self + .axis + .combine(input_shapes) + .map_err(|e| NeuraRecursiveErr::Current(AxisErr(e)))?; let layer = self .layer .construct(layer_input_shape) - .map_err(|e| LayerErr(e))?; + .map_err(|e| NeuraRecursiveErr::Current(Layer(e)))?; let layer_shape = Rc::new(layer.output_shape()); if self.offsets.len() == 0 { - return Err(NoOutput); + return Err(NeuraRecursiveErr::Current(NoOutput)); } for &offset in &self.offsets { @@ -109,7 +64,7 @@ where let child_network = self .child_network .construct_residual(rest_inputs, rest_indices, current_index + 1) - .map_err(|e| ChildErr(e))?; + .map_err(|e| NeuraRecursiveErr::Child(e))?; Ok(NeuraResidualNode { layer, diff --git a/src/network/residual/last.rs b/src/network/residual/last.rs index 4d82220..d6c0e1f 100644 --- a/src/network/residual/last.rs +++ b/src/network/residual/last.rs @@ -1,7 +1,10 @@ +use crate::err::*; use crate::layer::*; use crate::network::*; use crate::utils::unwrap_or_clone; +use NeuraResidualConstructErr::*; + use std::borrow::Cow; use super::construct::*; @@ -28,7 +31,7 @@ impl Default for NeuraResidualLast { impl NeuraResidualConstruct for NeuraResidualLast { type Constructed = NeuraResidualLast; - type Err = NeuraResidualConstructErr<(), ()>; + type Err = NeuraRecursiveErr, ()>; fn construct_residual( self, @@ -39,15 +42,15 @@ impl NeuraResidualConstruct for NeuraResidualLast { let (this_input, _rest) = input.shift(); let index = indices .get_first() - .ok_or(Self::Err::AxisErr(NeuraAxisErr::NoInput))?; + .ok_or(Self::Err::Current(AxisErr(NeuraAxisErr::NoInput)))?; if *index != current_index - 1 { - return Err(Self::Err::WrongConnection( + return Err(Self::Err::Current(WrongConnection( current_index as isize - *index as isize - 1, - )); + ))); } if this_input.len() != 1 { - return Err(Self::Err::AxisErr(NeuraAxisErr::NoInput)); + return Err(Self::Err::Current(AxisErr(NeuraAxisErr::NoInput))); } // TODO: check that rest contains nothing else diff --git a/src/network/residual/mod.rs b/src/network/residual/mod.rs index 9b7835c..f88ed24 100644 --- a/src/network/residual/mod.rs +++ b/src/network/residual/mod.rs @@ -12,7 +12,7 @@ mod axis; pub use axis::*; mod construct; -pub use construct::NeuraResidualConstructErr; +pub use construct::NeuraResidualConstruct; mod node; pub use node::*; diff --git a/src/network/sequential/construct.rs b/src/network/sequential/construct.rs index 74977ce..a8b47eb 100644 --- a/src/network/sequential/construct.rs +++ b/src/network/sequential/construct.rs @@ -1,13 +1,7 @@ -use crate::layer::NeuraShapedLayer; +use crate::{err::NeuraRecursiveErr, layer::NeuraShapedLayer}; use super::*; -#[derive(Debug, Clone)] -pub enum NeuraSequentialConstructErr { - Current(Err), - Child(ChildErr), -} - impl NeuraPartialLayer for NeuraSequential { type Constructed = NeuraSequential; type Err = Layer::Err; @@ -24,19 +18,19 @@ impl NeuraPartialLaye for NeuraSequential { type Constructed = NeuraSequential; - type Err = NeuraSequentialConstructErr; + type Err = NeuraRecursiveErr; fn construct(self, input_shape: NeuraShape) -> Result { let layer = self .layer .construct(input_shape) - .map_err(|e| NeuraSequentialConstructErr::Current(e))?; + .map_err(|e| NeuraRecursiveErr::Current(e))?; // TODO: ensure that this operation (and all recursive operations) are directly allocated on the heap let child_network = self .child_network .construct(layer.output_shape()) - .map_err(|e| NeuraSequentialConstructErr::Child(e))?; + .map_err(|e| NeuraRecursiveErr::Child(e))?; let child_network = Box::new(child_network); Ok(NeuraSequential { diff --git a/src/utils.rs b/src/utils.rs index 259c65e..d1296e9 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,4 +1,4 @@ -use crate::algebra::NeuraVector; +use nalgebra::DVector; #[allow(dead_code)] pub(crate) fn assign_add_vector(sum: &mut [f64; N], operand: &[f64; N]) { @@ -90,17 +90,15 @@ where #[cfg(test)] pub(crate) fn uniform_vector(length: usize) -> nalgebra::DVector { - use nalgebra::DVector; use rand::Rng; let mut rng = rand::thread_rng(); DVector::from_fn(length, |_, _| -> f64 { rng.gen() }) } -#[deprecated] -pub fn one_hot(value: usize) -> NeuraVector { - let mut res = NeuraVector::default(); - if value < N { +pub fn one_hot(value: usize, categories: usize) -> DVector { + let mut res = DVector::from_element(categories, 0.0); + if value < categories { res[value] = 1.0; } res