From b34b1e630b79aa07a4a9227a81be1c32e09a3773 Mon Sep 17 00:00:00 2001 From: Adrien Burgun Date: Fri, 28 Apr 2023 23:20:34 +0200 Subject: [PATCH] :sparkles: Implement NeuraNetwork for NeuraResidual and NeuraResidualNode --- Cargo.toml | 3 + examples/densenet-fwdfwd.rs | 138 ++++++++++++++++ src/algebra/vector.rs | 26 +-- src/lib.rs | 7 +- src/network/mod.rs | 2 +- src/network/residual/layer_impl.rs | 246 +++++++++++++++++------------ src/network/residual/mod.rs | 12 +- src/network/residual/wrapper.rs | 187 ++++++++++++++++++++++ 8 files changed, 490 insertions(+), 131 deletions(-) create mode 100644 examples/densenet-fwdfwd.rs create mode 100644 src/network/residual/wrapper.rs diff --git a/Cargo.toml b/Cargo.toml index 8ff9962..8b861e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,3 +21,6 @@ viuer = "0.6.2" rust-mnist = "0.2.0" serde_json = "1.0.96" approx = "0.5.1" + +[profile.release] +debug = true diff --git a/examples/densenet-fwdfwd.rs b/examples/densenet-fwdfwd.rs new file mode 100644 index 0000000..b7e1c1e --- /dev/null +++ b/examples/densenet-fwdfwd.rs @@ -0,0 +1,138 @@ +use nalgebra::{dvector, DVector}; +use neuramethyst::derivable::activation::Tanh; +use neuramethyst::derivable::regularize::NeuraL1; +use neuramethyst::gradient_solver::NeuraForwardForward; +use neuramethyst::{plot_losses, prelude::*}; + +use rand::Rng; + +fn main() { + let mut network = neura_residual![ + <= 0, 2; + neura_layer!("dense", 6).regularization(NeuraL1(0.001)); + neura_layer!("normalize"); + neura_layer!("dense", 6).regularization(NeuraL1(0.001)); + ] + .construct(NeuraShape::Vector(3)) + .unwrap(); + + let inputs = (0..1).cycle().map(move |_| { + let mut rng = rand::thread_rng(); + let category = rng.gen_bool(0.5); + let good = rng.gen_bool(0.5); + let (x, y) = if category { + let radius: f32 = rng.gen_range(0.0..2.0); + let angle = rng.gen_range(0.0..std::f32::consts::TAU); + (angle.cos() * radius, angle.sin() * radius) + } else { + let radius: f32 = rng.gen_range(3.0..5.0); + let angle = rng.gen_range(0.0..std::f32::consts::TAU); + (angle.cos() * radius, angle.sin() * radius) + }; + + if good { + (dvector![x, y, category as u8 as f32], true) + } else { + (dvector![x, y, 1.0 - category as u8 as f32], false) + } + }); + + let test_inputs: Vec<_> = inputs.clone().filter(|(_, good)| *good).take(10).collect(); + let threshold = 0.5f32; + + if std::env::args().any(|arg| arg == "draw") { + for epoch in 0..200 { + let mut trainer = NeuraBatchedTrainer::new(0.03, 10); + trainer.batch_size = 50; + + trainer.train( + &NeuraForwardForward::new(Tanh, threshold as f64), + &mut network, + inputs.clone(), + &test_inputs, + ); + + // let network = network.clone().trim_tail().trim_tail(); + draw_neuron_activation( + |input| { + let cat0 = network.eval(&dvector![input[0] as f32, input[1] as f32, 0.0]); + let cat1 = network.eval(&dvector![input[0] as f32, input[1] as f32, 1.0]); + + let cat0_good = cat0.map(|x| x * x).sum(); + let cat1_good = cat1.map(|x| x * x).sum(); + let estimation = cat1_good / (cat0_good + cat1_good); + + let cat0_norm = cat0 / cat0_good.sqrt(); + let mut cat0_rgb = DVector::from_element(3, 0.0); + + for i in 0..cat0_norm.len() { + cat0_rgb[i % 3] += cat0_norm[i].abs(); + } + + (cat0_rgb * estimation) + .into_iter() + .map(|x| *x as f64) + .collect() + }, + 6.0, + ); + println!("{}", epoch); + + std::thread::sleep(std::time::Duration::new(0, 50_000_000)); + } + } else { + let mut trainer = NeuraBatchedTrainer::new(0.03, 20 * 50); + trainer.batch_size = 50; + trainer.log_iterations = 20; + + plot_losses( + trainer.train( + &NeuraForwardForward::new(Tanh, threshold as f64), + &mut network, + inputs.clone(), + &test_inputs, + ), + 128, + 48, + ); + + // println!("{}", String::from("\n").repeat(64)); + // draw_neuron_activation(|input| network.eval(&input).into_iter().collect(), 6.0); + } +} + +// TODO: move this to the library? +fn draw_neuron_activation Vec>(callback: F, scale: f64) { + use viuer::Config; + + const WIDTH: u32 = 64; + const HEIGHT: u32 = 64; + + let mut image = image::RgbImage::new(WIDTH, HEIGHT); + + fn sigmoid(x: f64) -> f64 { + 0.1 + 0.9 * x.abs().powf(0.8) + } + + for y in 0..HEIGHT { + let y2 = 2.0 * y as f64 / HEIGHT as f64 - 1.0; + for x in 0..WIDTH { + let x2 = 2.0 * x as f64 / WIDTH as f64 - 1.0; + let activation = callback([x2 * scale, y2 * scale]); + let r = (sigmoid(activation.get(0).copied().unwrap_or(-1.0)) * 255.0).floor() as u8; + let g = (sigmoid(activation.get(1).copied().unwrap_or(-1.0)) * 255.0).floor() as u8; + let b = (sigmoid(activation.get(2).copied().unwrap_or(-1.0)) * 255.0).floor() as u8; + + *image.get_pixel_mut(x, y) = image::Rgb([r, g, b]); + } + } + + let config = Config { + use_kitty: false, + truecolor: true, + // absolute_offset: false, + ..Default::default() + }; + + viuer::print(&image::DynamicImage::ImageRgb8(image), &config).unwrap(); +} diff --git a/src/algebra/vector.rs b/src/algebra/vector.rs index 52f4a66..89dee93 100644 --- a/src/algebra/vector.rs +++ b/src/algebra/vector.rs @@ -259,20 +259,20 @@ impl<'a, const LENGTH: usize, F: Default + Clone> FromIterator for NeuraVecto mod test { use super::*; - #[test] - fn test_reverse_dot() { - let left: NeuraVector<_, f64> = [2.0, 3.0, 5.0].into(); - let right: NeuraVector<_, f64> = [7.0, 11.0, 13.0, 17.0].into(); + // #[test] + // fn test_reverse_dot() { + // let left: NeuraVector<_, f64> = [2.0, 3.0, 5.0].into(); + // let right: NeuraVector<_, f64> = [7.0, 11.0, 13.0, 17.0].into(); - let expected: NeuraMatrix<_, _, f64> = [ - [14.0, 22.0, 26.0, 34.0], - [21.0, 33.0, 39.0, 51.0], - [35.0, 55.0, 65.0, 85.0], - ] - .into(); + // let expected: NeuraMatrix<_, _, f64> = [ + // [14.0, 22.0, 26.0, 34.0], + // [21.0, 33.0, 39.0, 51.0], + // [35.0, 55.0, 65.0, 85.0], + // ] + // .into(); - let actual = left.reverse_dot(right); + // let actual = left.reverse_dot(right); - assert_eq!(expected, actual); - } + // assert_eq!(expected, actual); + // } } diff --git a/src/lib.rs b/src/lib.rs index f455cac..7edd616 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,3 @@ -#![feature(generic_arg_infer)] -// #![feature(generic_const_exprs)] -#![feature(associated_type_defaults)] -#![feature(arc_unwrap_or_clone)] - pub mod algebra; pub mod derivable; pub mod gradient_solver; @@ -20,7 +15,7 @@ pub use utils::{argmax, cycle_shuffling, one_hot, plot_losses}; /// so there should not be any conflicts when doing a wildcard import of `prelude`. pub mod prelude { // Macros - pub use crate::{neura_layer, neura_sequential}; + pub use crate::{neura_layer, neura_residual, neura_sequential}; // Structs and traits pub use crate::gradient_solver::NeuraBackprop; diff --git a/src/network/mod.rs b/src/network/mod.rs index c4b7962..fcc8836 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -1,4 +1,4 @@ -// pub mod residual; +pub mod residual; pub mod sequential; mod traits; diff --git a/src/network/residual/layer_impl.rs b/src/network/residual/layer_impl.rs index b5c67cd..c78b87b 100644 --- a/src/network/residual/layer_impl.rs +++ b/src/network/residual/layer_impl.rs @@ -1,14 +1,18 @@ //! Implementations for NeuraLayer* +use std::borrow::Cow; -use crate::{gradient_solver::NeuraGradientSolverTransient, network::{NeuraTrainableNetwork, NeuraTrainableNetworkBase}}; +use crate::network::*; use super::*; impl NeuraResidualNode { - fn process_input(&self, input: &NeuraResidualInput) -> (Axis::Combined, NeuraResidualInput) + fn process_input( + &self, + input: &NeuraResidualInput, + ) -> (Axis::Combined, NeuraResidualInput) where Axis: NeuraCombineInputs, - Layer: NeuraLayer + Layer: NeuraLayer, { let (inputs, rest) = input.shift(); @@ -17,7 +21,11 @@ impl NeuraResidualNode { (layer_input, rest) } - fn combine_outputs(&self, layer_output: Data, output: &mut NeuraResidualInput) -> Rc { + fn combine_outputs( + &self, + layer_output: Data, + output: &mut NeuraResidualInput, + ) -> Rc { let layer_output = Rc::new(layer_output); for &offset in &self.offsets { @@ -26,6 +34,13 @@ impl NeuraResidualNode { layer_output } + + pub(crate) fn map_input_owned(&self, input: &NeuraResidualInput) -> Axis::Combined + where + Axis: NeuraCombineInputs, + { + self.axis.combine(input.shift().0) + } } impl NeuraLayer>> @@ -46,52 +61,23 @@ where } } -impl NeuraLayer> for NeuraResidual -where - Layers: NeuraLayer>, Output = NeuraResidualInput>, -{ - type Output = Output; - - fn eval(&self, input: &DVector) -> Self::Output { - let input: Rc> = Rc::new((*input).clone()); - let mut inputs = NeuraResidualInput::new(); - - for &offset in &self.initial_offsets { - inputs.push(offset, Rc::clone(&input)); - } - - drop(input); - - let output = self.layers.eval(&inputs); - - let result = output.get_first() - .expect("Invalid NeuraResidual state: network returned no data, did you forget to link the last layer?") - .into(); - - Rc::unwrap_or_clone(result) - } -} - +#[allow(dead_code)] pub struct NeuraResidualIntermediary { layer_intermediary: LayerIntermediary, layer_output: Rc, child_intermediary: Box, } -impl< - Data, - Axis: NeuraCombineInputs, - Layer: NeuraTrainableLayerBase, - ChildNetwork: NeuraTrainableLayerBase> -> NeuraTrainableLayerBase> for NeuraResidualNode -where - NeuraResidualNode: NeuraLayer, Output=ChildNetwork::Output> +impl + NeuraTrainableLayerBase for NeuraResidualNode { type Gradient = (Layer::Gradient, Box); - type IntermediaryRepr = NeuraResidualIntermediary; fn default_gradient(&self) -> Self::Gradient { - (self.layer.default_gradient(), Box::new(self.child_network.default_gradient())) + ( + self.layer.default_gradient(), + Box::new(self.child_network.default_gradient()), + ) } fn apply_gradient(&mut self, gradient: &Self::Gradient) { @@ -99,7 +85,33 @@ where self.child_network.apply_gradient(&gradient.1); } - fn eval_training(&self, input: &NeuraResidualInput) -> (Self::Output, Self::IntermediaryRepr) { + fn prepare_layer(&mut self, is_training: bool) { + self.layer.prepare_layer(is_training); + self.child_network.prepare_layer(is_training); + } +} + +impl< + Data, + Axis: NeuraCombineInputs, + Layer: NeuraTrainableLayerEval, + ChildNetwork: NeuraTrainableLayerEval>, + > NeuraTrainableLayerEval> + for NeuraResidualNode +where + NeuraResidualNode: + NeuraLayer, Output = ChildNetwork::Output>, +{ + type IntermediaryRepr = NeuraResidualIntermediary< + Layer::IntermediaryRepr, + Layer::Output, + ChildNetwork::IntermediaryRepr, + >; + + fn eval_training( + &self, + input: &NeuraResidualInput, + ) -> (Self::Output, Self::IntermediaryRepr) { let (layer_input, mut rest) = self.process_input(input); let (layer_output, layer_intermediary) = self.layer.eval_training(&layer_input); @@ -110,89 +122,119 @@ where let intermediary = NeuraResidualIntermediary { layer_intermediary, layer_output, - child_intermediary: Box::new(child_intermediary) + child_intermediary: Box::new(child_intermediary), }; (output, intermediary) } - - fn prepare_layer(&mut self, is_training: bool) { - self.layer.prepare_layer(is_training); - self.child_network.prepare_layer(is_training); - } } impl< - Data, - Axis: NeuraCombineInputs, - Layer: NeuraTrainableLayerSelf, - ChildNetwork: NeuraTrainableNetworkBase>, -> NeuraTrainableNetworkBase> for NeuraResidualNode + Data, + Axis: NeuraCombineInputs, + Layer: NeuraTrainableLayerSelf, + ChildNetwork: NeuraTrainableLayerSelf>, + > NeuraTrainableLayerSelf> + for NeuraResidualNode where - Self: NeuraTrainableLayerBase, Gradient = (Layer::Gradient, Box)>, + NeuraResidualNode: + NeuraLayer, Output = ChildNetwork::Output>, { - type Gradient = >>::Gradient; - type LayerOutput = Layer::Output; + fn regularize_layer(&self) -> Self::Gradient { + ( + self.layer.regularize_layer(), + Box::new(self.child_network.regularize_layer()), + ) + } - fn default_gradient(&self) -> Self::Gradient { - >>::default_gradient(self) + fn get_gradient( + &self, + input: &NeuraResidualInput, + intermediary: &Self::IntermediaryRepr, + epsilon: &Self::Output, + ) -> Self::Gradient { + unimplemented!(); } +} - fn apply_gradient(&mut self, gradient: &Self::Gradient) { - >>::apply_gradient(self, gradient) +impl NeuraNetworkBase for NeuraResidualNode { + type Layer = Layer; + + fn get_layer(&self) -> &Self::Layer { + &self.layer } +} + +impl NeuraNetworkRec + for NeuraResidualNode +{ + type NextNode = ChildNetwork; - fn regularize(&self) -> Self::Gradient { - (self.layer.regularize_layer(), Box::new(self.child_network.regularize())) + fn get_next(&self) -> &Self::NextNode { + &self.child_network } - fn prepare(&mut self, train_iteration: bool) { - self.layer.prepare_layer(train_iteration); - self.child_network.prepare(train_iteration); + fn merge_gradient( + &self, + rec_gradient: ::Gradient, + layer_gradient: ::Gradient, + ) -> Self::Gradient { + (layer_gradient, Box::new(rec_gradient)) } } impl< - Data, - Axis: NeuraSplitInputs, - Layer: NeuraTrainableLayerSelf, - Optimizer: NeuraGradientSolverTransient, - ChildNetwork: NeuraTrainableNetwork, Optimizer>, -> NeuraTrainableNetwork, Optimizer> for NeuraResidualNode + Data: Clone, + Axis: NeuraCombineInputs, + Layer: NeuraLayer, + ChildNetwork, + > NeuraNetwork> for NeuraResidualNode where - Self: NeuraTrainableLayerBase, Gradient = (Layer::Gradient, Box)>, + Layer::Output: Clone, + Axis::Combined: Clone, { - fn traverse( - &self, - input: &NeuraResidualInput, - optimizer: &Optimizer, - ) -> Optimizer::Output, Self::Gradient> { - let (layer_input, mut rest) = self.process_input(input); - let (layer_output, layer_intermediary) = self.layer.eval_training(&layer_input); - let layer_output = self.combine_outputs(layer_output, &mut rest); + type LayerInput = Axis::Combined; + + type NodeOutput = NeuraResidualInput; + + fn map_input<'a>(&'_ self, input: &'a NeuraResidualInput) -> Cow<'a, Self::LayerInput> { + Cow::Owned(self.map_input_owned(input)) + } + + fn map_output<'a>( + &'_ self, + input: &'_ NeuraResidualInput, + layer_output: &'a >::Output, + ) -> Cow<'a, Self::NodeOutput> { + let mut remaining_inputs = input.shift().1; + self.combine_outputs(layer_output.clone(), &mut remaining_inputs); + + Cow::Owned(remaining_inputs) + } + + #[allow(unused_variables)] + fn map_gradient_in<'a>( + &'_ self, + input: &'_ NeuraResidualInput, + gradient_in: &'a Self::NodeOutput, + ) -> Cow<'a, >::Output> { + // To convert from gradient_in to layer's gradient_in: + // Pop the first value from `epsilon`, then: + // - compute its sum + // - use it to compute the outcoming epsilon of the current layer + // - split the oucoming epsilon into its original components, and push those back onto the rest + // At this point, the value for `epsilon` in the gradient solver's state should be ready for another iteration, + // with the first value containing the unsummed incoming epsilon values from the downstream layers + unimplemented!() + } - let child_result = self.child_network.traverse(&rest, optimizer); - // TODO: maybe move this to a custom impl of NeuraGradientSolverTransient for NeuraResidualInput? - // Or have a different set of traits for NeuraTrainableNetwork specific to NeuraResidualNodes - let child_result = optimizer.map_epsilon(child_result, |_epsilon| { - // Pop the first value from `epsilon`, then: - // - compute its sum - // - use it to compute the outcoming epsilon of the current layer - // - split the oucoming epsilon into its original components, and push those back onto the rest - // At this point, the value for `epsilon` in the gradient solver's state should be ready for another iteration, - // with the first value containing the unsummed incoming epsilon values from the downstream layers - todo!() - }); - - optimizer.eval_layer( - &self.layer, - &layer_input, - &layer_output, - &layer_intermediary, - child_result, - |this_gradient, child_gradient| (this_gradient, Box::new(child_gradient)) - ); - - todo!(); + #[allow(unused_variables)] + fn map_gradient_out<'a>( + &'_ self, + input: &'_ NeuraResidualInput, + gradient_in: &'_ Self::NodeOutput, + gradient_out: &'a Self::LayerInput, + ) -> Cow<'a, NeuraResidualInput> { + unimplemented!() } } diff --git a/src/network/residual/mod.rs b/src/network/residual/mod.rs index a013ed2..5e12a81 100644 --- a/src/network/residual/mod.rs +++ b/src/network/residual/mod.rs @@ -7,6 +7,9 @@ use crate::layer::*; mod layer_impl; +mod wrapper; +pub use wrapper::*; + mod input; pub use input::*; @@ -16,15 +19,6 @@ pub use axis::*; mod construct; pub use construct::NeuraResidualConstructErr; -#[derive(Clone, Debug, PartialEq)] -pub struct NeuraResidual { - /// Instance of NeuraResidualNode - layers: Layers, - - /// Array of which layers to send the input to, defaults to `vec![0]` - initial_offsets: Vec, -} - impl NeuraResidual { pub fn new(layers: Layers) -> Self { Self { diff --git a/src/network/residual/wrapper.rs b/src/network/residual/wrapper.rs new file mode 100644 index 0000000..a851da3 --- /dev/null +++ b/src/network/residual/wrapper.rs @@ -0,0 +1,187 @@ +use std::borrow::Cow; + +use crate::network::*; + +use super::*; + +#[derive(Clone, Debug, PartialEq)] +pub struct NeuraResidual { + /// Instance of NeuraResidualNode + pub(crate) layers: Layers, + + /// Array of which layers to send the input to, defaults to `vec![0]` + pub(crate) initial_offsets: Vec, +} + +impl NeuraResidual { + fn input_to_residual_input(&self, input: &Input) -> NeuraResidualInput { + let input: Rc = Rc::new((*input).clone()); + let mut inputs = NeuraResidualInput::new(); + + for &offset in &self.initial_offsets { + inputs.push(offset, Rc::clone(&input)); + } + + drop(input); + + inputs + } +} + +impl NeuraLayer for NeuraResidual +where + Layers: NeuraLayer, Output = NeuraResidualInput>, +{ + type Output = Output; + + fn eval(&self, input: &Input) -> Self::Output { + let output = self.layers.eval(&self.input_to_residual_input(input)); + + let result: Rc = output.get_first() + .expect("Invalid NeuraResidual state: network returned no data, did you forget to link the last layer?") + .into(); + + // TODO: replace with Rc::unwrap_or_clone once https://github.com/rust-lang/rust/issues/93610 is closed + Rc::try_unwrap(result).unwrap_or_else(|result| (*result).clone()) + } +} + +impl NeuraTrainableLayerBase for NeuraResidual { + type Gradient = Layers::Gradient; + + #[inline(always)] + fn default_gradient(&self) -> Self::Gradient { + self.layers.default_gradient() + } + + #[inline(always)] + fn apply_gradient(&mut self, gradient: &Self::Gradient) { + self.layers.apply_gradient(gradient); + } +} + +impl< + Data: Clone, + Layers: NeuraTrainableLayerEval, Output = NeuraResidualInput>, + > NeuraTrainableLayerEval for NeuraResidual +{ + type IntermediaryRepr = Layers::IntermediaryRepr; + + fn eval_training(&self, input: &Data) -> (Self::Output, Self::IntermediaryRepr) { + let (output, intermediary) = self + .layers + .eval_training(&self.input_to_residual_input(input)); + + let result: Rc = output.get_first().unwrap().into(); + + ( + Rc::try_unwrap(result).unwrap_or_else(|result| (*result).clone()), + intermediary, + ) + } +} + +impl< + Data: Clone, + Layers: NeuraTrainableLayerSelf, Output = NeuraResidualInput>, + > NeuraTrainableLayerSelf for NeuraResidual +{ + fn regularize_layer(&self) -> Self::Gradient { + self.layers.regularize_layer() + } + + fn get_gradient( + &self, + input: &Data, + intermediary: &Self::IntermediaryRepr, + epsilon: &Self::Output, + ) -> Self::Gradient { + let epsilon = Rc::new(epsilon.clone()); + let mut epsilon_residual = NeuraResidualInput::new(); + + epsilon_residual.push(0, epsilon); + + self.layers.get_gradient( + &self.input_to_residual_input(input), + intermediary, + &epsilon_residual, + ) + } +} + +impl NeuraNetworkBase for NeuraResidual { + type Layer = (); + + #[inline(always)] + fn get_layer(&self) -> &Self::Layer { + &() + } +} + +impl NeuraNetworkRec for NeuraResidual { + type NextNode = Layers; + + #[inline(always)] + fn get_next(&self) -> &Self::NextNode { + &self.layers + } + + #[inline(always)] + fn merge_gradient( + &self, + rec_gradient: ::Gradient, + _layer_gradient: ::Gradient, + ) -> Self::Gradient { + rec_gradient + } +} + +impl NeuraNetwork for NeuraResidual { + type LayerInput = Data; + type NodeOutput = NeuraResidualInput; + + #[inline(always)] + fn map_input<'a>(&'_ self, input: &'a Data) -> Cow<'a, Self::LayerInput> { + Cow::Borrowed(input) + } + + #[inline(always)] + fn map_output<'a>( + &'_ self, + _input: &'_ Data, + layer_output: &'a Data, + ) -> Cow<'a, Self::NodeOutput> { + let layer_output = Rc::new(layer_output.clone()); + let mut outputs = NeuraResidualInput::new(); + + for &offset in &self.initial_offsets { + outputs.push(offset, Rc::clone(&layer_output)); + } + + Cow::Owned(outputs) + } + + #[inline(always)] + fn map_gradient_in<'a>( + &'_ self, + _input: &'_ Data, + gradient_in: &'a Self::NodeOutput, + ) -> Cow<'a, Data> { + let first = gradient_in + .clone() + .get_first() + .expect("No outgoing gradient in NeuraResidual on the last node"); + + Cow::Owned((*first).clone()) + } + + #[inline(always)] + fn map_gradient_out<'a>( + &'_ self, + _input: &'_ Data, + _gradient_in: &'_ Self::NodeOutput, + gradient_out: &'a Self::LayerInput, + ) -> Cow<'a, Data> { + Cow::Borrowed(gradient_out) + } +}