parent
741cf1ced6
commit
b34b1e630b
@ -0,0 +1,138 @@
|
|||||||
|
use nalgebra::{dvector, DVector};
|
||||||
|
use neuramethyst::derivable::activation::Tanh;
|
||||||
|
use neuramethyst::derivable::regularize::NeuraL1;
|
||||||
|
use neuramethyst::gradient_solver::NeuraForwardForward;
|
||||||
|
use neuramethyst::{plot_losses, prelude::*};
|
||||||
|
|
||||||
|
use rand::Rng;
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
let mut network = neura_residual![
|
||||||
|
<= 0, 2;
|
||||||
|
neura_layer!("dense", 6).regularization(NeuraL1(0.001));
|
||||||
|
neura_layer!("normalize");
|
||||||
|
neura_layer!("dense", 6).regularization(NeuraL1(0.001));
|
||||||
|
]
|
||||||
|
.construct(NeuraShape::Vector(3))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let inputs = (0..1).cycle().map(move |_| {
|
||||||
|
let mut rng = rand::thread_rng();
|
||||||
|
let category = rng.gen_bool(0.5);
|
||||||
|
let good = rng.gen_bool(0.5);
|
||||||
|
let (x, y) = if category {
|
||||||
|
let radius: f32 = rng.gen_range(0.0..2.0);
|
||||||
|
let angle = rng.gen_range(0.0..std::f32::consts::TAU);
|
||||||
|
(angle.cos() * radius, angle.sin() * radius)
|
||||||
|
} else {
|
||||||
|
let radius: f32 = rng.gen_range(3.0..5.0);
|
||||||
|
let angle = rng.gen_range(0.0..std::f32::consts::TAU);
|
||||||
|
(angle.cos() * radius, angle.sin() * radius)
|
||||||
|
};
|
||||||
|
|
||||||
|
if good {
|
||||||
|
(dvector![x, y, category as u8 as f32], true)
|
||||||
|
} else {
|
||||||
|
(dvector![x, y, 1.0 - category as u8 as f32], false)
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let test_inputs: Vec<_> = inputs.clone().filter(|(_, good)| *good).take(10).collect();
|
||||||
|
let threshold = 0.5f32;
|
||||||
|
|
||||||
|
if std::env::args().any(|arg| arg == "draw") {
|
||||||
|
for epoch in 0..200 {
|
||||||
|
let mut trainer = NeuraBatchedTrainer::new(0.03, 10);
|
||||||
|
trainer.batch_size = 50;
|
||||||
|
|
||||||
|
trainer.train(
|
||||||
|
&NeuraForwardForward::new(Tanh, threshold as f64),
|
||||||
|
&mut network,
|
||||||
|
inputs.clone(),
|
||||||
|
&test_inputs,
|
||||||
|
);
|
||||||
|
|
||||||
|
// let network = network.clone().trim_tail().trim_tail();
|
||||||
|
draw_neuron_activation(
|
||||||
|
|input| {
|
||||||
|
let cat0 = network.eval(&dvector![input[0] as f32, input[1] as f32, 0.0]);
|
||||||
|
let cat1 = network.eval(&dvector![input[0] as f32, input[1] as f32, 1.0]);
|
||||||
|
|
||||||
|
let cat0_good = cat0.map(|x| x * x).sum();
|
||||||
|
let cat1_good = cat1.map(|x| x * x).sum();
|
||||||
|
let estimation = cat1_good / (cat0_good + cat1_good);
|
||||||
|
|
||||||
|
let cat0_norm = cat0 / cat0_good.sqrt();
|
||||||
|
let mut cat0_rgb = DVector::from_element(3, 0.0);
|
||||||
|
|
||||||
|
for i in 0..cat0_norm.len() {
|
||||||
|
cat0_rgb[i % 3] += cat0_norm[i].abs();
|
||||||
|
}
|
||||||
|
|
||||||
|
(cat0_rgb * estimation)
|
||||||
|
.into_iter()
|
||||||
|
.map(|x| *x as f64)
|
||||||
|
.collect()
|
||||||
|
},
|
||||||
|
6.0,
|
||||||
|
);
|
||||||
|
println!("{}", epoch);
|
||||||
|
|
||||||
|
std::thread::sleep(std::time::Duration::new(0, 50_000_000));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
let mut trainer = NeuraBatchedTrainer::new(0.03, 20 * 50);
|
||||||
|
trainer.batch_size = 50;
|
||||||
|
trainer.log_iterations = 20;
|
||||||
|
|
||||||
|
plot_losses(
|
||||||
|
trainer.train(
|
||||||
|
&NeuraForwardForward::new(Tanh, threshold as f64),
|
||||||
|
&mut network,
|
||||||
|
inputs.clone(),
|
||||||
|
&test_inputs,
|
||||||
|
),
|
||||||
|
128,
|
||||||
|
48,
|
||||||
|
);
|
||||||
|
|
||||||
|
// println!("{}", String::from("\n").repeat(64));
|
||||||
|
// draw_neuron_activation(|input| network.eval(&input).into_iter().collect(), 6.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: move this to the library?
|
||||||
|
fn draw_neuron_activation<F: Fn([f64; 2]) -> Vec<f64>>(callback: F, scale: f64) {
|
||||||
|
use viuer::Config;
|
||||||
|
|
||||||
|
const WIDTH: u32 = 64;
|
||||||
|
const HEIGHT: u32 = 64;
|
||||||
|
|
||||||
|
let mut image = image::RgbImage::new(WIDTH, HEIGHT);
|
||||||
|
|
||||||
|
fn sigmoid(x: f64) -> f64 {
|
||||||
|
0.1 + 0.9 * x.abs().powf(0.8)
|
||||||
|
}
|
||||||
|
|
||||||
|
for y in 0..HEIGHT {
|
||||||
|
let y2 = 2.0 * y as f64 / HEIGHT as f64 - 1.0;
|
||||||
|
for x in 0..WIDTH {
|
||||||
|
let x2 = 2.0 * x as f64 / WIDTH as f64 - 1.0;
|
||||||
|
let activation = callback([x2 * scale, y2 * scale]);
|
||||||
|
let r = (sigmoid(activation.get(0).copied().unwrap_or(-1.0)) * 255.0).floor() as u8;
|
||||||
|
let g = (sigmoid(activation.get(1).copied().unwrap_or(-1.0)) * 255.0).floor() as u8;
|
||||||
|
let b = (sigmoid(activation.get(2).copied().unwrap_or(-1.0)) * 255.0).floor() as u8;
|
||||||
|
|
||||||
|
*image.get_pixel_mut(x, y) = image::Rgb([r, g, b]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let config = Config {
|
||||||
|
use_kitty: false,
|
||||||
|
truecolor: true,
|
||||||
|
// absolute_offset: false,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
viuer::print(&image::DynamicImage::ImageRgb8(image), &config).unwrap();
|
||||||
|
}
|
||||||
@ -0,0 +1,187 @@
|
|||||||
|
use std::borrow::Cow;
|
||||||
|
|
||||||
|
use crate::network::*;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq)]
|
||||||
|
pub struct NeuraResidual<Layers> {
|
||||||
|
/// Instance of NeuraResidualNode
|
||||||
|
pub(crate) layers: Layers,
|
||||||
|
|
||||||
|
/// Array of which layers to send the input to, defaults to `vec![0]`
|
||||||
|
pub(crate) initial_offsets: Vec<usize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<Layers> NeuraResidual<Layers> {
|
||||||
|
fn input_to_residual_input<Input: Clone>(&self, input: &Input) -> NeuraResidualInput<Input> {
|
||||||
|
let input: Rc<Input> = Rc::new((*input).clone());
|
||||||
|
let mut inputs = NeuraResidualInput::new();
|
||||||
|
|
||||||
|
for &offset in &self.initial_offsets {
|
||||||
|
inputs.push(offset, Rc::clone(&input));
|
||||||
|
}
|
||||||
|
|
||||||
|
drop(input);
|
||||||
|
|
||||||
|
inputs
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<Input: Clone, Output: Clone, Layers> NeuraLayer<Input> for NeuraResidual<Layers>
|
||||||
|
where
|
||||||
|
Layers: NeuraLayer<NeuraResidualInput<Input>, Output = NeuraResidualInput<Output>>,
|
||||||
|
{
|
||||||
|
type Output = Output;
|
||||||
|
|
||||||
|
fn eval(&self, input: &Input) -> Self::Output {
|
||||||
|
let output = self.layers.eval(&self.input_to_residual_input(input));
|
||||||
|
|
||||||
|
let result: Rc<Self::Output> = output.get_first()
|
||||||
|
.expect("Invalid NeuraResidual state: network returned no data, did you forget to link the last layer?")
|
||||||
|
.into();
|
||||||
|
|
||||||
|
// TODO: replace with Rc::unwrap_or_clone once https://github.com/rust-lang/rust/issues/93610 is closed
|
||||||
|
Rc::try_unwrap(result).unwrap_or_else(|result| (*result).clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<Layers: NeuraTrainableLayerBase> NeuraTrainableLayerBase for NeuraResidual<Layers> {
|
||||||
|
type Gradient = Layers::Gradient;
|
||||||
|
|
||||||
|
#[inline(always)]
|
||||||
|
fn default_gradient(&self) -> Self::Gradient {
|
||||||
|
self.layers.default_gradient()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline(always)]
|
||||||
|
fn apply_gradient(&mut self, gradient: &Self::Gradient) {
|
||||||
|
self.layers.apply_gradient(gradient);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<
|
||||||
|
Data: Clone,
|
||||||
|
Layers: NeuraTrainableLayerEval<NeuraResidualInput<Data>, Output = NeuraResidualInput<Data>>,
|
||||||
|
> NeuraTrainableLayerEval<Data> for NeuraResidual<Layers>
|
||||||
|
{
|
||||||
|
type IntermediaryRepr = Layers::IntermediaryRepr;
|
||||||
|
|
||||||
|
fn eval_training(&self, input: &Data) -> (Self::Output, Self::IntermediaryRepr) {
|
||||||
|
let (output, intermediary) = self
|
||||||
|
.layers
|
||||||
|
.eval_training(&self.input_to_residual_input(input));
|
||||||
|
|
||||||
|
let result: Rc<Self::Output> = output.get_first().unwrap().into();
|
||||||
|
|
||||||
|
(
|
||||||
|
Rc::try_unwrap(result).unwrap_or_else(|result| (*result).clone()),
|
||||||
|
intermediary,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<
|
||||||
|
Data: Clone,
|
||||||
|
Layers: NeuraTrainableLayerSelf<NeuraResidualInput<Data>, Output = NeuraResidualInput<Data>>,
|
||||||
|
> NeuraTrainableLayerSelf<Data> for NeuraResidual<Layers>
|
||||||
|
{
|
||||||
|
fn regularize_layer(&self) -> Self::Gradient {
|
||||||
|
self.layers.regularize_layer()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_gradient(
|
||||||
|
&self,
|
||||||
|
input: &Data,
|
||||||
|
intermediary: &Self::IntermediaryRepr,
|
||||||
|
epsilon: &Self::Output,
|
||||||
|
) -> Self::Gradient {
|
||||||
|
let epsilon = Rc::new(epsilon.clone());
|
||||||
|
let mut epsilon_residual = NeuraResidualInput::new();
|
||||||
|
|
||||||
|
epsilon_residual.push(0, epsilon);
|
||||||
|
|
||||||
|
self.layers.get_gradient(
|
||||||
|
&self.input_to_residual_input(input),
|
||||||
|
intermediary,
|
||||||
|
&epsilon_residual,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<Layers> NeuraNetworkBase for NeuraResidual<Layers> {
|
||||||
|
type Layer = ();
|
||||||
|
|
||||||
|
#[inline(always)]
|
||||||
|
fn get_layer(&self) -> &Self::Layer {
|
||||||
|
&()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<Layers: NeuraTrainableLayerBase> NeuraNetworkRec for NeuraResidual<Layers> {
|
||||||
|
type NextNode = Layers;
|
||||||
|
|
||||||
|
#[inline(always)]
|
||||||
|
fn get_next(&self) -> &Self::NextNode {
|
||||||
|
&self.layers
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline(always)]
|
||||||
|
fn merge_gradient(
|
||||||
|
&self,
|
||||||
|
rec_gradient: <Self::NextNode as NeuraTrainableLayerBase>::Gradient,
|
||||||
|
_layer_gradient: <Self::Layer as NeuraTrainableLayerBase>::Gradient,
|
||||||
|
) -> Self::Gradient {
|
||||||
|
rec_gradient
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<Data: Clone, Layers> NeuraNetwork<Data> for NeuraResidual<Layers> {
|
||||||
|
type LayerInput = Data;
|
||||||
|
type NodeOutput = NeuraResidualInput<Data>;
|
||||||
|
|
||||||
|
#[inline(always)]
|
||||||
|
fn map_input<'a>(&'_ self, input: &'a Data) -> Cow<'a, Self::LayerInput> {
|
||||||
|
Cow::Borrowed(input)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline(always)]
|
||||||
|
fn map_output<'a>(
|
||||||
|
&'_ self,
|
||||||
|
_input: &'_ Data,
|
||||||
|
layer_output: &'a Data,
|
||||||
|
) -> Cow<'a, Self::NodeOutput> {
|
||||||
|
let layer_output = Rc::new(layer_output.clone());
|
||||||
|
let mut outputs = NeuraResidualInput::new();
|
||||||
|
|
||||||
|
for &offset in &self.initial_offsets {
|
||||||
|
outputs.push(offset, Rc::clone(&layer_output));
|
||||||
|
}
|
||||||
|
|
||||||
|
Cow::Owned(outputs)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline(always)]
|
||||||
|
fn map_gradient_in<'a>(
|
||||||
|
&'_ self,
|
||||||
|
_input: &'_ Data,
|
||||||
|
gradient_in: &'a Self::NodeOutput,
|
||||||
|
) -> Cow<'a, Data> {
|
||||||
|
let first = gradient_in
|
||||||
|
.clone()
|
||||||
|
.get_first()
|
||||||
|
.expect("No outgoing gradient in NeuraResidual on the last node");
|
||||||
|
|
||||||
|
Cow::Owned((*first).clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline(always)]
|
||||||
|
fn map_gradient_out<'a>(
|
||||||
|
&'_ self,
|
||||||
|
_input: &'_ Data,
|
||||||
|
_gradient_in: &'_ Self::NodeOutput,
|
||||||
|
gradient_out: &'a Self::LayerInput,
|
||||||
|
) -> Cow<'a, Data> {
|
||||||
|
Cow::Borrowed(gradient_out)
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in new issue