Implement NeuraNetwork for NeuraResidual and NeuraResidualNode

main
Shad Amethyst 2 years ago
parent 741cf1ced6
commit b34b1e630b

@ -21,3 +21,6 @@ viuer = "0.6.2"
rust-mnist = "0.2.0"
serde_json = "1.0.96"
approx = "0.5.1"
[profile.release]
debug = true

@ -0,0 +1,138 @@
use nalgebra::{dvector, DVector};
use neuramethyst::derivable::activation::Tanh;
use neuramethyst::derivable::regularize::NeuraL1;
use neuramethyst::gradient_solver::NeuraForwardForward;
use neuramethyst::{plot_losses, prelude::*};
use rand::Rng;
fn main() {
let mut network = neura_residual![
<= 0, 2;
neura_layer!("dense", 6).regularization(NeuraL1(0.001));
neura_layer!("normalize");
neura_layer!("dense", 6).regularization(NeuraL1(0.001));
]
.construct(NeuraShape::Vector(3))
.unwrap();
let inputs = (0..1).cycle().map(move |_| {
let mut rng = rand::thread_rng();
let category = rng.gen_bool(0.5);
let good = rng.gen_bool(0.5);
let (x, y) = if category {
let radius: f32 = rng.gen_range(0.0..2.0);
let angle = rng.gen_range(0.0..std::f32::consts::TAU);
(angle.cos() * radius, angle.sin() * radius)
} else {
let radius: f32 = rng.gen_range(3.0..5.0);
let angle = rng.gen_range(0.0..std::f32::consts::TAU);
(angle.cos() * radius, angle.sin() * radius)
};
if good {
(dvector![x, y, category as u8 as f32], true)
} else {
(dvector![x, y, 1.0 - category as u8 as f32], false)
}
});
let test_inputs: Vec<_> = inputs.clone().filter(|(_, good)| *good).take(10).collect();
let threshold = 0.5f32;
if std::env::args().any(|arg| arg == "draw") {
for epoch in 0..200 {
let mut trainer = NeuraBatchedTrainer::new(0.03, 10);
trainer.batch_size = 50;
trainer.train(
&NeuraForwardForward::new(Tanh, threshold as f64),
&mut network,
inputs.clone(),
&test_inputs,
);
// let network = network.clone().trim_tail().trim_tail();
draw_neuron_activation(
|input| {
let cat0 = network.eval(&dvector![input[0] as f32, input[1] as f32, 0.0]);
let cat1 = network.eval(&dvector![input[0] as f32, input[1] as f32, 1.0]);
let cat0_good = cat0.map(|x| x * x).sum();
let cat1_good = cat1.map(|x| x * x).sum();
let estimation = cat1_good / (cat0_good + cat1_good);
let cat0_norm = cat0 / cat0_good.sqrt();
let mut cat0_rgb = DVector::from_element(3, 0.0);
for i in 0..cat0_norm.len() {
cat0_rgb[i % 3] += cat0_norm[i].abs();
}
(cat0_rgb * estimation)
.into_iter()
.map(|x| *x as f64)
.collect()
},
6.0,
);
println!("{}", epoch);
std::thread::sleep(std::time::Duration::new(0, 50_000_000));
}
} else {
let mut trainer = NeuraBatchedTrainer::new(0.03, 20 * 50);
trainer.batch_size = 50;
trainer.log_iterations = 20;
plot_losses(
trainer.train(
&NeuraForwardForward::new(Tanh, threshold as f64),
&mut network,
inputs.clone(),
&test_inputs,
),
128,
48,
);
// println!("{}", String::from("\n").repeat(64));
// draw_neuron_activation(|input| network.eval(&input).into_iter().collect(), 6.0);
}
}
// TODO: move this to the library?
fn draw_neuron_activation<F: Fn([f64; 2]) -> Vec<f64>>(callback: F, scale: f64) {
use viuer::Config;
const WIDTH: u32 = 64;
const HEIGHT: u32 = 64;
let mut image = image::RgbImage::new(WIDTH, HEIGHT);
fn sigmoid(x: f64) -> f64 {
0.1 + 0.9 * x.abs().powf(0.8)
}
for y in 0..HEIGHT {
let y2 = 2.0 * y as f64 / HEIGHT as f64 - 1.0;
for x in 0..WIDTH {
let x2 = 2.0 * x as f64 / WIDTH as f64 - 1.0;
let activation = callback([x2 * scale, y2 * scale]);
let r = (sigmoid(activation.get(0).copied().unwrap_or(-1.0)) * 255.0).floor() as u8;
let g = (sigmoid(activation.get(1).copied().unwrap_or(-1.0)) * 255.0).floor() as u8;
let b = (sigmoid(activation.get(2).copied().unwrap_or(-1.0)) * 255.0).floor() as u8;
*image.get_pixel_mut(x, y) = image::Rgb([r, g, b]);
}
}
let config = Config {
use_kitty: false,
truecolor: true,
// absolute_offset: false,
..Default::default()
};
viuer::print(&image::DynamicImage::ImageRgb8(image), &config).unwrap();
}

@ -259,20 +259,20 @@ impl<'a, const LENGTH: usize, F: Default + Clone> FromIterator<F> for NeuraVecto
mod test {
use super::*;
#[test]
fn test_reverse_dot() {
let left: NeuraVector<_, f64> = [2.0, 3.0, 5.0].into();
let right: NeuraVector<_, f64> = [7.0, 11.0, 13.0, 17.0].into();
// #[test]
// fn test_reverse_dot() {
// let left: NeuraVector<_, f64> = [2.0, 3.0, 5.0].into();
// let right: NeuraVector<_, f64> = [7.0, 11.0, 13.0, 17.0].into();
let expected: NeuraMatrix<_, _, f64> = [
[14.0, 22.0, 26.0, 34.0],
[21.0, 33.0, 39.0, 51.0],
[35.0, 55.0, 65.0, 85.0],
]
.into();
// let expected: NeuraMatrix<_, _, f64> = [
// [14.0, 22.0, 26.0, 34.0],
// [21.0, 33.0, 39.0, 51.0],
// [35.0, 55.0, 65.0, 85.0],
// ]
// .into();
let actual = left.reverse_dot(right);
// let actual = left.reverse_dot(right);
assert_eq!(expected, actual);
}
// assert_eq!(expected, actual);
// }
}

@ -1,8 +1,3 @@
#![feature(generic_arg_infer)]
// #![feature(generic_const_exprs)]
#![feature(associated_type_defaults)]
#![feature(arc_unwrap_or_clone)]
pub mod algebra;
pub mod derivable;
pub mod gradient_solver;
@ -20,7 +15,7 @@ pub use utils::{argmax, cycle_shuffling, one_hot, plot_losses};
/// so there should not be any conflicts when doing a wildcard import of `prelude`.
pub mod prelude {
// Macros
pub use crate::{neura_layer, neura_sequential};
pub use crate::{neura_layer, neura_residual, neura_sequential};
// Structs and traits
pub use crate::gradient_solver::NeuraBackprop;

@ -1,4 +1,4 @@
// pub mod residual;
pub mod residual;
pub mod sequential;
mod traits;

@ -1,14 +1,18 @@
//! Implementations for NeuraLayer*
use std::borrow::Cow;
use crate::{gradient_solver::NeuraGradientSolverTransient, network::{NeuraTrainableNetwork, NeuraTrainableNetworkBase}};
use crate::network::*;
use super::*;
impl<Axis, Layer, ChildNetwork> NeuraResidualNode<Layer, ChildNetwork, Axis> {
fn process_input<Data>(&self, input: &NeuraResidualInput<Data>) -> (Axis::Combined, NeuraResidualInput<Data>)
fn process_input<Data>(
&self,
input: &NeuraResidualInput<Data>,
) -> (Axis::Combined, NeuraResidualInput<Data>)
where
Axis: NeuraCombineInputs<Data>,
Layer: NeuraLayer<Axis::Combined>
Layer: NeuraLayer<Axis::Combined>,
{
let (inputs, rest) = input.shift();
@ -17,7 +21,11 @@ impl<Axis, Layer, ChildNetwork> NeuraResidualNode<Layer, ChildNetwork, Axis> {
(layer_input, rest)
}
fn combine_outputs<Data>(&self, layer_output: Data, output: &mut NeuraResidualInput<Data>) -> Rc<Data> {
fn combine_outputs<Data>(
&self,
layer_output: Data,
output: &mut NeuraResidualInput<Data>,
) -> Rc<Data> {
let layer_output = Rc::new(layer_output);
for &offset in &self.offsets {
@ -26,6 +34,13 @@ impl<Axis, Layer, ChildNetwork> NeuraResidualNode<Layer, ChildNetwork, Axis> {
layer_output
}
pub(crate) fn map_input_owned<Data>(&self, input: &NeuraResidualInput<Data>) -> Axis::Combined
where
Axis: NeuraCombineInputs<Data>,
{
self.axis.combine(input.shift().0)
}
}
impl<F: Float + Scalar, Layer, ChildNetwork, Axis> NeuraLayer<NeuraResidualInput<DVector<F>>>
@ -46,52 +61,23 @@ where
}
}
impl<F: Clone, Output: Clone, Layers> NeuraLayer<DVector<F>> for NeuraResidual<Layers>
where
Layers: NeuraLayer<NeuraResidualInput<DVector<F>>, Output = NeuraResidualInput<Output>>,
{
type Output = Output;
fn eval(&self, input: &DVector<F>) -> Self::Output {
let input: Rc<DVector<F>> = Rc::new((*input).clone());
let mut inputs = NeuraResidualInput::new();
for &offset in &self.initial_offsets {
inputs.push(offset, Rc::clone(&input));
}
drop(input);
let output = self.layers.eval(&inputs);
let result = output.get_first()
.expect("Invalid NeuraResidual state: network returned no data, did you forget to link the last layer?")
.into();
Rc::unwrap_or_clone(result)
}
}
#[allow(dead_code)]
pub struct NeuraResidualIntermediary<LayerIntermediary, LayerOutput, ChildIntermediary> {
layer_intermediary: LayerIntermediary,
layer_output: Rc<LayerOutput>,
child_intermediary: Box<ChildIntermediary>,
}
impl<
Data,
Axis: NeuraCombineInputs<Data>,
Layer: NeuraTrainableLayerBase<Axis::Combined, Output = Data>,
ChildNetwork: NeuraTrainableLayerBase<NeuraResidualInput<Data>>
> NeuraTrainableLayerBase<NeuraResidualInput<Data>> for NeuraResidualNode<Layer, ChildNetwork, Axis>
where
NeuraResidualNode<Layer, ChildNetwork, Axis>: NeuraLayer<NeuraResidualInput<Data>, Output=ChildNetwork::Output>
impl<Layer: NeuraTrainableLayerBase, ChildNetwork: NeuraTrainableLayerBase, Axis>
NeuraTrainableLayerBase for NeuraResidualNode<Layer, ChildNetwork, Axis>
{
type Gradient = (Layer::Gradient, Box<ChildNetwork::Gradient>);
type IntermediaryRepr = NeuraResidualIntermediary<Layer::IntermediaryRepr, Layer::Output, ChildNetwork::IntermediaryRepr>;
fn default_gradient(&self) -> Self::Gradient {
(self.layer.default_gradient(), Box::new(self.child_network.default_gradient()))
(
self.layer.default_gradient(),
Box::new(self.child_network.default_gradient()),
)
}
fn apply_gradient(&mut self, gradient: &Self::Gradient) {
@ -99,7 +85,33 @@ where
self.child_network.apply_gradient(&gradient.1);
}
fn eval_training(&self, input: &NeuraResidualInput<Data>) -> (Self::Output, Self::IntermediaryRepr) {
fn prepare_layer(&mut self, is_training: bool) {
self.layer.prepare_layer(is_training);
self.child_network.prepare_layer(is_training);
}
}
impl<
Data,
Axis: NeuraCombineInputs<Data>,
Layer: NeuraTrainableLayerEval<Axis::Combined, Output = Data>,
ChildNetwork: NeuraTrainableLayerEval<NeuraResidualInput<Data>>,
> NeuraTrainableLayerEval<NeuraResidualInput<Data>>
for NeuraResidualNode<Layer, ChildNetwork, Axis>
where
NeuraResidualNode<Layer, ChildNetwork, Axis>:
NeuraLayer<NeuraResidualInput<Data>, Output = ChildNetwork::Output>,
{
type IntermediaryRepr = NeuraResidualIntermediary<
Layer::IntermediaryRepr,
Layer::Output,
ChildNetwork::IntermediaryRepr,
>;
fn eval_training(
&self,
input: &NeuraResidualInput<Data>,
) -> (Self::Output, Self::IntermediaryRepr) {
let (layer_input, mut rest) = self.process_input(input);
let (layer_output, layer_intermediary) = self.layer.eval_training(&layer_input);
@ -110,89 +122,119 @@ where
let intermediary = NeuraResidualIntermediary {
layer_intermediary,
layer_output,
child_intermediary: Box::new(child_intermediary)
child_intermediary: Box::new(child_intermediary),
};
(output, intermediary)
}
fn prepare_layer(&mut self, is_training: bool) {
self.layer.prepare_layer(is_training);
self.child_network.prepare_layer(is_training);
}
}
impl<
Data,
Axis: NeuraCombineInputs<Data>,
Layer: NeuraTrainableLayerSelf<Axis::Combined, Output = Data>,
ChildNetwork: NeuraTrainableNetworkBase<NeuraResidualInput<Data>>,
> NeuraTrainableNetworkBase<NeuraResidualInput<Data>> for NeuraResidualNode<Layer, ChildNetwork, Axis>
ChildNetwork: NeuraTrainableLayerSelf<NeuraResidualInput<Data>>,
> NeuraTrainableLayerSelf<NeuraResidualInput<Data>>
for NeuraResidualNode<Layer, ChildNetwork, Axis>
where
Self: NeuraTrainableLayerBase<NeuraResidualInput<Data>, Gradient = (Layer::Gradient, Box<ChildNetwork::Gradient>)>,
NeuraResidualNode<Layer, ChildNetwork, Axis>:
NeuraLayer<NeuraResidualInput<Data>, Output = ChildNetwork::Output>,
{
type Gradient = <Self as NeuraTrainableLayerBase<NeuraResidualInput<Data>>>::Gradient;
type LayerOutput = Layer::Output;
fn regularize_layer(&self) -> Self::Gradient {
(
self.layer.regularize_layer(),
Box::new(self.child_network.regularize_layer()),
)
}
fn default_gradient(&self) -> Self::Gradient {
<Self as NeuraTrainableLayerBase<NeuraResidualInput<Data>>>::default_gradient(self)
fn get_gradient(
&self,
input: &NeuraResidualInput<Data>,
intermediary: &Self::IntermediaryRepr,
epsilon: &Self::Output,
) -> Self::Gradient {
unimplemented!();
}
}
fn apply_gradient(&mut self, gradient: &Self::Gradient) {
<Self as NeuraTrainableLayerBase<NeuraResidualInput<Data>>>::apply_gradient(self, gradient)
impl<Axis, Layer, ChildNetwork> NeuraNetworkBase for NeuraResidualNode<Layer, ChildNetwork, Axis> {
type Layer = Layer;
fn get_layer(&self) -> &Self::Layer {
&self.layer
}
}
impl<Axis, Layer: NeuraTrainableLayerBase, ChildNetwork: NeuraTrainableLayerBase> NeuraNetworkRec
for NeuraResidualNode<Layer, ChildNetwork, Axis>
{
type NextNode = ChildNetwork;
fn regularize(&self) -> Self::Gradient {
(self.layer.regularize_layer(), Box::new(self.child_network.regularize()))
fn get_next(&self) -> &Self::NextNode {
&self.child_network
}
fn prepare(&mut self, train_iteration: bool) {
self.layer.prepare_layer(train_iteration);
self.child_network.prepare(train_iteration);
fn merge_gradient(
&self,
rec_gradient: <Self::NextNode as NeuraTrainableLayerBase>::Gradient,
layer_gradient: <Self::Layer as NeuraTrainableLayerBase>::Gradient,
) -> Self::Gradient {
(layer_gradient, Box::new(rec_gradient))
}
}
impl<
Data,
Axis: NeuraSplitInputs<Data>,
Layer: NeuraTrainableLayerSelf<Axis::Combined, Output = Data>,
Optimizer: NeuraGradientSolverTransient<Axis::Combined, Layer>,
ChildNetwork: NeuraTrainableNetwork<NeuraResidualInput<Data>, Optimizer>,
> NeuraTrainableNetwork<NeuraResidualInput<Data>, Optimizer> for NeuraResidualNode<Layer, ChildNetwork, Axis>
Data: Clone,
Axis: NeuraCombineInputs<Data>,
Layer: NeuraLayer<Axis::Combined, Output = Data>,
ChildNetwork,
> NeuraNetwork<NeuraResidualInput<Data>> for NeuraResidualNode<Layer, ChildNetwork, Axis>
where
Self: NeuraTrainableLayerBase<NeuraResidualInput<Data>, Gradient = (Layer::Gradient, Box<ChildNetwork::Gradient>)>,
Layer::Output: Clone,
Axis::Combined: Clone,
{
fn traverse(
&self,
input: &NeuraResidualInput<Data>,
optimizer: &Optimizer,
) -> Optimizer::Output<NeuraResidualInput<Data>, Self::Gradient> {
let (layer_input, mut rest) = self.process_input(input);
let (layer_output, layer_intermediary) = self.layer.eval_training(&layer_input);
let layer_output = self.combine_outputs(layer_output, &mut rest);
type LayerInput = Axis::Combined;
type NodeOutput = NeuraResidualInput<Data>;
fn map_input<'a>(&'_ self, input: &'a NeuraResidualInput<Data>) -> Cow<'a, Self::LayerInput> {
Cow::Owned(self.map_input_owned(input))
}
fn map_output<'a>(
&'_ self,
input: &'_ NeuraResidualInput<Data>,
layer_output: &'a <Self::Layer as NeuraLayer<Self::LayerInput>>::Output,
) -> Cow<'a, Self::NodeOutput> {
let mut remaining_inputs = input.shift().1;
self.combine_outputs(layer_output.clone(), &mut remaining_inputs);
let child_result = self.child_network.traverse(&rest, optimizer);
// TODO: maybe move this to a custom impl of NeuraGradientSolverTransient for NeuraResidualInput?
// Or have a different set of traits for NeuraTrainableNetwork specific to NeuraResidualNodes
let child_result = optimizer.map_epsilon(child_result, |_epsilon| {
Cow::Owned(remaining_inputs)
}
#[allow(unused_variables)]
fn map_gradient_in<'a>(
&'_ self,
input: &'_ NeuraResidualInput<Data>,
gradient_in: &'a Self::NodeOutput,
) -> Cow<'a, <Self::Layer as NeuraLayer<Self::LayerInput>>::Output> {
// To convert from gradient_in to layer's gradient_in:
// Pop the first value from `epsilon`, then:
// - compute its sum
// - use it to compute the outcoming epsilon of the current layer
// - split the oucoming epsilon into its original components, and push those back onto the rest
// At this point, the value for `epsilon` in the gradient solver's state should be ready for another iteration,
// with the first value containing the unsummed incoming epsilon values from the downstream layers
todo!()
});
optimizer.eval_layer(
&self.layer,
&layer_input,
&layer_output,
&layer_intermediary,
child_result,
|this_gradient, child_gradient| (this_gradient, Box::new(child_gradient))
);
todo!();
unimplemented!()
}
#[allow(unused_variables)]
fn map_gradient_out<'a>(
&'_ self,
input: &'_ NeuraResidualInput<Data>,
gradient_in: &'_ Self::NodeOutput,
gradient_out: &'a Self::LayerInput,
) -> Cow<'a, NeuraResidualInput<Data>> {
unimplemented!()
}
}

@ -7,6 +7,9 @@ use crate::layer::*;
mod layer_impl;
mod wrapper;
pub use wrapper::*;
mod input;
pub use input::*;
@ -16,15 +19,6 @@ pub use axis::*;
mod construct;
pub use construct::NeuraResidualConstructErr;
#[derive(Clone, Debug, PartialEq)]
pub struct NeuraResidual<Layers> {
/// Instance of NeuraResidualNode
layers: Layers,
/// Array of which layers to send the input to, defaults to `vec![0]`
initial_offsets: Vec<usize>,
}
impl<Layers> NeuraResidual<Layers> {
pub fn new(layers: Layers) -> Self {
Self {

@ -0,0 +1,187 @@
use std::borrow::Cow;
use crate::network::*;
use super::*;
#[derive(Clone, Debug, PartialEq)]
pub struct NeuraResidual<Layers> {
/// Instance of NeuraResidualNode
pub(crate) layers: Layers,
/// Array of which layers to send the input to, defaults to `vec![0]`
pub(crate) initial_offsets: Vec<usize>,
}
impl<Layers> NeuraResidual<Layers> {
fn input_to_residual_input<Input: Clone>(&self, input: &Input) -> NeuraResidualInput<Input> {
let input: Rc<Input> = Rc::new((*input).clone());
let mut inputs = NeuraResidualInput::new();
for &offset in &self.initial_offsets {
inputs.push(offset, Rc::clone(&input));
}
drop(input);
inputs
}
}
impl<Input: Clone, Output: Clone, Layers> NeuraLayer<Input> for NeuraResidual<Layers>
where
Layers: NeuraLayer<NeuraResidualInput<Input>, Output = NeuraResidualInput<Output>>,
{
type Output = Output;
fn eval(&self, input: &Input) -> Self::Output {
let output = self.layers.eval(&self.input_to_residual_input(input));
let result: Rc<Self::Output> = output.get_first()
.expect("Invalid NeuraResidual state: network returned no data, did you forget to link the last layer?")
.into();
// TODO: replace with Rc::unwrap_or_clone once https://github.com/rust-lang/rust/issues/93610 is closed
Rc::try_unwrap(result).unwrap_or_else(|result| (*result).clone())
}
}
impl<Layers: NeuraTrainableLayerBase> NeuraTrainableLayerBase for NeuraResidual<Layers> {
type Gradient = Layers::Gradient;
#[inline(always)]
fn default_gradient(&self) -> Self::Gradient {
self.layers.default_gradient()
}
#[inline(always)]
fn apply_gradient(&mut self, gradient: &Self::Gradient) {
self.layers.apply_gradient(gradient);
}
}
impl<
Data: Clone,
Layers: NeuraTrainableLayerEval<NeuraResidualInput<Data>, Output = NeuraResidualInput<Data>>,
> NeuraTrainableLayerEval<Data> for NeuraResidual<Layers>
{
type IntermediaryRepr = Layers::IntermediaryRepr;
fn eval_training(&self, input: &Data) -> (Self::Output, Self::IntermediaryRepr) {
let (output, intermediary) = self
.layers
.eval_training(&self.input_to_residual_input(input));
let result: Rc<Self::Output> = output.get_first().unwrap().into();
(
Rc::try_unwrap(result).unwrap_or_else(|result| (*result).clone()),
intermediary,
)
}
}
impl<
Data: Clone,
Layers: NeuraTrainableLayerSelf<NeuraResidualInput<Data>, Output = NeuraResidualInput<Data>>,
> NeuraTrainableLayerSelf<Data> for NeuraResidual<Layers>
{
fn regularize_layer(&self) -> Self::Gradient {
self.layers.regularize_layer()
}
fn get_gradient(
&self,
input: &Data,
intermediary: &Self::IntermediaryRepr,
epsilon: &Self::Output,
) -> Self::Gradient {
let epsilon = Rc::new(epsilon.clone());
let mut epsilon_residual = NeuraResidualInput::new();
epsilon_residual.push(0, epsilon);
self.layers.get_gradient(
&self.input_to_residual_input(input),
intermediary,
&epsilon_residual,
)
}
}
impl<Layers> NeuraNetworkBase for NeuraResidual<Layers> {
type Layer = ();
#[inline(always)]
fn get_layer(&self) -> &Self::Layer {
&()
}
}
impl<Layers: NeuraTrainableLayerBase> NeuraNetworkRec for NeuraResidual<Layers> {
type NextNode = Layers;
#[inline(always)]
fn get_next(&self) -> &Self::NextNode {
&self.layers
}
#[inline(always)]
fn merge_gradient(
&self,
rec_gradient: <Self::NextNode as NeuraTrainableLayerBase>::Gradient,
_layer_gradient: <Self::Layer as NeuraTrainableLayerBase>::Gradient,
) -> Self::Gradient {
rec_gradient
}
}
impl<Data: Clone, Layers> NeuraNetwork<Data> for NeuraResidual<Layers> {
type LayerInput = Data;
type NodeOutput = NeuraResidualInput<Data>;
#[inline(always)]
fn map_input<'a>(&'_ self, input: &'a Data) -> Cow<'a, Self::LayerInput> {
Cow::Borrowed(input)
}
#[inline(always)]
fn map_output<'a>(
&'_ self,
_input: &'_ Data,
layer_output: &'a Data,
) -> Cow<'a, Self::NodeOutput> {
let layer_output = Rc::new(layer_output.clone());
let mut outputs = NeuraResidualInput::new();
for &offset in &self.initial_offsets {
outputs.push(offset, Rc::clone(&layer_output));
}
Cow::Owned(outputs)
}
#[inline(always)]
fn map_gradient_in<'a>(
&'_ self,
_input: &'_ Data,
gradient_in: &'a Self::NodeOutput,
) -> Cow<'a, Data> {
let first = gradient_in
.clone()
.get_first()
.expect("No outgoing gradient in NeuraResidual on the last node");
Cow::Owned((*first).clone())
}
#[inline(always)]
fn map_gradient_out<'a>(
&'_ self,
_input: &'_ Data,
_gradient_in: &'_ Self::NodeOutput,
gradient_out: &'a Self::LayerInput,
) -> Cow<'a, Data> {
Cow::Borrowed(gradient_out)
}
}
Loading…
Cancel
Save