amethyst
/
neuramethyst
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

🔥 🎨 Remove deprecated traits (NeuraOldTrainableNetwork and NeuraGradientSolver*)

Browse Source
main
Shad Amethyst 2 years ago
parent 4da4be22b4
commit 741cf1ced6
7 changed files with 13 additions and 297 deletions
Show Stats Download Patch File Download Diff File

45
src/gradient_solver/backprop.rs
Unescape View File

@ -45,51 +45,6 @@ where
}
}
impl<Loss, Target> NeuraGradientSolverBase for (&NeuraBackprop<Loss>, &Target) {
type Output<NetworkInput, NetworkGradient> = (NetworkInput, NetworkGradient); // epsilon, gradient
}
impl<LayerOutput, Target, Loss: NeuraLoss<LayerOutput, Target = Target>>
NeuraGradientSolverFinal<LayerOutput> for (&NeuraBackprop<Loss>, &Target)
{
fn eval_final(&self, output: LayerOutput) -> Self::Output<LayerOutput, ()> {
(self.0.loss.nabla(self.1, &output), ())
}
}
impl<
Input,
Target,
Loss,
Layer: NeuraTrainableLayerBackprop<Input> + NeuraTrainableLayerSelf<Input>,
> NeuraGradientSolverTransient<Input, Layer> for (&NeuraBackprop<Loss>, &Target)
{
fn eval_layer<NetworkGradient, RecGradient>(
&self,
layer: &Layer,
input: &Input,
_output: &Layer::Output,
intermediary: &Layer::IntermediaryRepr,
rec_opt_output: Self::Output<Layer::Output, RecGradient>,
combine_gradients: impl Fn(Layer::Gradient, RecGradient) -> NetworkGradient,
) -> Self::Output<Input, NetworkGradient> {
let (epsilon_in, rec_gradient) = rec_opt_output;
let epsilon_out = layer.backprop_layer(input, intermediary, &epsilon_in);
let layer_gradient = layer.get_gradient(input, intermediary, &epsilon_in);
(epsilon_out, combine_gradients(layer_gradient, rec_gradient))
}
fn map_epsilon<From, To, Gradient, Cb: Fn(From) -> To>(
&self,
rec_opt_output: Self::Output<From, Gradient>,
callback: Cb,
) -> Self::Output<To, Gradient> {
(callback(rec_opt_output.0), rec_opt_output.1)
}
}
trait BackpropRecurse<Input, Network, Gradient> {
fn recurse(&self, network: &Network, input: &Input) -> (Input, Gradient);
}

81
src/gradient_solver/forward_forward.rs
Unescape View File

@ -1,5 +1,5 @@
use nalgebra::{DVector, Scalar};
use num::{traits::NumAssignOps, Float, ToPrimitive};
use num::{traits::NumAssignOps, Float};
use crate::{
derivable::NeuraDerivable,
@ -65,16 +65,13 @@ struct NeuraForwardPair<Act> {
activation: Act,
}
impl<F, Act: Clone + NeuraDerivable<f64>, Input: Clone, Trainable: NeuraTrainableLayerBase>
NeuraGradientSolver<Input, bool, Trainable> for NeuraForwardForward<Act>
impl<
F: Float,
Act: Clone + NeuraDerivable<f64>,
Input: Clone,
Trainable: NeuraTrainableLayerBase + NeuraLayer<Input, Output = DVector<F>>,
> NeuraGradientSolver<Input, bool, Trainable> for NeuraForwardForward<Act>
where
F: ToPrimitive,
Trainable: NeuraOldTrainableNetwork<
Input,
NeuraForwardPair<Act>,
Output = DVector<F>,
Gradient = <Trainable as NeuraTrainableLayerBase>::Gradient,
>,
NeuraForwardPair<Act>:
ForwardForwardRecurse<Input, Trainable, <Trainable as NeuraTrainableLayerBase>::Gradient>,
{
@ -126,70 +123,6 @@ where
}
}
impl<Act> NeuraGradientSolverBase for NeuraForwardPair<Act> {
type Output<NetworkInput, NetworkGradient> = NetworkGradient;
}
impl<Act, LayerOutput> NeuraGradientSolverFinal<LayerOutput> for NeuraForwardPair<Act> {
fn eval_final(&self, _output: LayerOutput) -> Self::Output<LayerOutput, ()> {
()
}
}
impl<
F: Float + Scalar + NumAssignOps,
Act: NeuraDerivable<F>,
Input,
Layer: NeuraTrainableLayerSelf<Input, Output = DVector<F>>,
> NeuraGradientSolverTransient<Input, Layer> for NeuraForwardPair<Act>
{
fn eval_layer<NetworkGradient, RecGradient>(
&self,
layer: &Layer,
input: &Input,
output: &Layer::Output,
intermediary: &Layer::IntermediaryRepr,
rec_gradient: RecGradient,
combine_gradients: impl Fn(Layer::Gradient, RecGradient) -> NetworkGradient,
) -> Self::Output<Input, NetworkGradient> {
// let output = layer.eval(input);
let goodness = output
.iter()
.copied()
.reduce(|acc, x| acc + x * x)
.unwrap_or(F::zero());
let goodness = if self.maximize {
goodness - F::from(self.threshold).unwrap()
} else {
F::from(self.threshold).unwrap() - goodness
};
// We skip self.activation.eval(goodness)
let two = F::from(2.0).unwrap();
// The original formula does not have a 1/2 term,
// so we must multiply by 2
let mut goodness_derivative = output * (two * self.activation.derivate(goodness));
if self.maximize {
goodness_derivative = -goodness_derivative;
}
// TODO: split backprop_layer into eval_training, get_gradient and get_backprop
let layer_gradient = layer.get_gradient(input, intermediary, &goodness_derivative);
combine_gradients(layer_gradient, rec_gradient)
}
fn map_epsilon<From, To, Gradient, Cb: Fn(From) -> To>(
&self,
rec_opt_output: Self::Output<From, Gradient>,
_callback: Cb,
) -> Self::Output<To, Gradient> {
rec_opt_output
}
}
trait ForwardForwardRecurse<Input, Network, Gradient> {
fn recurse(&self, network: &Network, input: &Input) -> Gradient;
}

33
src/gradient_solver/mod.rs
Unescape View File

@ -4,38 +4,7 @@ pub use backprop::NeuraBackprop;
mod forward_forward;
pub use forward_forward::NeuraForwardForward;
use crate::{
layer::{NeuraTrainableLayerBase, NeuraTrainableLayerEval},
network::{NeuraOldTrainableNetwork, NeuraOldTrainableNetworkBase},
};
pub trait NeuraGradientSolverBase {
type Output<NetworkInput, NetworkGradient>;
}
pub trait NeuraGradientSolverFinal<LayerOutput>: NeuraGradientSolverBase {
fn eval_final(&self, output: LayerOutput) -> Self::Output<LayerOutput, ()>;
}
pub trait NeuraGradientSolverTransient<Input, Layer: NeuraTrainableLayerEval<Input>>:
NeuraGradientSolverBase
{
fn eval_layer<NetworkGradient, RecGradient>(
&self,
layer: &Layer,
input: &Input,
output: &Layer::Output,
layer_intermediary: &Layer::IntermediaryRepr,
rec_opt_output: Self::Output<Layer::Output, RecGradient>,
combine_gradients: impl Fn(Layer::Gradient, RecGradient) -> NetworkGradient,
) -> Self::Output<Input, NetworkGradient>;
fn map_epsilon<From, To, Gradient, Cb: Fn(From) -> To>(
&self,
rec_opt_output: Self::Output<From, Gradient>,
callback: Cb,
) -> Self::Output<To, Gradient>;
}
use crate::layer::{NeuraTrainableLayerBase, NeuraTrainableLayerEval};
pub trait NeuraGradientSolver<Input, Target, Trainable: NeuraTrainableLayerBase> {
fn get_gradient(

45
src/network/mod.rs
Unescape View File

@ -1,50 +1,5 @@
use crate::{
algebra::NeuraVectorSpace,
gradient_solver::{NeuraGradientSolverBase, NeuraGradientSolverFinal},
layer::NeuraLayer,
};
// pub mod residual;
pub mod sequential;
mod traits;
pub use traits::*;
// TODO: extract regularize from this, so that we can drop the trait constraints on NeuraSequential's impl
pub trait NeuraOldTrainableNetworkBase<Input>: NeuraLayer<Input> {
type Gradient: NeuraVectorSpace;
type LayerOutput;
fn default_gradient(&self) -> Self::Gradient;
fn apply_gradient(&mut self, gradient: &Self::Gradient);
/// Should return the regularization gradient
fn regularize(&self) -> Self::Gradient;
/// Called before an iteration begins, to allow the network to set itself up for training or not.
fn prepare(&mut self, train_iteration: bool);
}
pub trait NeuraOldTrainableNetwork<Input, Optimizer>: NeuraOldTrainableNetworkBase<Input>
where
Optimizer: NeuraGradientSolverBase,
{
fn traverse(
&self,
input: &Input,
optimizer: &Optimizer,
) -> Optimizer::Output<Input, Self::Gradient>;
}
impl<Input: Clone, Optimizer: NeuraGradientSolverFinal<Input>>
NeuraOldTrainableNetwork<Input, Optimizer> for ()
{
fn traverse(
&self,
input: &Input,
optimizer: &Optimizer,
) -> Optimizer::Output<Input, Self::Gradient> {
optimizer.eval_final(input.clone())
}
}

97
src/network/sequential/mod.rs
Unescape View File

@ -1,12 +1,9 @@
use std::borrow::Cow;
use super::*;
use crate::{
gradient_solver::NeuraGradientSolverTransient,
layer::{
NeuraLayer, NeuraPartialLayer, NeuraShape, NeuraTrainableLayerBase,
NeuraTrainableLayerEval, NeuraTrainableLayerSelf,
},
use crate::layer::{
NeuraLayer, NeuraPartialLayer, NeuraShape, NeuraTrainableLayerBase, NeuraTrainableLayerEval,
NeuraTrainableLayerSelf,
};
mod construct;
@ -82,94 +79,6 @@ impl<Layer, ChildNetwork> NeuraSequential<Layer, ChildNetwork> {
}
}
impl<
Input,
Layer: NeuraTrainableLayerEval<Input> + NeuraTrainableLayerSelf<Input>,
ChildNetwork: NeuraOldTrainableNetworkBase<Layer::Output>,
> NeuraOldTrainableNetworkBase<Input> for NeuraSequential<Layer, ChildNetwork>
{
type Gradient = (Layer::Gradient, Box<ChildNetwork::Gradient>);
type LayerOutput = Layer::Output;
fn default_gradient(&self) -> Self::Gradient {
(
self.layer.default_gradient(),
Box::new(self.child_network.default_gradient()),
)
}
fn apply_gradient(&mut self, gradient: &Self::Gradient) {
self.layer.apply_gradient(&gradient.0);
self.child_network.apply_gradient(&gradient.1);
}
fn regularize(&self) -> Self::Gradient {
(
self.layer.regularize_layer(),
Box::new(self.child_network.regularize()),
)
}
fn prepare(&mut self, is_training: bool) {
self.layer.prepare_layer(is_training);
self.child_network.prepare(is_training);
}
}
/// A dummy implementation of `NeuraTrainableNetwork`, which simply calls `loss.eval` in `backpropagate`.
impl<Input: Clone> NeuraOldTrainableNetworkBase<Input> for () {
type Gradient = ();
type LayerOutput = Input;
#[inline(always)]
fn default_gradient(&self) -> () {
()
}
#[inline(always)]
fn apply_gradient(&mut self, _gradient: &()) {
// Noop
}
#[inline(always)]
fn regularize(&self) -> () {
()
}
#[inline(always)]
fn prepare(&mut self, _is_training: bool) {
// Noop
}
}
impl<
Input,
Layer: NeuraTrainableLayerEval<Input> + NeuraTrainableLayerSelf<Input>,
Optimizer: NeuraGradientSolverTransient<Input, Layer>,
ChildNetwork: NeuraOldTrainableNetworkBase<Layer::Output>,
> NeuraOldTrainableNetwork<Input, Optimizer> for NeuraSequential<Layer, ChildNetwork>
where
ChildNetwork: NeuraOldTrainableNetwork<Layer::Output, Optimizer>,
{
fn traverse(
&self,
input: &Input,
optimizer: &Optimizer,
) -> Optimizer::Output<Input, Self::Gradient> {
let (next_activation, intermediary) = self.layer.eval_training(input);
let child_result = self.child_network.traverse(&next_activation, optimizer);
optimizer.eval_layer(
&self.layer,
input,
&next_activation,
&intermediary,
child_result,
|layer_gradient, child_gradient| (layer_gradient, Box::new(child_gradient)),
)
}
}
impl<Layer> From<Layer> for NeuraSequential<Layer, ()> {
fn from(layer: Layer) -> Self {
Self {

4
src/network/traits.rs
Unescape View File

@ -1,8 +1,6 @@
use std::borrow::Cow;
use crate::prelude::NeuraTrainableLayerBase;
use super::*;
use crate::layer::*;
/// This trait has to be non-generic, to ensure that no downstream crate can implement it for foreign types,
/// as that would otherwise cause infinite recursion when dealing with `NeuraNetworkRec`.

5
src/train.rs
Unescape View File

@ -1,7 +1,4 @@
use crate::{
algebra::NeuraVectorSpace, gradient_solver::NeuraGradientSolver, layer::*,
network::NeuraOldTrainableNetworkBase,
};
use crate::{algebra::NeuraVectorSpace, gradient_solver::NeuraGradientSolver, layer::*};
#[non_exhaustive]
pub struct NeuraBatchedTrainer {

Write Preview
Loading…
Cancel
Save