1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102

//! Cost Functions.
//!
//! This module contains a number of structs implementing the `CostFunc` trait.
//!
//! These structs are used within Neural Networks and
//! Generalized Linear Regression (not yet implemented).
//!
//! You can also create your own custom cost functions for use in your models.
//! Just create a struct implementing the `CostFunc` trait.

use linalg::Matrix;
use linalg::Vector;

/// Trait for cost functions in models.
pub trait CostFunc<T> {
    /// The cost function.
    fn cost(outputs: &T, targets: &T) -> f64;

    /// The gradient of the cost function.
    fn grad_cost(outputs: &T, targets: &T) -> T;
}

/// The mean squared error cost function.
#[derive(Clone, Copy, Debug)]
pub struct MeanSqError;

// For generics we need a trait for "Hadamard product" here
// Which is "Elementwise multiplication".
impl CostFunc<Matrix<f64>> for MeanSqError {
    fn cost(outputs: &Matrix<f64>, targets: &Matrix<f64>) -> f64 {
        let diff = outputs - targets;
        let sq_diff = &diff.elemul(&diff);

        let n = diff.rows();

        sq_diff.sum() / (2f64 * (n as f64))
    }

    fn grad_cost(outputs: &Matrix<f64>, targets: &Matrix<f64>) -> Matrix<f64> {
        outputs - targets
    }
}

impl CostFunc<Vector<f64>> for MeanSqError {
    fn cost(outputs: &Vector<f64>, targets: &Vector<f64>) -> f64 {
        let diff = outputs - targets;
        let sq_diff = &diff.elemul(&diff);

        let n = diff.size();

        sq_diff.sum() / (2f64 * (n as f64))
    }

    fn grad_cost(outputs: &Vector<f64>, targets: &Vector<f64>) -> Vector<f64> {
        outputs - targets
    }
}

/// The cross entropy error cost function.
#[derive(Clone, Copy, Debug)]
pub struct CrossEntropyError;

impl CostFunc<Matrix<f64>> for CrossEntropyError {
    fn cost(outputs: &Matrix<f64>, targets: &Matrix<f64>) -> f64 {
        // The cost for a single
        let log_inv_output = (-outputs + 1f64).apply(&ln);
        let log_output = outputs.clone().apply(&ln);

        let mat_cost = targets.elemul(&log_output) + (-targets + 1f64).elemul(&log_inv_output);

        let n = outputs.rows();

        -(mat_cost.sum()) / (n as f64)
    }

    fn grad_cost(outputs: &Matrix<f64>, targets: &Matrix<f64>) -> Matrix<f64> {
        (outputs - targets).elediv(&(outputs.elemul(&(-outputs + 1f64))))
    }
}

impl CostFunc<Vector<f64>> for CrossEntropyError {
    fn cost(outputs: &Vector<f64>, targets: &Vector<f64>) -> f64 {
        // The cost for a single
        let log_inv_output = (-outputs + 1f64).apply(&ln);
        let log_output = outputs.clone().apply(&ln);

        let mat_cost = targets.elemul(&log_output) + (-targets + 1f64).elemul(&log_inv_output);

        let n = outputs.size();

        -(mat_cost.sum()) / (n as f64)
    }

    fn grad_cost(outputs: &Vector<f64>, targets: &Vector<f64>) -> Vector<f64> {
        (outputs - targets).elediv(&(outputs.elemul(&(-outputs + 1f64))))
    }
}

/// Logarithm for applying within cost function.
fn ln(x: f64) -> f64 {
    x.ln()
}