Source code for ztlearn.objectives

# -*- coding: utf-8 -*-

import math as mt
import numpy as np


[docs]class Objective(object):
[docs] def clip(self, predictions, epsilon = 1e-15): clipped_predictions = np.clip(predictions, epsilon, 1 - epsilon) clipped_divisor = np.maximum(np.multiply(predictions, 1 - predictions), epsilon) return clipped_predictions, clipped_divisor
[docs] def error(self, predictions, targets): error = targets - predictions abs_error = np.absolute(error) return error, abs_error
[docs] def add_fuzz_factor(self, np_array, epsilon = 1e-05): return np.add(np_array, epsilon)
@property def objective_name(self): return self.__class__.__name__
[docs]class MeanSquaredError: """ **Mean Squared error (MSE)** MSE measures the average squared difference between the predictions and the targets. The closer the predictions are to the targets the more efficient the estimator. References: [1] Mean Squared error * [Wikipedia Article] https://en.wikipedia.org/wiki/Mean_squared_error """
[docs] def loss(self, predictions, targets, np_type): """ Applies the MeanSquaredError Loss to prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of MeanSquaredError Loss to prediction and targets """ return 0.5 * np.mean(np.sum(np.square(predictions - targets), axis = 1))
[docs] def derivative(self, predictions, targets, np_type): """ Applies the MeanSquaredError Derivative to prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of MeanSquaredError Derivative to prediction and targets """ return predictions - targets
[docs] def accuracy(self, predictions, targets, threshold = 0.5): return 0
@property def objective_name(self): return self.__class__.__name__
[docs]class HellingerDistance: """ **Hellinger Distance** Hellinger Distance is used to quantify the similarity between two probability distributions. References: [1] Hellinger Distance * [Wikipedia Article] https://en.wikipedia.org/wiki/Hellinger_distance """ SQRT_2 = np.sqrt(2)
[docs] def sqrt_difference(self, predictions, targets): return np.sqrt(predictions) - np.sqrt(targets)
[docs] def loss(self, predictions, targets, np_type): """ Applies the HellingerDistance Loss to prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of HellingerDistance Loss to prediction and targets """ root_difference = self.sqrt_difference(predictions, targets) return np.mean(np.true_divide(np.sum(np.square(root_difference), axis = 1), HellingerDistance.SQRT_2))
[docs] def derivative(self, predictions, targets, np_type): """ Applies the HellingerDistance Derivative to prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of HellingerDistance Derivative to prediction and targets """ root_difference = self.sqrt_difference(predictions, targets) return np.true_divide(root_difference, np.multiply(HellingerDistance.SQRT_2, np.sqrt(predictions)))
[docs] def accuracy(self, predictions, targets, threshold = 0.5): return 0
@property def objective_name(self): return self.__class__.__name__
[docs]class HingeLoss: """ **Hinge Loss** Hinge Loss also known as SVM Loss is used "maximum-margin" classification, most notably for support vector machines (SVMs) References: [1] Hinge loss * [Wikipedia Article] https://en.wikipedia.org/wiki/Hinge_loss """
[docs] def loss(self, predictions, targets, np_type): """ Applies the Hinge-Loss to Loss prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of Hinge-Loss Loss to prediction and targets """ correct_class = predictions[np.arange(predictions.shape[0]), np.argmax(targets, axis = 1)] margins = np.maximum(0, predictions - correct_class[:, np.newaxis] + 1.0) # delta = 1.0 margins[np.arange(predictions.shape[0]), np.argmax(targets, axis = 1)] = 0 return np.mean(np.sum(margins, axis = 0))
[docs] def derivative(self, predictions, targets, np_type): """ Applies the Hinge-Loss Derivative to prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of Hinge-Loss Derivative to prediction and targets """ correct_class = predictions[np.arange(predictions.shape[0]), np.argmax(targets, axis = 1)] binary = np.maximum(0, predictions - correct_class[:, np.newaxis] + 1.0) # delta = 1.0 binary[binary > 0] = 1 incorrect_class = np.sum(binary, axis = 1) binary[np.arange(predictions.shape[0]), np.argmax(targets, axis = 1)] = -incorrect_class return binary
[docs] def accuracy(self, predictions, targets, threshold = 0.5): """ Calculates the Hinge-Loss Accuracy Score given prediction and targets Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.float32: the output of Hinge-Loss Accuracy Score """ return np.mean(np.argmax(predictions, axis = 1) == np.argmax(targets, axis = 1))
@property def objective_name(self): return self.__class__.__name__
[docs]class BinaryCrossEntropy(Objective): """ **Binary Cross Entropy** Binary CrossEntropy measures the performance of a classification model whose output is a probability value between 0 & 1. 'Binary' is meant for discrete classification tasks in which the classes are independent and not mutually exclusive. Targets here could be either 0 or 1 scalar References: [1] Cross Entropy * [Wikipedia Article] https://en.wikipedia.org/wiki/Cross_entropy """
[docs] def loss(self, predictions, targets, np_type): """ Applies the BinaryCrossEntropy Loss to prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of BinaryCrossEntropy Loss to prediction and targets """ clipped_predictions, _ = super(BinaryCrossEntropy, self).clip(predictions) return np.mean(-np.sum(np.multiply(targets, np.log(clipped_predictions)) + np.multiply((1 - targets), np.log(1 - clipped_predictions)), axis = 1))
[docs] def derivative(self, predictions, targets, np_type): """ Applies the BinaryCrossEntropy Derivative to prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of BinaryCrossEntropy Derivative to prediction and targets """ clipped_predictions, clipped_divisor = super(BinaryCrossEntropy, self).clip(predictions) return np.true_divide((clipped_predictions - targets), clipped_divisor)
[docs] def accuracy(self, predictions, targets, threshold = 0.5): """ Calculates the BinaryCrossEntropy Accuracy Score given prediction and targets Args: predictions (numpy.array) : the predictions numpy array targets (numpy.array) : the targets numpy array threshold (numpy.float32): the threshold value Returns: numpy.float32: the output of BinaryCrossEntropy Accuracy Score """ return 1 - np.true_divide(np.count_nonzero((predictions > threshold) == targets), float(targets.size))
@property def objective_name(self): return self.__class__.__name__
[docs]class CategoricalCrossEntropy(Objective): """ **Categorical Cross Entropy** Categorical Cross Entropy measures the performance of a classification model whose output is a probability value between 0 and 1. 'Categorical' is meant for discrete classification tasks in which the classes are mutually exclusive. References: [1] Cross Entropy * [Wikipedia Article] https://en.wikipedia.org/wiki/Cross_entropy """
[docs] def loss(self, predictions, targets, np_type): """ Applies the CategoricalCrossEntropy Loss to prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of CategoricalCrossEntropy Loss to prediction and targets """ clipped_predictions, _ = super(CategoricalCrossEntropy, self).clip(predictions) return np.mean(-np.sum(np.multiply(targets, np.log(clipped_predictions)), axis = 1))
[docs] def derivative(self, predictions, targets, np_type): """ Applies the CategoricalCrossEntropy Derivative to prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of CategoricalCrossEntropy Derivative to prediction and targets """ clipped_predictions, _ = super(CategoricalCrossEntropy, self).clip(predictions) return clipped_predictions - targets
[docs] def accuracy(self, predictions, targets): """ Calculates the CategoricalCrossEntropy Accuracy Score given prediction and targets Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.float32: the output of CategoricalCrossEntropy Accuracy Score """ return np.mean(np.argmax(predictions, axis = 1) == np.argmax(targets, axis = 1))
@property def objective_name(self): return self.__class__.__name__
[docs]class KLDivergence(Objective): """ **KL Divergence** KullbackÔÇôLeibler divergence (also called relative entropy) is a measure of divergence between two probability distributions. """
[docs] def loss(self, predictions, targets, np_type): """ Applies the KLDivergence Loss to prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of KLDivergence Loss to prediction and targets """ targets = super(KLDivergence, self).add_fuzz_factor(targets) predictions = super(KLDivergence, self).add_fuzz_factor(predictions) return np.sum(np.multiply(targets, np.log(np.true_divide(targets, predictions))), axis = 1)
[docs] def derivative(self, predictions, targets, np_type): """ Applies the KLDivergence Derivative to prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of KLDivergence Derivative to prediction and targets """ targets = super(KLDivergence, self).add_fuzz_factor(targets) predictions = super(KLDivergence, self).add_fuzz_factor(predictions) d_log_diff = np.multiply((predictions - targets), (np.log(np.true_divide(targets, predictions)))) return np.multiply((1 + np.log(np.true_divide(targets, predictions))), d_log_diff)
[docs] def accuracy(self, predictions, targets): """ Calculates the KLDivergence Accuracy Score given prediction and targets Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.float32: the output of KLDivergence Accuracy Score """ return np.mean(np.argmax(predictions, axis = 1) == np.argmax(targets, axis = 1))
@property def objective_name(self): return self.__class__.__name__
[docs]class HuberLoss(Objective): """ **Huber Loss** Huber Loss: is a loss function used in robust regression where it is found to be less sensitive to outliers in data than the squared error loss. References: [1] Huber Loss * [Wikipedia Article] https://en.wikipedia.org/wiki/Huber_loss [2] Huber loss * [Wikivisually Article] https://wikivisually.com/wiki/Huber_loss """
[docs] def loss(self, predictions, targets, np_type, delta = 1.): """ Applies the HuberLoss Loss to prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of KLDivergence Loss to prediction and targets """ error, abs_error = super(HuberLoss, self).error(predictions, targets) return np.sum(np.where(abs_error < delta, 0.5 * (np.square(error)), delta * abs_error - 0.5 * (mt.pow(delta, 2))))
[docs] def derivative(self, predictions, targets, np_type, delta = 1.): """ Applies the HuberLoss Derivative to prediction and targets provided Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.array: the output of KLDivergence Derivative to prediction and targets """ error, abs_error = super(HuberLoss, self).error(predictions, targets) return np.sum(np.where(abs_error > delta, delta * np.sign(error), error))
[docs] def accuracy(self, predictions, targets): """ Calculates the HuberLoss Accuracy Score given prediction and targets Args: predictions (numpy.array): the predictions numpy array targets (numpy.array): the targets numpy array Returns: numpy.float32: the output of KLDivergence Accuracy Score """ return np.mean(predictions - targets)
@property def objective_name(self): return self.__class__.__name__
[docs]class ObjectiveFunction: _functions = { 'svm' : HingeLoss, 'hinge' : HingeLoss, 'huber' : HuberLoss, 'huber_loss' : HuberLoss, 'hinge_loss' : HingeLoss, 'kld' : KLDivergence, 'mse' : MeanSquaredError, 'bce' : BinaryCrossEntropy, 'cce' : CategoricalCrossEntropy, 'mean_squared_error' : MeanSquaredError, 'hellinger_distance' : HellingerDistance, 'binary_crossentropy' : BinaryCrossEntropy, 'kullback_leibler_divergence' : KLDivergence, 'categorical_crossentropy' : CategoricalCrossEntropy } def __init__(self, name): if name not in self._functions.keys(): raise Exception('Objective function must be either one of the following: {}.'.format(', '.join(self._functions.keys()))) self.objective_func = self._functions[name]() @property def name(self): return self.objective_func.objective_name
[docs] def forward(self, predictions, targets, np_type = np.float32): return self.objective_func.loss(predictions, targets, np_type)
[docs] def backward(self, predictions, targets, np_type = np.float32): return self.objective_func.derivative(predictions, targets, np_type)
[docs] def accuracy(self, predictions, targets): return self.objective_func.accuracy(predictions, targets)