Source code for monte_carlo_analysis.uncertainty_metrics.MutualInformationMultipleDistribution

"""
**Author** : Robin Camarasa

**Institution** : Erasmus Medical Center

**Position** : PhD student

**Contact** : r.camarasa@erasmusmc.nl

**Date** : 2020-10-14

**Project** : monte_carlo_analysis

**Implement class MutualInformationMultipleDistributions**

"""
from monte_carlo_analysis.uncertainty_metrics import MultipleDistributionsUncertaintyMetric
import numpy as np
from numba import jit


[docs]class MutualInformationMultipleDistributions(MultipleDistributionsUncertaintyMetric):
    """Implement MutualInformationMultipleDistributions. The formula applied to the family of distributions :math:`(q_{c'}(y_j|x))_{1 \leq c \leq C}` is:

        .. math::
            M^m((q_c(y_{j}|x))_{1 \leq c \leq C}) = -\sum_{c=1}^C \mathbb{E}(q_c(y_j |x)) log(\mathbb{E}(q_c(y_j |x)) + T^{-1} \sum_{t=1}^T \sum_{c=1}^C  p_c(y_j | x, w=w_t) log(p_c(y_j | x, w=w_t))
    """
    def __init__(self):
        super(MutualInformationMultipleDistributions, self)
        self.transformation = self.get_transformation()

[docs]    def get_transformation(self) -> callable:
        """Define the variance transformation applied to the family of distributions

        :return: Transformation to apply to the family of distributions
        """
        @jit
        def transformation(distributions: np.array) -> float:
            # Compute entropy
            mean_prediction_per_class = np.mean(distributions, axis=0)

            # Generate the log_mean_prediction_per_class taking into account
            # the case of zero means
            log_mean_prediction_per_class = mean_prediction_per_class[:]
            log_mean_prediction_per_class[
                np.where(log_mean_prediction_per_class == 0)
            ] = 1
            log_mean_prediction_per_class = np.log(log_mean_prediction_per_class)
            entropy = - np.sum(
                log_mean_prediction_per_class * mean_prediction_per_class
            )

            # Compute conditional entropy

            # Generate the log prediction taking into account
            # the case of zero means
            log_prediction = distributions[:]
            log_prediction[
                np.where(distributions == 0)
            ] = 1
            log_prediction = np.log(log_prediction)
            conditional_entropy = -1.0/distributions.shape[0] * np.sum(
                log_prediction * distributions
            )

            return entropy - conditional_entropy
        return transformation