Source code for evalml.objectives.sensitivity_low_alert

"""Sensitivity at Low Alert Rates objective."""

import logging

import numpy as np

from evalml.objectives.binary_classification_objective import (
    BinaryClassificationObjective,
)

logger = logging.getLogger(__name__)


[docs]class SensitivityLowAlert(BinaryClassificationObjective):
    """Sensitivity at Low Alert Rates.

    Args:
        alert_rate (float): percentage of top scores to classify as high risk.
    """

    name = "Sensitivity at Low Alert Rates"
    greater_is_better = True
    score_needs_proba = False
    perfect_score = 1.0
    is_bounded_like_percentage = True
    expected_range = [0, 1]

    def __init__(self, alert_rate=0.01):
        if (alert_rate > 1) or (alert_rate < 0):
            raise ValueError("Alert rate is outside of valid range [0,1]")

        self.alert_rate = alert_rate

[docs]    def decision_function(self, ypred_proba, **kwargs):
        """Determine if an observation is high risk given an alert rate.

        Args:
            ypred_proba (pd.Series): Predicted probabilities.
            **kwargs: Additional abritrary parameters.

        Returns:
            pd.Series: Whether or not an observation is high risk given an alert rate.
        """
        ypred_proba = self._standardize_input_type(ypred_proba)
        if len(ypred_proba.unique()) == 1:
            logger.debug(
                f"All predicted probabilities have the same value: {ypred_proba.unique()}",
            )

        prob_thresh = np.quantile(ypred_proba, 1 - self.alert_rate)
        if (prob_thresh == 0) or (prob_thresh == 1):
            logger.debug(f"Extreme threshold of {prob_thresh}")

        return ypred_proba.astype(float) >= prob_thresh

[docs]    def objective_function(self, y_true, y_predicted, **kwargs):
        """Calculate sensitivity across all predictions, using the top alert_rate percent of observations as the predicted positive class.

        Args:
            y_true (pd.Series): True labels.
            y_predicted (pd.Series): Predicted labels based on alert_rate.
            **kwargs: Additional abritrary parameters.

        Returns:
            float: sensitivity using the observations with the top scores as the predicted positive class.
        """
        y_true = self._standardize_input_type(y_true)
        y_predicted = self._standardize_input_type(y_predicted)
        self.validate_inputs(y_true, y_predicted)

        tp = y_true & y_predicted
        fn = y_true & (~y_predicted)

        if (tp.sum() + fn.sum()) > 0:
            sensitivity = tp.sum() / (tp.sum() + fn.sum())
        else:
            sensitivity = np.nan

        return sensitivity