import warnings import numpy as np import pandas as pd from sklearn.inspection import \ permutation_importance as sk_permutation_importance from sklearn.metrics import auc as sklearn_auc from sklearn.metrics import confusion_matrix as sklearn_confusion_matrix from sklearn.metrics import \ precision_recall_curve as sklearn_precision_recall_curve from sklearn.metrics import roc_curve as sklearn_roc_curve from sklearn.preprocessing import LabelBinarizer from sklearn.utils.multiclass import unique_labels from evalml.objectives import get_objective from evalml.utils import import_or_raise [docs]def precision_recall_curve(y_true, y_pred_proba): """ Given labels and binary classifier predicted probabilities, compute and return the data representing a precision-recall curve. Arguments: y_true (pd.Series or np.array): true binary labels. y_pred_proba (pd.Series or np.array): predictions from a binary classifier, before thresholding has been applied. Note this should be the predicted probability for the "true" label. Returns: list: Dictionary containing metrics used to generate a precision-recall plot, with the following keys: * `precision`: Precision values. * `recall`: Recall values. * `thresholds`: Threshold values used to produce the precision and recall. * `auc_score`: The area under the ROC curve. """ precision, recall, thresholds = sklearn_precision_recall_curve(y_true, y_pred_proba) auc_score = sklearn_auc(recall, precision) return {'precision': precision, 'recall': recall, 'thresholds': thresholds, 'auc_score': auc_score} [docs]def graph_precision_recall_curve(y_true, y_pred_proba, title_addition=None): """Generate and display a precision-recall plot. Arguments: y_true (pd.Series or np.array): true binary labels. y_pred_proba (pd.Series or np.array): predictions from a binary classifier, before thresholding has been applied. Note this should be the predicted probability for the "true" label. title_addition (str or None): if not None, append to plot title. Default None. Returns: plotly.Figure representing the precision-recall plot generated """ _go = import_or_raise("plotly.graph_objects", error_msg="Cannot find dependency plotly.graph_objects") if isinstance(y_true, pd.Series): y_true = y_true.to_numpy() if isinstance(y_pred_proba, (pd.Series, pd.DataFrame)): y_pred_proba = y_pred_proba.to_numpy() precision_recall_curve_data = precision_recall_curve(y_true, y_pred_proba) title = 'Precision-Recall{}'.format('' if title_addition is None else (' ' + title_addition)) layout = _go.Layout(title={'text': title}, xaxis={'title': 'Recall', 'range': [-0.05, 1.05]}, yaxis={'title': 'Precision', 'range': [-0.05, 1.05]}) data = [] data.append(_go.Scatter(x=precision_recall_curve_data['recall'], y=precision_recall_curve_data['precision'], name='Precision-Recall (AUC {:06f})'.format(precision_recall_curve_data['auc_score']), line=dict(width=3))) return _go.Figure(layout=layout, data=data) [docs]def roc_curve(y_true, y_pred_proba): """ Given labels and classifier predicted probabilities, compute and return the data representing a Receiver Operating Characteristic (ROC) curve. Arguments: y_true (pd.Series or np.array): true labels. y_pred_proba (pd.Series or np.array): predictions from a classifier, before thresholding has been applied. Note that 1 dimensional input is expected. Returns: dict: Dictionary containing metrics used to generate an ROC plot, with the following keys: * `fpr_rate`: False positive rate. * `tpr_rate`: True positive rate. * `threshold`: Threshold values used to produce each pair of true/false positive rates. * `auc_score`: The area under the ROC curve. """ fpr_rates, tpr_rates, thresholds = sklearn_roc_curve(y_true, y_pred_proba) auc_score = sklearn_auc(fpr_rates, tpr_rates) return {'fpr_rates': fpr_rates, 'tpr_rates': tpr_rates, 'thresholds': thresholds, 'auc_score': auc_score} [docs]def graph_roc_curve(y_true, y_pred_proba, custom_class_names=None, title_addition=None): """Generate and display a Receiver Operating Characteristic (ROC) plot. Arguments: y_true (pd.Series or np.array): true labels. y_pred_proba (pd.Series or np.array): predictions from a classifier, before thresholding has been applied. Note this should a one dimensional array with the predicted probability for the "true" label in the binary case. custom_class_labels (list or None): if not None, custom labels for classes. Default None. title_addition (str or None): if not None, append to plot title. Default None. Returns: plotly.Figure representing the ROC plot generated """ _go = import_or_raise("plotly.graph_objects", error_msg="Cannot find dependency plotly.graph_objects") if isinstance(y_true, pd.Series): y_true = y_true.to_numpy() if isinstance(y_pred_proba, (pd.Series, pd.DataFrame)): y_pred_proba = y_pred_proba.to_numpy() if y_pred_proba.ndim == 1: y_pred_proba = y_pred_proba.reshape(-1, 1) nan_indices = np.logical_or(np.isnan(y_true), np.isnan(y_pred_proba).any(axis=1)) y_true = y_true[~nan_indices] y_pred_proba = y_pred_proba[~nan_indices] lb = LabelBinarizer() lb.fit(np.unique(y_true)) y_one_hot_true = lb.transform(y_true) n_classes = y_one_hot_true.shape[1] if custom_class_names and len(custom_class_names) != n_classes: raise ValueError('Number of custom class names does not match number of classes') title = 'Receiver Operating Characteristic{}'.format('' if title_addition is None else (' ' + title_addition)) layout = _go.Layout(title={'text': title}, xaxis={'title': 'False Positive Rate', 'range': [-0.05, 1.05]}, yaxis={'title': 'True Positive Rate', 'range': [-0.05, 1.05]}) data = [] for i in range(n_classes): roc_curve_data = roc_curve(y_one_hot_true[:, i], y_pred_proba[:, i]) data.append(_go.Scatter(x=roc_curve_data['fpr_rates'], y=roc_curve_data['tpr_rates'], name='Class {name} (AUC {:06f})' .format(roc_curve_data['auc_score'], name=i + 1 if custom_class_names is None else custom_class_names[i]), line=dict(width=3))) data.append(_go.Scatter(x=[0, 1], y=[0, 1], name='Trivial Model (AUC 0.5)', line=dict(dash='dash'))) return _go.Figure(layout=layout, data=data) [docs]def confusion_matrix(y_true, y_predicted, normalize_method='true'): """Confusion matrix for binary and multiclass classification. Arguments: y_true (pd.Series or np.array): true binary labels. y_pred (pd.Series or np.array): predictions from a binary classifier. normalize_method ({'true', 'pred', 'all'}): Normalization method. Supported options are: 'true' to normalize by row, 'pred' to normalize by column, or 'all' to normalize by all values. Defaults to 'true'. Returns: np.array: confusion matrix """ if isinstance(y_true, pd.Series): y_true = y_true.to_numpy() if isinstance(y_predicted, pd.Series): y_predicted = y_predicted.to_numpy() labels = unique_labels(y_true, y_predicted) conf_mat = sklearn_confusion_matrix(y_true, y_predicted) conf_mat = pd.DataFrame(conf_mat, columns=labels) if normalize_method is not None: return normalize_confusion_matrix(conf_mat, normalize_method=normalize_method) return conf_mat [docs]def normalize_confusion_matrix(conf_mat, normalize_method='true'): """Normalizes a confusion matrix. Arguments: conf_mat (pd.DataFrame or np.array): confusion matrix to normalize. normalize_method ({'true', 'pred', 'all'}): Normalization method. Supported options are: 'true' to normalize by row, 'pred' to normalize by column, or 'all' to normalize by all values. Defaults to 'true'. Returns: A normalized version of the input confusion matrix. """ with warnings.catch_warnings(record=True) as w: if normalize_method == 'true': conf_mat = conf_mat.astype('float') / conf_mat.sum(axis=1)[:, np.newaxis] elif normalize_method == 'pred': conf_mat = conf_mat.astype('float') / conf_mat.sum(axis=0) elif normalize_method == 'all': conf_mat = conf_mat.astype('float') / conf_mat.sum().sum() else: raise ValueError('Invalid value provided for "normalize_method": %s'.format(normalize_method)) if w and "invalid value encountered in" in str(w[0].message): raise ValueError("Sum of given axis is 0 and normalization is not possible. Please select another option.") return conf_mat [docs]def graph_confusion_matrix(y_true, y_pred, normalize_method='true', title_addition=None): """Generate and display a confusion matrix plot. If `normalize_method` is set, hover text will show raw count, otherwise hover text will show count normalized with method 'true'. Arguments: y_true (pd.Series or np.array): true binary labels. y_pred (pd.Series or np.array): predictions from a binary classifier. normalize_method ({'true', 'pred', 'all'}): Normalization method. Supported options are: 'true' to normalize by row, 'pred' to normalize by column, or 'all' to normalize by all values. Defaults to 'true'. title_addition (str or None): if not None, append to plot title. Default None. Returns: plotly.Figure representing the confusion matrix plot generated """ _go = import_or_raise("plotly.graph_objects", error_msg="Cannot find dependency plotly.graph_objects") if isinstance(y_true, pd.Series): y_true = y_true.to_numpy() if isinstance(y_pred, pd.Series): y_pred = y_pred.to_numpy() conf_mat = confusion_matrix(y_true, y_pred, normalize_method=None) conf_mat_normalized = confusion_matrix(y_true, y_pred, normalize_method=normalize_method or 'true') labels = conf_mat.columns title = 'Confusion matrix{}{}'.format( '' if title_addition is None else (' ' + title_addition), '' if normalize_method is None else (', normalized using method "' + normalize_method + '"')) z_data, custom_data = (conf_mat, conf_mat_normalized) if normalize_method is None else (conf_mat_normalized, conf_mat) primary_heading, secondary_heading = ('Raw', 'Normalized') if normalize_method is None else ('Normalized', 'Raw') hover_text = '<br><b>' + primary_heading + ' Count</b>: %{z}<br><b>' + secondary_heading + ' Count</b>: %{customdata} <br>' # the "<extra> tags at the end are necessary to remove unwanted trace info hover_template = '<b>True</b>: %{y}<br><b>Predicted</b>: %{x}' + hover_text + '<extra></extra>' layout = _go.Layout(title={'text': title}, xaxis={'title': 'Predicted Label', 'type': 'category', 'tickvals': labels}, yaxis={'title': 'True Label', 'type': 'category', 'tickvals': labels}) fig = _go.Figure(data=_go.Heatmap(x=labels, y=labels, z=z_data, customdata=custom_data, hovertemplate=hover_template, colorscale='Blues'), layout=layout) # plotly Heatmap y axis defaults to the reverse of what we want: https://community.plotly.com/t/heatmap-y-axis-is-reversed-by-default-going-against-standard-convention-for-matrices/32180 fig.update_yaxes(autorange="reversed") return fig [docs]def calculate_permutation_importance(pipeline, X, y, objective, n_repeats=5, n_jobs=None, random_state=0): """Calculates permutation importance for features. Arguments: pipeline (PipelineBase or subclass): fitted pipeline X (pd.DataFrame): the input data used to score and compute permutation importance y (pd.Series): the target labels objective (str, ObjectiveBase): objective to score on n_repeats (int): Number of times to permute a feature. Defaults to 5. n_jobs (int or None): Non-negative integer describing level of parallelism used for pipelines. None and 1 are equivalent. If set to -1, all CPUs are used. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used. random_state (int, np.random.RandomState): The random seed/state. Defaults to 0. Returns: Mean feature importance scores over 5 shuffles. """ objective = get_objective(objective) if objective.problem_type != pipeline.problem_type: raise ValueError(f"Given objective '{objective.name}' cannot be used with '{pipeline.name}'") def scorer(pipeline, X, y): scores = pipeline.score(X, y, objectives=[objective]) return scores[objective.name] if objective.greater_is_better else -scores[objective.name] perm_importance = sk_permutation_importance(pipeline, X, y, n_repeats=n_repeats, scoring=scorer, n_jobs=n_jobs, random_state=random_state) mean_perm_importance = perm_importance["importances_mean"] if not isinstance(X, pd.DataFrame): X = pd.DataFrame(X) feature_names = list(X.columns) mean_perm_importance = list(zip(feature_names, mean_perm_importance)) mean_perm_importance.sort(key=lambda x: x[1], reverse=True) return pd.DataFrame(mean_perm_importance, columns=["feature", "importance"]) [docs]def graph_permutation_importance(pipeline, X, y, objective, show_all_features=False): """Generate a bar graph of the pipeline's permutation importance. Arguments: pipeline (PipelineBase or subclass): Fitted pipeline X (pd.DataFrame): The input data used to score and compute permutation importance y (pd.Series): The target labels objective (str, ObjectiveBase): Objective to score on show_all_features (bool, optional) : If True, graph features with a permutation importance value of zero. Defaults to False. Returns: plotly.Figure, a bar graph showing features and their respective permutation importance. """ go = import_or_raise("plotly.graph_objects", error_msg="Cannot find dependency plotly.graph_objects") perm_importance = calculate_permutation_importance(pipeline, X, y, objective) perm_importance['importance'] = perm_importance['importance'] if not show_all_features: # Remove features with close to zero importance perm_importance = perm_importance[abs(perm_importance['importance']) >= 1e-3] # List is reversed to go from ascending order to descending order perm_importance = perm_importance.iloc[::-1] title = "Permutation Importance" subtitle = "The relative importance of each input feature's "\ "overall influence on the pipelines' predictions, computed using "\ "the permutation importance algorithm." data = [go.Bar(x=perm_importance['importance'], y=perm_importance['feature'], orientation='h' )] layout = { 'title': '{0}<br><sub>{1}</sub>'.format(title, subtitle), 'height': 800, 'xaxis_title': 'Permutation Importance', 'yaxis_title': 'Feature', 'yaxis': { 'type': 'category' } } fig = go.Figure(data=data, layout=layout) return fig