Source code for evalml.model_understanding.permutation_importance

"""Permutation importance methods."""

import numpy as np
import pandas as pd
from joblib import Parallel, delayed

from evalml.objectives.utils import get_objective
from evalml.problem_types import is_classification
from evalml.problem_types.utils import is_regression
from evalml.utils import import_or_raise, infer_feature_types, jupyter_check


[docs]def calculate_permutation_importance( pipeline, X, y, objective, n_repeats=5, n_jobs=None, random_seed=0, ): """Calculates permutation importance for features. Args: pipeline (PipelineBase or subclass): Fitted pipeline. X (pd.DataFrame): The input data used to score and compute permutation importance. y (pd.Series): The target data. objective (str, ObjectiveBase): Objective to score on. n_repeats (int): Number of times to permute a feature. Defaults to 5. n_jobs (int or None): Non-negative integer describing level of parallelism used for pipelines. None and 1 are equivalent. If set to -1, all CPUs are used. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used. Defaults to None. random_seed (int): Seed for the random number generator. Defaults to 0. Returns: pd.DataFrame: Mean feature importance scores over a number of shuffles. Raises: ValueError: If objective cannot be used with the given pipeline. """ X = infer_feature_types(X) y = infer_feature_types(y) objective = get_objective(objective, return_instance=True) if not objective.is_defined_for_problem_type(pipeline.problem_type): raise ValueError( f"Given objective '{objective.name}' cannot be used with '{pipeline.name}'", ) if pipeline._supports_fast_permutation_importance: precomputed_features = pipeline.transform_all_but_final(X, y) perm_importance = _fast_permutation_importance( pipeline, X, y, objective, precomputed_features, n_repeats=n_repeats, n_jobs=n_jobs, random_seed=random_seed, ) else: perm_importance = _slow_permutation_importance( pipeline, X, y, objective, n_repeats=n_repeats, n_jobs=n_jobs, random_seed=random_seed, ) mean_perm_importance = perm_importance["importances_mean"] feature_names = list(X.columns) mean_perm_importance = list(zip(feature_names, mean_perm_importance)) mean_perm_importance.sort(key=lambda x: x[1], reverse=True) return pd.DataFrame(mean_perm_importance, columns=["feature", "importance"])
[docs]def graph_permutation_importance(pipeline, X, y, objective, importance_threshold=0): """Generate a bar graph of the pipeline's permutation importance. Args: pipeline (PipelineBase or subclass): Fitted pipeline. X (pd.DataFrame): The input data used to score and compute permutation importance. y (pd.Series): The target data. objective (str, ObjectiveBase): Objective to score on. importance_threshold (float, optional): If provided, graph features with a permutation importance whose absolute value is larger than importance_threshold. Defaults to 0. Returns: plotly.Figure, a bar graph showing features and their respective permutation importance. Raises: ValueError: If importance_threshold is not greater than or equal to 0. """ go = import_or_raise( "plotly.graph_objects", error_msg="Cannot find dependency plotly.graph_objects", ) if jupyter_check(): import_or_raise("ipywidgets", warning=True) perm_importance = calculate_permutation_importance(pipeline, X, y, objective) perm_importance["importance"] = perm_importance["importance"] if importance_threshold < 0: raise ValueError( f"Provided importance threshold of {importance_threshold} must be greater than or equal to 0", ) # Remove features with close to zero importance perm_importance = perm_importance[ abs(perm_importance["importance"]) >= importance_threshold ] # List is reversed to go from ascending order to descending order perm_importance = perm_importance.iloc[::-1] title = "Permutation Importance" subtitle = ( "The relative importance of each input feature's " "overall influence on the pipelines' predictions, computed using " "the permutation importance algorithm." ) data = [ go.Bar( x=perm_importance["importance"], y=perm_importance["feature"], orientation="h", ), ] layout = { "title": "{0}<br><sub>{1}</sub>".format(title, subtitle), "height": 800, "xaxis_title": "Permutation Importance", "yaxis_title": "Feature", "yaxis": {"type": "category"}, } fig = go.Figure(data=data, layout=layout) return fig
[docs]def calculate_permutation_importance_one_column( pipeline, X, y, col_name, objective, n_repeats=5, fast=True, precomputed_features=None, random_seed=0, ): """Calculates permutation importance for one column in the original dataframe. Args: pipeline (PipelineBase or subclass): Fitted pipeline. X (pd.DataFrame): The input data used to score and compute permutation importance. y (pd.Series): The target data. col_name (str, int): The column in X to calculate permutation importance for. objective (str, ObjectiveBase): Objective to score on. n_repeats (int): Number of times to permute a feature. Defaults to 5. fast (bool): Whether to use the fast method of calculating the permutation importance or not. Defaults to True. precomputed_features (pd.DataFrame): Precomputed features necessary to calculate permutation importance using the fast method. Defaults to None. random_seed (int): Seed for the random number generator. Defaults to 0. Returns: float: Mean feature importance scores over a number of shuffles. Raises: ValueError: If pipeline does not support fast permutation importance calculation. ValueError: If precomputed_features is None. """ X = infer_feature_types(X) y = infer_feature_types(y) objective = get_objective(objective, return_instance=True) if fast: if not pipeline._supports_fast_permutation_importance: raise ValueError( "Pipeline does not support fast permutation importance calculation", ) if precomputed_features is None: raise ValueError( "Fast method of calculating permutation importance requires precomputed_features", ) permutation_importance = _fast_permutation_importance( pipeline, X, y, objective, precomputed_features, col_name=col_name, n_repeats=n_repeats, random_seed=random_seed, ) else: permutation_importance = _slow_permutation_importance( pipeline, X, y, objective, col_name=col_name, n_repeats=n_repeats, random_seed=random_seed, ) return permutation_importance["importances_mean"]
def _fast_permutation_importance( pipeline, X, y, objective, precomputed_features, col_name=None, n_repeats=5, n_jobs=None, random_seed=None, ): """Calculate permutation importance faster by only computing the estimator features once. Only used for pipelines that support this optimization. """ if is_classification(pipeline.problem_type): y = pipeline._encode_targets(y) baseline_score = _fast_scorer(pipeline, precomputed_features, X, y, objective) if col_name is None: scores = Parallel(n_jobs=n_jobs)( delayed(_calculate_permutation_scores_fast)( pipeline, precomputed_features, y, objective, col_name, random_seed, n_repeats, _fast_scorer, baseline_score, ) for col_name in X.columns ) importances = baseline_score - np.array(scores) return {"importances_mean": np.mean(importances, axis=1)} else: scores = _calculate_permutation_scores_fast( pipeline, precomputed_features, y, objective, col_name, random_seed, n_repeats, _fast_scorer, baseline_score, ) importances = baseline_score - np.array(scores) importances_mean = ( np.mean(importances, axis=1) if col_name is None else np.mean(importances) ) return {"importances_mean": importances_mean} def _calculate_permutation_scores_fast( pipeline, precomputed_features, y, objective, col_name, random_seed, n_repeats, scorer, baseline_score, ): """Calculate the permutation score when `col_name` is permuted.""" random_state = np.random.RandomState(random_seed) scores = np.zeros(n_repeats) # If column is not in the features or provenance, assume the column was dropped if ( col_name not in precomputed_features.columns and col_name not in pipeline._get_feature_provenance() ): return scores + baseline_score if col_name in precomputed_features.columns: col_idx = precomputed_features.columns.get_loc(col_name) else: col_idx = [ precomputed_features.columns.get_loc(col) for col in pipeline._get_feature_provenance()[col_name] ] return _shuffle_and_score_helper( pipeline, precomputed_features, y, objective, col_idx, n_repeats, scorer, random_state, is_fast=True, ) def _slow_permutation_importance( pipeline, X, y, objective, col_name=None, n_repeats=5, n_jobs=None, random_seed=None, ): """If `col_name` is not None, calculates permutation importance for only the column with that name. Otherwise, calculates the permutation importance for all columns in the input dataframe. """ baseline_score = _slow_scorer(pipeline, X, y, objective) if col_name is None: scores = Parallel(n_jobs=n_jobs)( delayed(_calculate_permutation_scores_slow)( pipeline, X, y, col_idx, objective, _slow_scorer, n_repeats, random_seed, ) for col_idx in range(X.shape[1]) ) else: baseline_score = _slow_scorer(pipeline, X, y, objective) scores = _calculate_permutation_scores_slow( pipeline, X, y, col_name, objective, _slow_scorer, n_repeats, random_seed, ) importances = baseline_score - np.array(scores) importances_mean = ( np.mean(importances, axis=1) if col_name is None else np.mean(importances) ) return {"importances_mean": importances_mean} def _calculate_permutation_scores_slow( estimator, X, y, col_name, objective, scorer, n_repeats, random_seed, ): """Calculate score when `col_idx` is permuted.""" random_state = np.random.RandomState(random_seed) col_idx = col_name if col_name in X.columns: col_idx = X.columns.get_loc(col_name) return _shuffle_and_score_helper( estimator, X, y, objective, col_idx, n_repeats, scorer, random_state, is_fast=False, ) def _shuffle_and_score_helper( pipeline, X_features, y, objective, col_idx, n_repeats, scorer, random_state, is_fast=True, ): scores = np.zeros(n_repeats) # This is what sk_permutation_importance does. Useful for thread safety X_permuted = X_features.copy() shuffling_idx = np.arange(X_features.shape[0]) for n_round in range(n_repeats): random_state.shuffle(shuffling_idx) col = X_permuted.iloc[shuffling_idx, col_idx] col.index = X_permuted.index X_permuted.iloc[:, col_idx] = col X_permuted.ww.init(schema=X_features.ww.schema) if is_fast: feature_score = scorer(pipeline, X_permuted, X_features, y, objective) else: feature_score = scorer(pipeline, X_permuted, y, objective) scores[n_round] = feature_score return scores def _slow_scorer(pipeline, X, y, objective): scores = pipeline.score(X, y, objectives=[objective]) return ( scores[objective.name] if objective.greater_is_better else -scores[objective.name] ) def _fast_scorer(pipeline, features, X, y, objective): if objective.score_needs_proba: preds = pipeline.estimator.predict_proba(features) else: preds = pipeline.estimator.predict(features) if is_regression(pipeline.problem_type): preds = pipeline.inverse_transform(preds) score = pipeline._score(X, y, preds, objective) return score if objective.greater_is_better else -score