Source code for evalml.objectives.utils

"""Utility methods for EvalML objectives."""
from evalml import objectives
from evalml.exceptions import ObjectiveCreationError, ObjectiveNotFoundError
from evalml.objectives.objective_base import ObjectiveBase
from evalml.problem_types import handle_problem_types
from evalml.utils.gen_utils import _get_subclasses


[docs]def get_non_core_objectives(): """Get non-core objective classes. Non-core objectives are objectives that are domain-specific. Users typically need to configure these objectives before using them in AutoMLSearch. Returns: List of ObjectiveBase classes """ return [ objectives.CostBenefitMatrix, objectives.FraudCost, objectives.LeadScoring, objectives.Recall, objectives.RecallMacro, objectives.RecallMicro, objectives.RecallWeighted, objectives.MAPE, objectives.MeanSquaredLogError, objectives.RootMeanSquaredLogError, objectives.SensitivityLowAlert, ]
[docs]def ranking_only_objectives(): """Get ranking-only objective classes. Ranking-only objectives are objectives that are useful for evaluating the performance of a model, but should not be used as an optimization objective during AutoMLSearch for various reasons. Returns: List of ObjectiveBase classes """ return [ objectives.Recall, objectives.RecallMacro, objectives.RecallMicro, objectives.RecallWeighted, objectives.MAPE, objectives.MeanSquaredLogError, objectives.RootMeanSquaredLogError, ]
[docs]def get_optimization_objectives(problem_type): """Get objectives for optimization. Args: problem_type (str/ProblemTypes): Type of problem Returns: List of ObjectiveBase instances """ problem_type = handle_problem_types(problem_type) ranking_only = ranking_only_objectives() objectives = [ obj for obj in get_ranking_objectives(problem_type) if obj.__class__ not in ranking_only ] return objectives
[docs]def get_ranking_objectives(problem_type): """Get objectives for pipeline rankings. Args: problem_type (str/ProblemTypes): Type of problem Returns: List of ObjectiveBase instances """ problem_type = handle_problem_types(problem_type) all_objectives_dict = _all_objectives_dict() objectives = [ obj() for obj in all_objectives_dict.values() if obj.is_defined_for_problem_type(problem_type) and (obj not in get_non_core_objectives() or obj in ranking_only_objectives()) ] return objectives
def _all_objectives_dict(): all_objectives = _get_subclasses(ObjectiveBase) objectives_dict = {} for objective in all_objectives: if "evalml.objectives" not in objective.__module__: continue objectives_dict[objective.name.lower()] = objective return objectives_dict
[docs]def get_all_objective_names(): """Get a list of the names of all objectives. Returns: list (str): Objective names """ all_objectives_dict = _all_objectives_dict() return list(all_objectives_dict.keys())
[docs]def get_core_objective_names(): """Get a list of all valid core objectives. Returns: list[str]: Objective names. """ all_objectives = _all_objectives_dict() non_core = get_non_core_objectives() return [ name for name, class_name in all_objectives.items() if class_name not in non_core ]
[docs]def get_objective(objective, return_instance=False, **kwargs): """Returns the Objective class corresponding to a given objective name. Args: objective (str or ObjectiveBase): Name or instance of the objective class. return_instance (bool): Whether to return an instance of the objective. This only applies if objective is of type str. Note that the instance will be initialized with default arguments. kwargs (Any): Any keyword arguments to pass into the objective. Only used when return_instance=True. Returns: ObjectiveBase if the parameter objective is of type ObjectiveBase. If objective is instead a valid objective name, function will return the class corresponding to that name. If return_instance is True, an instance of that objective will be returned. Raises: TypeError: If objective is None. TypeError: If objective is not a string and not an instance of ObjectiveBase. ObjectiveNotFoundError: If input objective is not a valid objective. ObjectiveCreationError: If objective cannot be created properly. """ if objective is None: raise TypeError("Objective parameter cannot be NoneType") if isinstance(objective, ObjectiveBase): return objective all_objectives_dict = _all_objectives_dict() if not isinstance(objective, str): raise TypeError( "If parameter objective is not a string, it must be an instance of ObjectiveBase!", ) if objective.lower() not in all_objectives_dict: raise ObjectiveNotFoundError( f"{objective} is not a valid Objective! " "Use evalml.objectives.get_all_objective_names()" "to get a list of all valid objective names. ", ) objective_class = all_objectives_dict[objective.lower()] if return_instance: try: return objective_class(**kwargs) except TypeError as e: raise ObjectiveCreationError( f"In get_objective, cannot pass in return_instance=True for {objective} because {str(e)}", ) return objective_class
[docs]def get_core_objectives(problem_type): """Returns all core objective instances associated with the given problem type. Core objectives are designed to work out-of-the-box for any dataset. Args: problem_type (str/ProblemTypes): Type of problem Returns: List of ObjectiveBase instances Examples: >>> for objective in get_core_objectives("regression"): ... print(objective.name) ExpVariance MaxError MedianAE MSE MAE R2 Root Mean Squared Error >>> for objective in get_core_objectives("binary"): ... print(objective.name) MCC Binary Log Loss Binary Gini AUC Precision F1 Balanced Accuracy Binary Accuracy Binary """ problem_type = handle_problem_types(problem_type) all_objectives_dict = _all_objectives_dict() objectives = [ obj() for obj in all_objectives_dict.values() if obj.is_defined_for_problem_type(problem_type) and obj not in get_non_core_objectives() ] return objectives