"""
Provides public plotting API
=============================
"""
import orion.plotting.backend_plotly as backend
[docs]def lpi(
experiment,
with_evc_tree=True,
model="RandomForestRegressor",
model_kwargs=None,
n_points=20,
n_runs=10,
**kwargs,
):
"""
Make a bar plot to visualize the local parameter importance metric.
For more information on the metric, see original paper at
https://ml.informatik.uni-freiburg.de/papers/18-LION12-CAVE.pdf.
Biedenkapp, André, et al. "Cave: Configuration assessment, visualization and evaluation."
International Conference on Learning and Intelligent Optimization. Springer, Cham, 2018.
Parameters
----------
experiment: ExperimentClient or Experiment
The orion object containing the experiment data
with_evc_tree: bool, optional
Fetch all trials from the EVC tree.
Default: True
model: str
Name of the regression model to use. Can be one of
- AdaBoostRegressor
- BaggingRegressor
- ExtraTreesRegressor
- GradientBoostingRegressor
- RandomForestRegressor (Default)
Arguments for the regressor model.
model_kwargs: dict
Arguments for the regressor model.
n: int
Number of points to compute the variances. Default is 20.
kwargs: dict
All other plotting keyword arguments to be passed to
:plotly:`express.line`.
Returns
-------
plotly.graph_objects.Figure
Raises
------
ValueError
If no experiment is provided or if regressor name is invalid.
"""
return backend.lpi(
experiment,
with_evc_tree=with_evc_tree,
model=model,
model_kwargs=model_kwargs,
n_points=n_points,
n_runs=n_runs,
**kwargs,
)
[docs]def parallel_coordinates(experiment, with_evc_tree=True, order=None, **kwargs):
"""
Make a Parallel Coordinates Plot to visualize the effect of the hyperparameters
on the objective.
Parameters
----------
experiment: ExperimentClient or Experiment
The orion object containing the experiment data
with_evc_tree: bool, optional
Fetch all trials from the EVC tree.
Default: True
order: list of str or None
Indicates the order of columns in the parallel coordinate plot. By default
the columns are sorted alphabetically with the exception of the first column
which is reserved for a fidelity dimension is there is one in the search space.
kwargs: dict
All other plotting keyword arguments to be passed to
:plotly:`express.line`.
Returns
-------
plotly.graph_objects.Figure
Raises
------
ValueError
If no experiment is provided.
"""
return backend.parallel_coordinates(
experiment, with_evc_tree=with_evc_tree, order=order, **kwargs
)
[docs]def partial_dependencies(
experiment,
with_evc_tree=True,
params=None,
smoothing=0.85,
verbose_hover=True,
n_grid_points=10,
n_samples=50,
colorscale="Blues",
model="RandomForestRegressor",
model_kwargs=None,
):
"""
Make contour plots to visualize the search space of each combination of params.
Parameters
----------
experiment: ExperimentClient or Experiment
The orion object containing the experiment data
with_evc_tree: bool, optional
Fetch all trials from the EVC tree.
Default: True
params: list of str, optional
Indicates the parameters to include in the plots. All parameters are included by default.
smoothing: float, optional
Smoothing applied to the countor plot. 0 corresponds to no smoothing. Default is 0.85.
verbose_hover: bool
Indicates whether to display the hyperparameter in hover tooltips. True by default.
colorscale: str, optional
The colorscale used for the contour plots. Supported values depends on the backend.
Default is 'Blues'.
n_grid_points: int, optional
Number of points in the grid to compute partial dependency. Default is 10.
n_samples: int, optional
Number of samples to randomly generate the grid used to compute the partial dependency.
Default is 50.
model: str
Name of the regression model to use. Can be one of
- AdaBoostRegressor
- BaggingRegressor
- ExtraTreesRegressor
- GradientBoostingRegressor
- RandomForestRegressor (Default)
model_kwargs: dict, optional
Arguments for the regressor model.
Returns
-------
plotly.graph_objects.Figure
Raises
------
ValueError
If no experiment is provided.
"""
return backend.partial_dependencies(
experiment,
with_evc_tree=with_evc_tree,
params=params,
smoothing=smoothing,
verbose_hover=verbose_hover,
n_grid_points=n_grid_points,
n_samples=n_samples,
colorscale=colorscale,
model=model,
model_kwargs=model_kwargs,
)
[docs]def rankings(experiments, with_evc_tree=True, order_by="suggested", **kwargs):
"""
Make a plot to visually compare the ranking of different hyper-optimization processes.
The x-axis contain the trials and the y-axis their respective ranking.
4 formats are supported for the experiments:
1. List of experiments. The names of the experiments will be used for the figure labels.
2. Dictionary of experiments. The keys of the dictionary will be used for the figure labels.
3. List of dictionary of experiments. The keys of the dictionary will be used for the figure \
labels. The ranking will be averaged across the dictionaries.
4. Dictionary of list of experiments. The keys of the dictionary will be used for the figure \
labels. A dictionary of experiments will be build grouping the i-th experiments of each \
list to result in a list of dictionary of experiments. Behavior will be same as format 3.
Parameters
----------
experiments: list or dict
List or dictionary of experiments.
with_evc_tree: bool, optional
Fetch all trials from the EVC tree.
Default: True
order_by: str
Indicates how the trials should be ordered. Acceptable options are below.
See attributes of ``Trial`` for more details.
* 'suggested': Sort by trial suggested time (default).
* 'reserved': Sort by trial reserved time.
* 'completed': Sort by trial completed time.
kwargs: dict
All other plotting keyword arguments to be passed to
:plotly:`express.line`.
Returns
-------
plotly.graph_objects.Figure
Raises
------
ValueError
If no experiment is provided or order_by is invalid.
"""
return backend.rankings(
experiments, with_evc_tree=with_evc_tree, order_by=order_by, **kwargs
)
[docs]def parallel_assessment(experiments, with_evc_tree=True):
"""
Make a plot to visualize the performance of running same experiment with different number of
workers.
The x-axis contain the worker number and the y-axis their respective best performance.
3 formats are supported for the experiments:
1. List of experiments. The names of the experiments will be used for the figure labels.
2. Dictionary of experiments. The keys of the dictionary will be used for the figure labels.
3. Dictionary of list of experiments. The keys of the dictionary will be used for the figure \
labels.
Parameters
----------
experiments: list or dict
List or dictionary of experiments.
with_evc_tree: bool, optional
Fetch all trials from the EVC tree.
Default: True
Returns
-------
plotly.graph_objects.Figure
Raises
------
ValueError
If no experiment is provided.
"""
return backend.regrets(
experiments,
with_evc_tree=with_evc_tree,
order_by="objective",
build_frame_fn=backend.build_parallel_frame,
return_var=False,
title="Parallel Assessment",
)
[docs]def durations(experiments, with_evc_tree=True):
"""
Make a plot to visualize the performance of experiment at different time duration.
The x-axis contain relative time duration start from the first trial submitted.
3 formats are supported for the experiments:
1. List of experiments. The names of the experiments will be used for the figure labels.
2. Dictionary of experiments. The keys of the dictionary will be used for the figure labels.
3. Dictionary of list of experiments. The keys of the dictionary will be used for the figure \
labels.
Parameters
----------
experiments: list or dict
List or dictionary of experiments.
with_evc_tree: bool, optional
Fetch all trials from the EVC tree.
Default: True
Returns
-------
plotly.graph_objects.Figure
Raises
------
ValueError
If no experiment is provided.
"""
return backend.regrets(
experiments,
with_evc_tree=with_evc_tree,
order_by="completed",
build_frame_fn=backend.build_durations_frame,
return_var=False,
title="Time to result",
)
[docs]def regret(
experiment, with_evc_tree=True, order_by="suggested", verbose_hover=True, **kwargs
):
"""
Make a plot to visualize the performance of the hyper-optimization process.
The x-axis contain the trials and the y-axis their respective best performance.
Parameters
----------
experiment: ExperimentClient or Experiment
The orion object containing the experiment data
with_evc_tree: bool, optional
Fetch all trials from the EVC tree.
Default: True
order_by: str
Indicates how the trials should be ordered. Acceptable options are below.
See attributes of ``Trial`` for more details.
* 'suggested': Sort by trial suggested time (default).
* 'reserved': Sort by trial reserved time.
* 'completed': Sort by trial completed time.
verbose_hover: bool
Indicates whether to display the hyperparameter in hover tooltips. True by default.
kwargs: dict
All other plotting keyword arguments to be passed to
:plotly:`express.line`.
Returns
-------
plotly.graph_objects.Figure
Raises
------
ValueError
If no experiment is provided.
"""
return backend.regret(
experiment,
with_evc_tree=with_evc_tree,
order_by=order_by,
verbose_hover=verbose_hover,
**kwargs,
)
[docs]def regrets(experiments, with_evc_tree=True, order_by="suggested", **kwargs):
"""
Make a plot to visually compare the performance of different hyper-optimization processes.
The x-axis contain the trials and the y-axis their respective best performance.
3 formats are supported for the experiments:
1. List of experiments. The names of the experiments will be used for the figure labels.
2. Dictionary of experiments. The keys of the dictionary will be used for the figure labels.
3. Dictionary of list of experiments. The keys of the dictionary will be used for the figure \
labels. The objective of the experiments in each list will be averaged at every time step \
(ex: across all first trials for a given list of experiments.)
Parameters
----------
experiments: list or dict
List or dictionary of experiments.
with_evc_tree: bool, optional
Fetch all trials from the EVC tree.
Default: True
order_by: str
Indicates how the trials should be ordered. Acceptable options are below.
See attributes of ``Trial`` for more details.
* 'suggested': Sort by trial suggested time (default).
* 'reserved': Sort by trial reserved time.
* 'completed': Sort by trial completed time.
kwargs: dict
All other plotting keyword arguments to be passed to
:plotly:`express.line`.
Returns
-------
plotly.graph_objects.Figure
Raises
------
ValueError
If no experiment is provided or order_by is invalid.
"""
return backend.regrets(
experiments,
with_evc_tree=with_evc_tree,
order_by=order_by,
build_frame_fn=backend.build_regrets_frame,
return_var=True,
)
PLOT_METHODS = {
"lpi": lpi,
"parallel_coordinates": parallel_coordinates,
"partial_dependencies": partial_dependencies,
"regret": regret,
"regrets": regrets,
"rankings": rankings,
"parallel_assessment": parallel_assessment,
"durations": durations,
}
SINGLE_EXPERIMENT_PLOTS = {
"lpi": lpi,
"parallel_coordinates": parallel_coordinates,
"partial_dependencies": partial_dependencies,
"regret": regret,
}
[docs]class PlotAccessor:
"""
Make plots of ExperimentClient.
Parameters
----------
experiment : ExperimentClient
The object for which the method is called.
Raises
------
ValueError
If no experiment is provided.
"""
def __init__(self, experiment):
if not experiment:
raise ValueError("Parameter 'experiment' is None")
self._experiment = experiment
[docs] def __call__(self, **kwargs):
"""
Make different kinds of plots of ExperimentClient.
Parameters
----------
kind : str
The kind of plot to produce:
- 'regret' : Regret plot (default)
"""
kind = kwargs.pop("kind", "regret")
if kind not in PLOT_METHODS.keys():
raise ValueError(
f"Plot of kind '{kind}' is not one of {list(PLOT_METHODS.keys())}"
)
return PLOT_METHODS[kind](self._experiment, **kwargs)
[docs] def lpi(self, **kwargs):
"""Make a bar plot of the local parameter importance metrics."""
__doc__ = lpi.__doc__
return self(kind="lpi", **kwargs)
[docs] def parallel_coordinates(self, **kwargs):
"""Make a parallel coordinates plot to visualize the performance of the
hyper-optimization process.
"""
__doc__ = parallel_coordinates.__doc__
return self(kind="parallel_coordinates", **kwargs)
[docs] def partial_dependencies(self, **kwargs):
"""Make contour plots to visualize the search space of each combination of params."""
__doc__ = partial_dependencies.__doc__
return self(kind="partial_dependencies", **kwargs)
[docs] def regret(self, **kwargs):
"""Make a plot to visualize the performance of the hyper-optimization process."""
__doc__ = regret.__doc__
return self(kind="regret", **kwargs)