Source code for orion.testing.plotting

"""
Plotting testing support module
===============================

Plotting testing support module providing defaults, functions and mocks.

"""
# pylint: disable=protected-access


[docs]def assert_regret_plot(plot): """Checks the layout of a regret plot""" assert plot.layout.title.text == "Regret for experiment 'experiment-name'" assert plot.layout.xaxis.title.text == "Trials ordered by suggested time" assert plot.layout.yaxis.title.text == "Objective 'loss'" trace1 = plot.data[0] assert trace1.type == "scatter" assert trace1.name == "trials" assert trace1.mode == "markers" assert len(trace1.y) == 1 assert not trace1.x trace2 = plot.data[1] assert trace2.type == "scatter" assert trace2.name == "best-to-date" assert trace2.mode == "lines" assert len(trace2.y) == 1 assert not trace2.x
[docs]def assert_regrets_plot(plot, names, balanced=10, with_avg=False): """Checks the layout of a regrets plot""" assert plot.layout.title.text == "Average Regret" assert plot.layout.xaxis.title.text == "Trials ordered by suggested time" assert plot.layout.yaxis.title.text == "loss" if with_avg: line_plots = plot.data[::2] err_plots = plot.data[1::2] else: line_plots = plot.data assert len(line_plots) == len(names) for name, trace in zip(names, line_plots): assert trace.type == "scatter" assert trace.name == name assert trace.mode == "lines+markers" if balanced: assert len(trace.y) == balanced assert len(trace.x) == balanced if with_avg: assert len(err_plots) == len(names) for name, trace in zip(names, err_plots): assert trace.fill == "toself" assert trace.name == name assert not trace.showlegend if balanced: assert len(trace.y) == balanced * 2 assert len(trace.x) == balanced * 2
def asset_parallel_assessment_plot(plot, names, n_experiments): assert plot.layout.title.text == "Parallel Assessment" assert plot.layout.xaxis.title.text == "Number of workers" assert plot.layout.yaxis.title.text == "loss" line_plots = plot.data assert len(line_plots) == len(names) for name, trace in zip(names, line_plots): assert trace.type == "scatter" assert trace.name == name assert trace.mode == "lines+markers" assert len(trace.x) == n_experiments
[docs]def assert_durations_plot(plot, names): """Checks the layout of a regrets plot""" assert plot.layout.title.text == "Time to result" assert plot.layout.xaxis.title.text == "Experiment duration by second(s)" assert plot.layout.yaxis.title.text == "loss" line_plots = plot.data assert len(line_plots) == len(names) for name, trace in zip([name for name in names], line_plots): assert trace.type == "scatter" assert trace.name == name assert trace.mode == "lines+markers"
[docs]def assert_rankings_plot(plot, names, balanced=10, with_avg=False): """Checks the layout of a rankings plot""" assert plot.layout.title.text == "Average Rankings" assert plot.layout.xaxis.title.text == "Trials ordered by suggested time" assert plot.layout.yaxis.title.text == "Ranking based on loss" if with_avg: line_plots = plot.data[::2] err_plots = plot.data[1::2] else: line_plots = plot.data assert len(line_plots) == len(names) for name, trace in zip(names, line_plots): assert trace.type == "scatter" assert trace.name == name assert trace.mode == "lines" if balanced: assert len(trace.y) == balanced assert len(trace.x) == balanced if with_avg: assert len(err_plots) == len(names) for name, trace in zip(names, err_plots): assert trace.fill == "toself" assert trace.name == name assert not trace.showlegend if balanced: assert len(trace.y) == balanced * 2 assert len(trace.x) == balanced * 2
[docs]def assert_lpi_plot(plot, dims): """Checks the layout of a LPI plot""" assert plot.layout.title.text == "LPI for experiment 'experiment-name'" assert plot.layout.xaxis.title.text == "Hyperparameters" assert plot.layout.yaxis.title.text == "Local Parameter Importance (LPI)" trace = plot.data[0] assert trace["x"] == tuple(dims) assert trace["y"][0] > trace["y"][1] assert len(trace["error_y"]["array"]) == len(dims)
[docs]def assert_partial_dependencies_plot( plot, dims, custom_colorscale=False, smoothing=0.85, n_grid_points=5, n_samples=4, log_dims=None, ): """Checks the layout of a partial dependencies plot""" if not isinstance(n_grid_points, dict): n_grid_points = {dim: n_grid_points for dim in dims} if log_dims is None: log_dims = {} def _ax_label(axis, index): if index == 0: return f"{axis}axis" return f"{axis}axis{index + 1}" def _ax_layout(axis, index): return plot.layout[_ax_label(axis, index)] assert ( plot.layout.title.text == "Partial dependencies for experiment 'experiment-name'" ) assert plot.layout.coloraxis.colorbar.title.text == "Objective" assert plot.layout.yaxis.title.text == "Objective" yrange = _ax_layout("y", 0).range def all_indices(): return { j * len(dims) + i + 1 for i in range(len(dims)) for j in range(i, len(dims)) } def first_column(): return {i * len(dims) + 1 for i in range(len(dims))} def last_row(): return {len(dims) * (len(dims) - 1) + i + 1 for i in range(len(dims))} def diagonal(): return {i * len(dims) + i + 1 for i in range(len(dims))} def assert_axis_log(axis, index, name): axis_type = _ax_layout(axis, index).type if name in log_dims: assert axis_type == "log" else: assert axis_type != "log" def assert_log_x(): x_tested = set() for dim_i, dim_name in enumerate(dims): x_index = dim_i * len(dims) + dim_i for row in range(dim_i, len(dims)): assert_axis_log("x", x_index, dim_name) x_tested.add(x_index + 1) x_index += len(dims) assert x_tested == all_indices() assert_log_x() def assert_shared_y_on_diagonal(): y_tested = set() for dim_i, dim_name in enumerate(dims): # Test shared y axis across the diagonal y_index = dim_i * len(dims) + dim_i assert _ax_layout("y", y_index).range == yrange y_tested.add(y_index + 1) assert y_tested == diagonal() assert_shared_y_on_diagonal() def assert_log_y(): y_tested = set() for dim_i, dim_name in enumerate(dims): # Test shared y axis across the diagonal y_index = dim_i * len(dims) + dim_i # Should not be log assert_axis_log("y", y_index, None) y_tested.add(y_index + 1) y_index = dim_i * len(dims) for column in range(max(dim_i, 0)): assert_axis_log("y", y_index, dim_name) y_tested.add(y_index + 1) y_index += 1 assert y_tested == all_indices() assert_log_y() def assert_x_labels(): x_tested = set() for dim_i, dim_name in enumerate(dims): x_index = len(dims) * (len(dims) - 1) + dim_i assert _ax_layout("x", x_index).title.text == dim_name assert x_tested == last_row() def assert_y_labels(): y_tested = set() for dim_i, dim_name in enumerate(dims): if dim_i > 0: # Test label at left of row y_index = dim_i * len(dims) assert _ax_layout("y", y_index).title.text == dim_name y_tested.add(y_index + 1) else: assert _ax_layout("y", 0).title.text == "Objective" y_tested.add(1) assert y_tested == first_column() assert_y_labels() # assert x_tested == {1, 4, 5, 7, 8, 9} # assert y_tested == {1, 4, 5, 7, 8, 9} if custom_colorscale: assert plot.layout.coloraxis.colorscale[0][1] != "rgb(247,251,255)" else: assert plot.layout.coloraxis.colorscale[0][1] == "rgb(247,251,255)" data = plot.data data_index = 0 for x_i, x_name in enumerate(dims): # Test scatter mean assert data[data_index].mode == "lines" assert data[data_index].showlegend is False assert len(data[data_index].x) == n_grid_points[x_name] assert len(data[data_index].y) == n_grid_points[x_name] data_index += 1 # Test scatter var assert data[data_index].mode == "lines" assert data[data_index].fill == "toself" assert data[data_index].showlegend is False assert len(data[data_index].x) == 2 * n_grid_points[x_name] assert len(data[data_index].y) == 2 * n_grid_points[x_name] data_index += 1 # Test scatter dots assert data[data_index].mode == "markers" assert data[data_index].showlegend is False assert data[data_index].customdata is not None assert data[data_index].hovertemplate is not None assert len(data[data_index].x) == n_samples assert len(data[data_index].y) == n_samples data_index += 1 for y_i in range(x_i + 1, len(dims)): y_name = dims[y_i] # Test contour assert data[data_index].line.smoothing == smoothing # To share colorscale across subplots assert data[data_index].coloraxis == "coloraxis" assert len(data[data_index].x) == n_grid_points[x_name] assert len(data[data_index].y) == n_grid_points[y_name] assert data[data_index].z.shape == ( n_grid_points[y_name], n_grid_points[x_name], ) data_index += 1 # Test scatter assert data[data_index].mode == "markers" assert data[data_index].showlegend is False assert len(data[data_index].x) == n_samples assert len(data[data_index].y) == n_samples data_index += 1 # Make sure we covered all data assert len(data) == data_index
[docs]def assert_parallel_coordinates_plot(plot, order): """Checks the layout of a parallel coordinates plot""" assert ( plot.layout.title.text == "Parallel Coordinates Plot for experiment 'experiment-name'" ) trace = plot.data[0] for i in range(len(order)): assert trace.dimensions[i].label == order[i]