|
| 1 | +""" |
| 2 | +======================= |
| 3 | +Visualizing the Results |
| 4 | +======================= |
| 5 | +
|
| 6 | +Auto-Pytorch uses SMAC to fit individual machine learning algorithms |
| 7 | +and then ensembles them together using `Ensemble Selection |
| 8 | +<https://www.cs.cornell.edu/~caruana/ctp/ct.papers/caruana.icml04.icdm06long.pdf>`_. |
| 9 | +
|
| 10 | +The following examples shows how to visualize both the performance |
| 11 | +of the individual models and their respective ensemble. |
| 12 | +
|
| 13 | +Additionally, as we are compatible with scikit-learn, |
| 14 | +we show how to further interact with `Scikit-Learn Inspection |
| 15 | +<https://scikit-learn.org/stable/inspection.html>`_ support. |
| 16 | +
|
| 17 | +
|
| 18 | +""" |
| 19 | +import os |
| 20 | +import pickle |
| 21 | +import tempfile as tmp |
| 22 | +import time |
| 23 | +import warnings |
| 24 | + |
| 25 | +# The following variables are not needed for every unix distribution, but are |
| 26 | +# highlighted in here to prevent problems with multiprocessing with scikit-learn. |
| 27 | +os.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir() |
| 28 | +os.environ['OMP_NUM_THREADS'] = '1' |
| 29 | +os.environ['OPENBLAS_NUM_THREADS'] = '1' |
| 30 | +os.environ['MKL_NUM_THREADS'] = '1' |
| 31 | + |
| 32 | +warnings.simplefilter(action='ignore', category=UserWarning) |
| 33 | +warnings.simplefilter(action='ignore', category=FutureWarning) |
| 34 | + |
| 35 | +import matplotlib.pyplot as plt |
| 36 | + |
| 37 | +import numpy as np |
| 38 | + |
| 39 | +import pandas as pd |
| 40 | + |
| 41 | + |
| 42 | +import sklearn.datasets |
| 43 | +import sklearn.model_selection |
| 44 | +from sklearn.inspection import permutation_importance |
| 45 | + |
| 46 | +from smac.tae import StatusType |
| 47 | + |
| 48 | + |
| 49 | +from autoPyTorch.api.tabular_classification import TabularClassificationTask |
| 50 | +from autoPyTorch.metrics import accuracy |
| 51 | + |
| 52 | + |
| 53 | +if __name__ == '__main__': |
| 54 | + |
| 55 | + ############################################################################ |
| 56 | + # Data Loading |
| 57 | + # ============ |
| 58 | + |
| 59 | + # We will use the iris dataset for this Toy example |
| 60 | + seed = 42 |
| 61 | + X, y = sklearn.datasets.fetch_openml(data_id=61, return_X_y=True, as_frame=True) |
| 62 | + X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( |
| 63 | + X, |
| 64 | + y, |
| 65 | + random_state=42, |
| 66 | + ) |
| 67 | + |
| 68 | + ############################################################################ |
| 69 | + # Build and fit a classifier |
| 70 | + # ========================== |
| 71 | + api = TabularClassificationTask(seed=seed) |
| 72 | + api.search( |
| 73 | + X_train=X_train, |
| 74 | + y_train=y_train, |
| 75 | + X_test=X_test.copy(), |
| 76 | + y_test=y_test.copy(), |
| 77 | + optimize_metric=accuracy.name, |
| 78 | + total_walltime_limit=200, |
| 79 | + func_eval_time_limit_secs=50 |
| 80 | + ) |
| 81 | + |
| 82 | + ############################################################################ |
| 83 | + # One can also save the model for future inference |
| 84 | + # ================================================ |
| 85 | + |
| 86 | + # For more details on how to deploy a model, please check |
| 87 | + # `Scikit-Learn persistence |
| 88 | + # <https://scikit-learn.org/stable/modules/model_persistence.html>`_ support. |
| 89 | + with open('estimator.pickle', 'wb') as handle: |
| 90 | + pickle.dump(api, handle, protocol=pickle.HIGHEST_PROTOCOL) |
| 91 | + |
| 92 | + # Then let us read it back and use it for our analysis |
| 93 | + with open('estimator.pickle', 'rb') as handle: |
| 94 | + estimator = pickle.load(handle) |
| 95 | + |
| 96 | + ############################################################################ |
| 97 | + # Plotting the model performance |
| 98 | + # ============================== |
| 99 | + |
| 100 | + # We will plot the search incumbent through time. |
| 101 | + |
| 102 | + # Collect the performance of individual machine learning algorithms |
| 103 | + # found by SMAC |
| 104 | + individual_performances = [] |
| 105 | + for run_key, run_value in estimator.run_history.data.items(): |
| 106 | + if run_value.status != StatusType.SUCCESS: |
| 107 | + # Ignore crashed runs |
| 108 | + continue |
| 109 | + individual_performances.append({ |
| 110 | + 'Timestamp': pd.Timestamp( |
| 111 | + time.strftime( |
| 112 | + '%Y-%m-%d %H:%M:%S', |
| 113 | + time.localtime(run_value.endtime) |
| 114 | + ) |
| 115 | + ), |
| 116 | + 'single_best_optimization_accuracy': accuracy._optimum - run_value.cost, |
| 117 | + 'single_best_test_accuracy': np.nan if run_value.additional_info is None else |
| 118 | + accuracy._optimum - run_value.additional_info['test_loss'], |
| 119 | + }) |
| 120 | + individual_performance_frame = pd.DataFrame(individual_performances) |
| 121 | + |
| 122 | + # Collect the performance of the ensemble through time |
| 123 | + # This ensemble is built from the machine learning algorithms |
| 124 | + # found by SMAC |
| 125 | + ensemble_performance_frame = pd.DataFrame(estimator.ensemble_performance_history) |
| 126 | + |
| 127 | + # As we are tracking the incumbent, we are interested in the cummax() performance |
| 128 | + ensemble_performance_frame['ensemble_optimization_accuracy'] = ensemble_performance_frame[ |
| 129 | + 'train_accuracy' |
| 130 | + ].cummax() |
| 131 | + ensemble_performance_frame['ensemble_test_accuracy'] = ensemble_performance_frame[ |
| 132 | + 'test_accuracy' |
| 133 | + ].cummax() |
| 134 | + ensemble_performance_frame.drop(columns=['test_accuracy', 'train_accuracy'], inplace=True) |
| 135 | + individual_performance_frame['single_best_optimization_accuracy'] = individual_performance_frame[ |
| 136 | + 'single_best_optimization_accuracy' |
| 137 | + ].cummax() |
| 138 | + individual_performance_frame['single_best_test_accuracy'] = individual_performance_frame[ |
| 139 | + 'single_best_test_accuracy' |
| 140 | + ].cummax() |
| 141 | + |
| 142 | + pd.merge( |
| 143 | + ensemble_performance_frame, |
| 144 | + individual_performance_frame, |
| 145 | + on="Timestamp", how='outer' |
| 146 | + ).sort_values('Timestamp').fillna(method='ffill').plot( |
| 147 | + x='Timestamp', |
| 148 | + kind='line', |
| 149 | + legend=True, |
| 150 | + title='Auto-PyTorch accuracy over time', |
| 151 | + grid=True, |
| 152 | + ) |
| 153 | + plt.show() |
| 154 | + |
| 155 | + # We then can understand the importance of each input feature using |
| 156 | + # a permutation importance analysis. This is done as a proof of concept, to |
| 157 | + # showcase that we can leverage of scikit-learn API. |
| 158 | + result = permutation_importance(estimator, X_train, y_train, n_repeats=5, |
| 159 | + scoring='accuracy', |
| 160 | + random_state=seed) |
| 161 | + sorted_idx = result.importances_mean.argsort() |
| 162 | + |
| 163 | + fig, ax = plt.subplots() |
| 164 | + ax.boxplot(result.importances[sorted_idx].T, |
| 165 | + vert=False, labels=X_test.columns[sorted_idx]) |
| 166 | + ax.set_title("Permutation Importances (Train set)") |
| 167 | + fig.tight_layout() |
| 168 | + plt.show() |
0 commit comments