diff --git a/development/_downloads/307f532dbef0476f85afc6b64b65f087/example_resampling_strategy.py b/development/_downloads/307f532dbef0476f85afc6b64b65f087/example_resampling_strategy.py index 6735fffee..d02859f1b 100644 --- a/development/_downloads/307f532dbef0476f85afc6b64b65f087/example_resampling_strategy.py +++ b/development/_downloads/307f532dbef0476f85afc6b64b65f087/example_resampling_strategy.py @@ -26,10 +26,13 @@ from autoPyTorch.api.tabular_classification import TabularClassificationTask from autoPyTorch.datasets.resampling_strategy import CrossValTypes, HoldoutValTypes +############################################################################ +# Default Resampling Strategy +# ============================ ############################################################################ # Data Loading -# ============ +# ------------ X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( X, @@ -39,7 +42,7 @@ ############################################################################ # Build and fit a classifier with default resampling strategy -# =========================================================== +# ----------------------------------------------------------- api = TabularClassificationTask( # 'HoldoutValTypes.holdout_validation' with 'val_share': 0.33 # is the default argument setting for TabularClassificationTask. @@ -51,7 +54,7 @@ ############################################################################ # Search for an ensemble of machine learning algorithms -# ===================================================== +# ----------------------------------------------------- api.search( X_train=X_train, y_train=y_train, @@ -64,7 +67,7 @@ ############################################################################ # Print the final ensemble performance -# ==================================== +# ------------------------------------ y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) @@ -76,9 +79,13 @@ ############################################################################ +############################################################################ +# Cross validation Resampling Strategy +# ===================================== + ############################################################################ # Build and fit a classifier with Cross validation resampling strategy -# ==================================================================== +# -------------------------------------------------------------------- api = TabularClassificationTask( resampling_strategy=CrossValTypes.k_fold_cross_validation, resampling_strategy_args={'num_splits': 3} @@ -86,7 +93,8 @@ ############################################################################ # Search for an ensemble of machine learning algorithms -# ===================================================== +# ----------------------------------------------------------------------- + api.search( X_train=X_train, y_train=y_train, @@ -99,7 +107,7 @@ ############################################################################ # Print the final ensemble performance -# ==================================== +# ------------ y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) @@ -111,9 +119,13 @@ ############################################################################ +############################################################################ +# Stratified Resampling Strategy +# =============================== + ############################################################################ # Build and fit a classifier with Stratified resampling strategy -# ============================================================== +# -------------------------------------------------------------- api = TabularClassificationTask( # For demonstration purposes, we use # Stratified hold out validation. However, @@ -124,7 +136,7 @@ ############################################################################ # Search for an ensemble of machine learning algorithms -# ===================================================== +# ----------------------------------------------------- api.search( X_train=X_train, y_train=y_train, diff --git a/development/_downloads/3f9c66ebcc4532fdade3cdaa4d769bde/example_custom_configuration_space.ipynb b/development/_downloads/3f9c66ebcc4532fdade3cdaa4d769bde/example_custom_configuration_space.ipynb index 57a84b7a5..d019dfb3a 100644 --- a/development/_downloads/3f9c66ebcc4532fdade3cdaa4d769bde/example_custom_configuration_space.ipynb +++ b/development/_downloads/3f9c66ebcc4532fdade3cdaa4d769bde/example_custom_configuration_space.ipynb @@ -15,7 +15,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\n# Tabular Classification with Custom Configuration Space\n\nThe following example shows how adjust the configuration space of\nthe search. Currently, there are two changes that can be made to the space:-\n1. Adjust individual hyperparameters in the pipeline\n2. Include or exclude components:\n a) include: Dictionary containing components to include. Key is the node\n name and Value is an Iterable of the names of the components\n to include. Only these components will be present in the\n search space.\n b) exclude: Dictionary containing components to exclude. Key is the node\n name and Value is an Iterable of the names of the components\n to exclude. All except these components will be present in\n the search space.\n" + "\n# Tabular Classification with Custom Configuration Space\n\nThe following example shows how adjust the configuration space of\nthe search. Currently, there are two changes that can be made to the space:-\n\n1. Adjust individual hyperparameters in the pipeline\n2. Include or exclude components:\n a) include: Dictionary containing components to include. Key is the node\n name and Value is an Iterable of the names of the components\n to include. Only these components will be present in the\n search space.\n b) exclude: Dictionary containing components to exclude. Key is the node\n name and Value is an Iterable of the names of the components\n to exclude. All except these components will be present in\n the search space.\n" ] }, { @@ -26,7 +26,133 @@ }, "outputs": [], "source": [ - "import os\nimport tempfile as tmp\nimport warnings\n\nos.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir()\nos.environ['OMP_NUM_THREADS'] = '1'\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\nos.environ['MKL_NUM_THREADS'] = '1'\n\nwarnings.simplefilter(action='ignore', category=UserWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport sklearn.datasets\nimport sklearn.model_selection\n\nfrom autoPyTorch.api.tabular_classification import TabularClassificationTask\nfrom autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates\n\n\ndef get_search_space_updates():\n \"\"\"\n Search space updates to the task can be added using HyperparameterSearchSpaceUpdates\n Returns:\n HyperparameterSearchSpaceUpdates\n \"\"\"\n updates = HyperparameterSearchSpaceUpdates()\n updates.append(node_name=\"data_loader\",\n hyperparameter=\"batch_size\",\n value_range=[16, 512],\n default_value=32)\n updates.append(node_name=\"lr_scheduler\",\n hyperparameter=\"CosineAnnealingLR:T_max\",\n value_range=[50, 60],\n default_value=55)\n updates.append(node_name='network_backbone',\n hyperparameter='ResNetBackbone:dropout',\n value_range=[0, 0.5],\n default_value=0.2)\n return updates\n\n\nif __name__ == '__main__':\n\n ############################################################################\n # Data Loading\n # ============\n X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True)\n X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(\n X,\n y,\n random_state=1,\n )\n\n ############################################################################\n # Build and fit a classifier with include components\n # ==================================================\n api = TabularClassificationTask(\n search_space_updates=get_search_space_updates(),\n include_components={'network_backbone': ['MLPBackbone', 'ResNetBackbone'],\n 'encoder': ['OneHotEncoder']}\n )\n\n ############################################################################\n # Search for an ensemble of machine learning algorithms\n # =====================================================\n api.search(\n X_train=X_train.copy(),\n y_train=y_train.copy(),\n X_test=X_test.copy(),\n y_test=y_test.copy(),\n optimize_metric='accuracy',\n total_walltime_limit=150,\n func_eval_time_limit_secs=30\n )\n\n ############################################################################\n # Print the final ensemble performance\n # ====================================\n y_pred = api.predict(X_test)\n score = api.score(y_pred, y_test)\n print(score)\n print(api.show_models())\n\n # Print statistics from search\n print(api.sprint_statistics())\n\n ############################################################################\n # Build and fit a classifier with exclude components\n # ==================================================\n api = TabularClassificationTask(\n search_space_updates=get_search_space_updates(),\n exclude_components={'network_backbone': ['MLPBackbone'],\n 'encoder': ['OneHotEncoder']}\n )\n\n ############################################################################\n # Search for an ensemble of machine learning algorithms\n # =====================================================\n api.search(\n X_train=X_train,\n y_train=y_train,\n X_test=X_test.copy(),\n y_test=y_test.copy(),\n optimize_metric='accuracy',\n total_walltime_limit=150,\n func_eval_time_limit_secs=30\n )\n\n ############################################################################\n # Print the final ensemble performance\n # ====================================\n y_pred = api.predict(X_test)\n score = api.score(y_pred, y_test)\n print(score)\n print(api.show_models())\n\n # Print statistics from search\n print(api.sprint_statistics())" + "import os\nimport tempfile as tmp\nimport warnings\n\nos.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir()\nos.environ['OMP_NUM_THREADS'] = '1'\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\nos.environ['MKL_NUM_THREADS'] = '1'\n\nwarnings.simplefilter(action='ignore', category=UserWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport sklearn.datasets\nimport sklearn.model_selection\n\nfrom autoPyTorch.api.tabular_classification import TabularClassificationTask\nfrom autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates\n\n\ndef get_search_space_updates():\n \"\"\"\n Search space updates to the task can be added using HyperparameterSearchSpaceUpdates\n Returns:\n HyperparameterSearchSpaceUpdates\n \"\"\"\n updates = HyperparameterSearchSpaceUpdates()\n updates.append(node_name=\"data_loader\",\n hyperparameter=\"batch_size\",\n value_range=[16, 512],\n default_value=32)\n updates.append(node_name=\"lr_scheduler\",\n hyperparameter=\"CosineAnnealingLR:T_max\",\n value_range=[50, 60],\n default_value=55)\n updates.append(node_name='network_backbone',\n hyperparameter='ResNetBackbone:dropout',\n value_range=[0, 0.5],\n default_value=0.2)\n return updates" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data Loading\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True)\nX_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(\n X,\n y,\n random_state=1,\n)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Build and fit a classifier with include components\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "api = TabularClassificationTask(\n search_space_updates=get_search_space_updates(),\n include_components={'network_backbone': ['MLPBackbone', 'ResNetBackbone'],\n 'encoder': ['OneHotEncoder']}\n)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Search for an ensemble of machine learning algorithms\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "api.search(\n X_train=X_train.copy(),\n y_train=y_train.copy(),\n X_test=X_test.copy(),\n y_test=y_test.copy(),\n optimize_metric='accuracy',\n total_walltime_limit=150,\n func_eval_time_limit_secs=30\n)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Print the final ensemble performance\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "y_pred = api.predict(X_test)\nscore = api.score(y_pred, y_test)\nprint(score)\nprint(api.show_models())\n\n# Print statistics from search\nprint(api.sprint_statistics())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Build and fit a classifier with exclude components\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "api = TabularClassificationTask(\n search_space_updates=get_search_space_updates(),\n exclude_components={'network_backbone': ['MLPBackbone'],\n 'encoder': ['OneHotEncoder']}\n)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Search for an ensemble of machine learning algorithms\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "api.search(\n X_train=X_train,\n y_train=y_train,\n X_test=X_test.copy(),\n y_test=y_test.copy(),\n optimize_metric='accuracy',\n total_walltime_limit=150,\n func_eval_time_limit_secs=30\n)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Print the final ensemble performance\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "y_pred = api.predict(X_test)\nscore = api.score(y_pred, y_test)\nprint(score)\nprint(api.show_models())\n\n# Print statistics from search\nprint(api.sprint_statistics())" ] } ], diff --git a/development/_downloads/4cbefcc88d68bf84110d315dc5fdb8e1/example_resampling_strategy.ipynb b/development/_downloads/4cbefcc88d68bf84110d315dc5fdb8e1/example_resampling_strategy.ipynb index 102baac50..9c163af7c 100644 --- a/development/_downloads/4cbefcc88d68bf84110d315dc5fdb8e1/example_resampling_strategy.ipynb +++ b/development/_downloads/4cbefcc88d68bf84110d315dc5fdb8e1/example_resampling_strategy.ipynb @@ -33,7 +33,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Data Loading\n\n" + "## Default Resampling Strategy\n\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data Loading\n\n" ] }, { @@ -51,7 +58,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Build and fit a classifier with default resampling strategy\n\n" + "### Build and fit a classifier with default resampling strategy\n\n" ] }, { @@ -69,7 +76,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Search for an ensemble of machine learning algorithms\n\n" + "### Search for an ensemble of machine learning algorithms\n\n" ] }, { @@ -87,7 +94,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Print the final ensemble performance\n\n" + "### Print the final ensemble performance\n\n" ] }, { @@ -105,7 +112,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Build and fit a classifier with Cross validation resampling strategy\n\n" + "## Cross validation Resampling Strategy\n\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Build and fit a classifier with Cross validation resampling strategy\n\n" ] }, { @@ -123,7 +137,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Search for an ensemble of machine learning algorithms\n\n" + "### Search for an ensemble of machine learning algorithms\n\n" ] }, { @@ -141,7 +155,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Print the final ensemble performance\n\n" + "### Print the final ensemble performance\n\n" ] }, { @@ -159,7 +173,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Build and fit a classifier with Stratified resampling strategy\n\n" + "## Stratified Resampling Strategy\n\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Build and fit a classifier with Stratified resampling strategy\n\n" ] }, { @@ -177,7 +198,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Search for an ensemble of machine learning algorithms\n\n" + "### Search for an ensemble of machine learning algorithms\n\n" ] }, { diff --git a/development/_downloads/5517f58caf37183d75ea0dc7cedf8b58/example_custom_configuration_space.py b/development/_downloads/5517f58caf37183d75ea0dc7cedf8b58/example_custom_configuration_space.py index c64a4fca1..985d9d9ff 100644 --- a/development/_downloads/5517f58caf37183d75ea0dc7cedf8b58/example_custom_configuration_space.py +++ b/development/_downloads/5517f58caf37183d75ea0dc7cedf8b58/example_custom_configuration_space.py @@ -5,6 +5,7 @@ The following example shows how adjust the configuration space of the search. Currently, there are two changes that can be made to the space:- + 1. Adjust individual hyperparameters in the pipeline 2. Include or exclude components: a) include: Dictionary containing components to include. Key is the node @@ -57,80 +58,78 @@ def get_search_space_updates(): return updates -if __name__ == '__main__': - - ############################################################################ - # Data Loading - # ============ - X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) - X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - X, - y, - random_state=1, - ) - - ############################################################################ - # Build and fit a classifier with include components - # ================================================== - api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - include_components={'network_backbone': ['MLPBackbone', 'ResNetBackbone'], - 'encoder': ['OneHotEncoder']} - ) - - ############################################################################ - # Search for an ensemble of machine learning algorithms - # ===================================================== - api.search( - X_train=X_train.copy(), - y_train=y_train.copy(), - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=150, - func_eval_time_limit_secs=30 - ) - - ############################################################################ - # Print the final ensemble performance - # ==================================== - y_pred = api.predict(X_test) - score = api.score(y_pred, y_test) - print(score) - print(api.show_models()) - - # Print statistics from search - print(api.sprint_statistics()) - - ############################################################################ - # Build and fit a classifier with exclude components - # ================================================== - api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - exclude_components={'network_backbone': ['MLPBackbone'], - 'encoder': ['OneHotEncoder']} - ) - - ############################################################################ - # Search for an ensemble of machine learning algorithms - # ===================================================== - api.search( - X_train=X_train, - y_train=y_train, - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=150, - func_eval_time_limit_secs=30 - ) - - ############################################################################ - # Print the final ensemble performance - # ==================================== - y_pred = api.predict(X_test) - score = api.score(y_pred, y_test) - print(score) - print(api.show_models()) - - # Print statistics from search - print(api.sprint_statistics()) +############################################################################ +# Data Loading +# ============ +X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) +X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, + y, + random_state=1, +) + +############################################################################ +# Build and fit a classifier with include components +# ================================================== +api = TabularClassificationTask( + search_space_updates=get_search_space_updates(), + include_components={'network_backbone': ['MLPBackbone', 'ResNetBackbone'], + 'encoder': ['OneHotEncoder']} +) + +############################################################################ +# Search for an ensemble of machine learning algorithms +# ===================================================== +api.search( + X_train=X_train.copy(), + y_train=y_train.copy(), + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=150, + func_eval_time_limit_secs=30 +) + +############################################################################ +# Print the final ensemble performance +# ==================================== +y_pred = api.predict(X_test) +score = api.score(y_pred, y_test) +print(score) +print(api.show_models()) + +# Print statistics from search +print(api.sprint_statistics()) + +############################################################################ +# Build and fit a classifier with exclude components +# ================================================== +api = TabularClassificationTask( + search_space_updates=get_search_space_updates(), + exclude_components={'network_backbone': ['MLPBackbone'], + 'encoder': ['OneHotEncoder']} +) + +############################################################################ +# Search for an ensemble of machine learning algorithms +# ===================================================== +api.search( + X_train=X_train, + y_train=y_train, + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=150, + func_eval_time_limit_secs=30 +) + +############################################################################ +# Print the final ensemble performance +# ==================================== +y_pred = api.predict(X_test) +score = api.score(y_pred, y_test) +print(score) +print(api.show_models()) + +# Print statistics from search +print(api.sprint_statistics()) diff --git a/development/_downloads/6e1b08ae2c5784892d9641cc2992248d/example_run_with_portfolio.py b/development/_downloads/6e1b08ae2c5784892d9641cc2992248d/example_run_with_portfolio.py index 01d8bef15..fef230fc5 100644 --- a/development/_downloads/6e1b08ae2c5784892d9641cc2992248d/example_run_with_portfolio.py +++ b/development/_downloads/6e1b08ae2c5784892d9641cc2992248d/example_run_with_portfolio.py @@ -24,50 +24,48 @@ from autoPyTorch.api.tabular_classification import TabularClassificationTask -if __name__ == '__main__': +############################################################################ +# Data Loading +# ============ +X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) +X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, + y, + random_state=42, +) - ############################################################################ - # Data Loading - # ============ - X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) - X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - X, - y, - random_state=42, - ) +############################################################################ +# Build and fit a classifier +# ========================== +api = TabularClassificationTask( + seed=42, +) - ############################################################################ - # Build and fit a classifier - # ========================== - api = TabularClassificationTask( - seed=42, - ) +############################################################################ +# Search for an ensemble of machine learning algorithms +# ===================================================== +api.search( + X_train=X_train, + y_train=y_train, + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=300, + func_eval_time_limit_secs=50, + # Setting this option to "greedy" + # will make smac run the configurations + # present in 'autoPyTorch/configs/greedy_portfolio.json' + portfolio_selection="greedy" +) - ############################################################################ - # Search for an ensemble of machine learning algorithms - # ===================================================== - api.search( - X_train=X_train, - y_train=y_train, - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=300, - func_eval_time_limit_secs=50, - # Setting this option to "greedy" - # will make smac run the configurations - # present in 'autoPyTorch/configs/greedy_portfolio.json' - portfolio_selection="greedy" - ) +############################################################################ +# Print the final ensemble performance +# ==================================== +y_pred = api.predict(X_test) +score = api.score(y_pred, y_test) +print(score) +# Print the final ensemble built by AutoPyTorch +print(api.show_models()) - ############################################################################ - # Print the final ensemble performance - # ==================================== - y_pred = api.predict(X_test) - score = api.score(y_pred, y_test) - print(score) - # Print the final ensemble built by AutoPyTorch - print(api.show_models()) - - # Print statistics from search - print(api.sprint_statistics()) +# Print statistics from search +print(api.sprint_statistics()) diff --git a/development/_downloads/87ab5d5bc35882bb85e7300281424079/example_parallel_n_jobs.py b/development/_downloads/87ab5d5bc35882bb85e7300281424079/example_parallel_n_jobs.py index 698f3ad61..d345c6fca 100644 --- a/development/_downloads/87ab5d5bc35882bb85e7300281424079/example_parallel_n_jobs.py +++ b/development/_downloads/87ab5d5bc35882bb85e7300281424079/example_parallel_n_jobs.py @@ -1,10 +1,11 @@ """ -====================== -Tabular Classification -====================== +============================================ +Tabular Classification with n parallel jobs +============================================ The following example shows how to fit a sample classification model parallely on 2 cores with AutoPyTorch + """ import os import tempfile as tmp @@ -60,9 +61,9 @@ ############################################################################ # Print the final ensemble performance # ==================================== - print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) # Print the final ensemble built by AutoPyTorch - print(api.show_models()) + print(api.sprint_statistics()) + diff --git a/development/_downloads/8cd648e2e60261ebda890b9c337a59bb/example_parallel_n_jobs.ipynb b/development/_downloads/8cd648e2e60261ebda890b9c337a59bb/example_parallel_n_jobs.ipynb index ab9914d14..5e8dc507d 100644 --- a/development/_downloads/8cd648e2e60261ebda890b9c337a59bb/example_parallel_n_jobs.ipynb +++ b/development/_downloads/8cd648e2e60261ebda890b9c337a59bb/example_parallel_n_jobs.ipynb @@ -15,7 +15,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\n# Tabular Classification\n\nThe following example shows how to fit a sample classification model parallely on 2 cores\nwith AutoPyTorch\n" + "\n# Tabular Classification with n parallel jobs\n\nThe following example shows how to fit a sample classification model parallely on 2 cores\nwith AutoPyTorch\n" ] }, { @@ -26,7 +26,7 @@ }, "outputs": [], "source": [ - "import os\nimport tempfile as tmp\nimport warnings\n\nos.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir()\nos.environ['OMP_NUM_THREADS'] = '1'\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\nos.environ['MKL_NUM_THREADS'] = '1'\n\nwarnings.simplefilter(action='ignore', category=UserWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport sklearn.datasets\nimport sklearn.model_selection\n\nfrom autoPyTorch.api.tabular_classification import TabularClassificationTask\n\nif __name__ == '__main__':\n ############################################################################\n # Data Loading\n # ============\n X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True)\n X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(\n X,\n y,\n random_state=1,\n )\n\n ############################################################################\n # Build and fit a classifier\n # ==========================\n api = TabularClassificationTask(\n n_jobs=2,\n seed=42,\n )\n\n ############################################################################\n # Search for an ensemble of machine learning algorithms\n # =====================================================\n api.search(\n X_train=X_train,\n y_train=y_train,\n X_test=X_test.copy(),\n y_test=y_test.copy(),\n optimize_metric='accuracy',\n total_walltime_limit=300,\n func_eval_time_limit_secs=50,\n # Each one of the 2 jobs is allocated 3GB\n memory_limit=3072,\n )\n\n ############################################################################\n # Print the final ensemble performance\n # ====================================\n print(api.run_history, api.trajectory)\n y_pred = api.predict(X_test)\n score = api.score(y_pred, y_test)\n print(score)\n # Print the final ensemble built by AutoPyTorch\n print(api.show_models())" + "import os\nimport tempfile as tmp\nimport warnings\n\nos.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir()\nos.environ['OMP_NUM_THREADS'] = '1'\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\nos.environ['MKL_NUM_THREADS'] = '1'\n\nwarnings.simplefilter(action='ignore', category=UserWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport sklearn.datasets\nimport sklearn.model_selection\n\nfrom autoPyTorch.api.tabular_classification import TabularClassificationTask\n\nif __name__ == '__main__':\n ############################################################################\n # Data Loading\n # ============\n X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True)\n X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(\n X,\n y,\n random_state=1,\n )\n\n ############################################################################\n # Build and fit a classifier\n # ==========================\n api = TabularClassificationTask(\n n_jobs=2,\n seed=42,\n )\n\n ############################################################################\n # Search for an ensemble of machine learning algorithms\n # =====================================================\n api.search(\n X_train=X_train,\n y_train=y_train,\n X_test=X_test.copy(),\n y_test=y_test.copy(),\n optimize_metric='accuracy',\n total_walltime_limit=300,\n func_eval_time_limit_secs=50,\n # Each one of the 2 jobs is allocated 3GB\n memory_limit=3072,\n )\n\n ############################################################################\n # Print the final ensemble performance\n # ====================================\n y_pred = api.predict(X_test)\n score = api.score(y_pred, y_test)\n print(score)\n # Print the final ensemble built by AutoPyTorch\n print(api.sprint_statistics())" ] } ], diff --git a/development/_downloads/a33a692c4a7e96947d47bf5940ec46a7/example_run_with_portfolio.ipynb b/development/_downloads/a33a692c4a7e96947d47bf5940ec46a7/example_run_with_portfolio.ipynb index 6613341fd..4e10c80b4 100644 --- a/development/_downloads/a33a692c4a7e96947d47bf5940ec46a7/example_run_with_portfolio.ipynb +++ b/development/_downloads/a33a692c4a7e96947d47bf5940ec46a7/example_run_with_portfolio.ipynb @@ -26,7 +26,79 @@ }, "outputs": [], "source": [ - "import os\nimport tempfile as tmp\nimport warnings\n\nos.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir()\nos.environ['OMP_NUM_THREADS'] = '1'\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\nos.environ['MKL_NUM_THREADS'] = '1'\n\nwarnings.simplefilter(action='ignore', category=UserWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport sklearn.datasets\nimport sklearn.model_selection\n\nfrom autoPyTorch.api.tabular_classification import TabularClassificationTask\n\n\nif __name__ == '__main__':\n\n ############################################################################\n # Data Loading\n # ============\n X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True)\n X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(\n X,\n y,\n random_state=42,\n )\n\n ############################################################################\n # Build and fit a classifier\n # ==========================\n api = TabularClassificationTask(\n seed=42,\n )\n\n ############################################################################\n # Search for an ensemble of machine learning algorithms\n # =====================================================\n api.search(\n X_train=X_train,\n y_train=y_train,\n X_test=X_test.copy(),\n y_test=y_test.copy(),\n optimize_metric='accuracy',\n total_walltime_limit=300,\n func_eval_time_limit_secs=50,\n # Setting this option to \"greedy\"\n # will make smac run the configurations\n # present in 'autoPyTorch/configs/greedy_portfolio.json'\n portfolio_selection=\"greedy\"\n )\n\n ############################################################################\n # Print the final ensemble performance\n # ====================================\n y_pred = api.predict(X_test)\n score = api.score(y_pred, y_test)\n print(score)\n # Print the final ensemble built by AutoPyTorch\n print(api.show_models())\n\n # Print statistics from search\n print(api.sprint_statistics())" + "import os\nimport tempfile as tmp\nimport warnings\n\nos.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir()\nos.environ['OMP_NUM_THREADS'] = '1'\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\nos.environ['MKL_NUM_THREADS'] = '1'\n\nwarnings.simplefilter(action='ignore', category=UserWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport sklearn.datasets\nimport sklearn.model_selection\n\nfrom autoPyTorch.api.tabular_classification import TabularClassificationTask" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data Loading\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True)\nX_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(\n X,\n y,\n random_state=42,\n)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Build and fit a classifier\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "api = TabularClassificationTask(\n seed=42,\n)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Search for an ensemble of machine learning algorithms\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "api.search(\n X_train=X_train,\n y_train=y_train,\n X_test=X_test.copy(),\n y_test=y_test.copy(),\n optimize_metric='accuracy',\n total_walltime_limit=300,\n func_eval_time_limit_secs=50,\n # Setting this option to \"greedy\"\n # will make smac run the configurations\n # present in 'autoPyTorch/configs/greedy_portfolio.json'\n portfolio_selection=\"greedy\"\n)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Print the final ensemble performance\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "y_pred = api.predict(X_test)\nscore = api.score(y_pred, y_test)\nprint(score)\n# Print the final ensemble built by AutoPyTorch\nprint(api.show_models())\n\n# Print statistics from search\nprint(api.sprint_statistics())" ] } ], diff --git a/development/_downloads/bc82bea3a5dd7bdba60b65220891d9e5/examples_python.zip b/development/_downloads/bc82bea3a5dd7bdba60b65220891d9e5/examples_python.zip index 0539a3216..32f0c5113 100644 Binary files a/development/_downloads/bc82bea3a5dd7bdba60b65220891d9e5/examples_python.zip and b/development/_downloads/bc82bea3a5dd7bdba60b65220891d9e5/examples_python.zip differ diff --git a/development/_downloads/fb625db3c50d423b1b7881136ffdeec8/examples_jupyter.zip b/development/_downloads/fb625db3c50d423b1b7881136ffdeec8/examples_jupyter.zip index 9bda5bd05..a7215b22d 100644 Binary files a/development/_downloads/fb625db3c50d423b1b7881136ffdeec8/examples_jupyter.zip and b/development/_downloads/fb625db3c50d423b1b7881136ffdeec8/examples_jupyter.zip differ diff --git a/development/_images/sphx_glr_example_visualization_001.png b/development/_images/sphx_glr_example_visualization_001.png index 4ab94e310..0150eb12e 100644 Binary files a/development/_images/sphx_glr_example_visualization_001.png and b/development/_images/sphx_glr_example_visualization_001.png differ diff --git a/development/_images/sphx_glr_example_visualization_002.png b/development/_images/sphx_glr_example_visualization_002.png index 061eda077..4e1219c4f 100644 Binary files a/development/_images/sphx_glr_example_visualization_002.png and b/development/_images/sphx_glr_example_visualization_002.png differ diff --git a/development/_images/sphx_glr_example_visualization_thumb.png b/development/_images/sphx_glr_example_visualization_thumb.png index 1d085b21e..ca145837e 100644 Binary files a/development/_images/sphx_glr_example_visualization_thumb.png and b/development/_images/sphx_glr_example_visualization_thumb.png differ diff --git a/development/_sources/examples/20_basics/example_image_classification.rst.txt b/development/_sources/examples/20_basics/example_image_classification.rst.txt index 555389979..9e152f2d8 100644 --- a/development/_sources/examples/20_basics/example_image_classification.rst.txt +++ b/development/_sources/examples/20_basics/example_image_classification.rst.txt @@ -86,16 +86,17 @@ Image Classification ________________________________________ Configuration: image_augmenter:GaussianBlur:use_augmenter, Value: False - image_augmenter:GaussianNoise:use_augmenter, Value: False - image_augmenter:RandomAffine:rotate, Value: 65 - image_augmenter:RandomAffine:scale_offset, Value: 0.10364752234694663 - image_augmenter:RandomAffine:shear, Value: 5 - image_augmenter:RandomAffine:translate_percent_offset, Value: 0.12303023778688212 + image_augmenter:GaussianNoise:sigma_offset, Value: 2.939416777169848 + image_augmenter:GaussianNoise:use_augmenter, Value: True + image_augmenter:RandomAffine:rotate, Value: 182 + image_augmenter:RandomAffine:scale_offset, Value: 0.2653043460236015 + image_augmenter:RandomAffine:shear, Value: 7 + image_augmenter:RandomAffine:translate_percent_offset, Value: 0.2981323677443226 image_augmenter:RandomAffine:use_augmenter, Value: True - image_augmenter:RandomCutout:p, Value: 0.9516838776905963 + image_augmenter:RandomCutout:p, Value: 0.46400105311756495 image_augmenter:RandomCutout:use_augmenter, Value: True image_augmenter:Resize:use_augmenter, Value: False - image_augmenter:ZeroPadAndCrop:percent, Value: 0.2732380932749452 + image_augmenter:ZeroPadAndCrop:percent, Value: 0.05116366366228997 normalizer:__choice__, Value: 'ImageNormalizer' Fitting the pipeline... @@ -175,7 +176,7 @@ Image Classification .. rst-class:: sphx-glr-timing - **Total running time of the script:** ( 0 minutes 6.216 seconds) + **Total running time of the script:** ( 0 minutes 7.489 seconds) .. _sphx_glr_download_examples_20_basics_example_image_classification.py: diff --git a/development/_sources/examples/20_basics/example_tabular_classification.rst.txt b/development/_sources/examples/20_basics/example_tabular_classification.rst.txt index f17476a79..8e420a9b1 100644 --- a/development/_sources/examples/20_basics/example_tabular_classification.rst.txt +++ b/development/_sources/examples/20_basics/example_tabular_classification.rst.txt @@ -134,7 +134,7 @@ Search for an ensemble of machine learning algorithms .. code-block:: none - + @@ -165,26 +165,28 @@ Print the final ensemble performance .. code-block:: none - {'accuracy': 0.8554913294797688} - | | Preprocessing | Estimator | Weight | - |---:|:----------------------------------------------------------|:----------------------------------------------------------------|---------:| - | 0 | None | RFLearner | 0.24 | - | 1 | SimpleImputer,OneHotEncoder,Normalizer,KernelPCA | embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.22 | - | 2 | None | SVMLearner | 0.18 | - | 3 | None | CBLearner | 0.1 | - | 4 | None | ETLearner | 0.08 | - | 5 | None | KNNLearner | 0.08 | - | 6 | SimpleImputer,OneHotEncoder,MinMaxScaler,PowerTransformer | embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.04 | - | 7 | SimpleImputer,NoEncoder,Normalizer,Nystroem | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.04 | - | 8 | None | LGBMLearner | 0.02 | + {'accuracy': 0.838150289017341} + | | Preprocessing | Estimator | Weight | + |---:|:------------------------------------------------------------------|:----------------------------------------------------------------|---------:| + | 0 | SimpleImputer,OneHotEncoder,Normalizer,KernelPCA | embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.2 | + | 1 | None | KNNLearner | 0.16 | + | 2 | None | CBLearner | 0.14 | + | 3 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.12 | + | 4 | SimpleImputer,OneHotEncoder,Normalizer,PowerTransformer | embedding,ResNetBackbone,FullyConnectedHead,nn.Sequential | 0.08 | + | 5 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.08 | + | 6 | SimpleImputer,OneHotEncoder,MinMaxScaler,PowerTransformer | embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.06 | + | 7 | SimpleImputer,NoEncoder,Normalizer,Nystroem | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.06 | + | 8 | None | SVMLearner | 0.04 | + | 9 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.04 | + | 10 | SimpleImputer,OneHotEncoder,MinMaxScaler,TruncSVD | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | autoPyTorch results: Dataset name: Australian Optimisation Metric: accuracy Best validation score: 0.8713450292397661 - Number of target algorithm runs: 23 + Number of target algorithm runs: 22 Number of successful target algorithm runs: 19 Number of crashed target algorithm runs: 2 - Number of target algorithms that exceeded the time limit: 2 + Number of target algorithms that exceeded the time limit: 1 Number of target algorithms that exceeded the memory limit: 0 @@ -194,7 +196,7 @@ Print the final ensemble performance .. rst-class:: sphx-glr-timing - **Total running time of the script:** ( 5 minutes 22.150 seconds) + **Total running time of the script:** ( 5 minutes 35.087 seconds) .. _sphx_glr_download_examples_20_basics_example_tabular_classification.py: diff --git a/development/_sources/examples/20_basics/example_tabular_regression.rst.txt b/development/_sources/examples/20_basics/example_tabular_regression.rst.txt index 75bc0b157..213ed65af 100644 --- a/development/_sources/examples/20_basics/example_tabular_regression.rst.txt +++ b/development/_sources/examples/20_basics/example_tabular_regression.rst.txt @@ -125,7 +125,7 @@ Search for an ensemble of machine learning algorithms .. code-block:: none - + @@ -159,7 +159,7 @@ Print the final ensemble performance .. code-block:: none - {'r2': 0.9445248186059718} + {'r2': 0.944631023189658} | | Preprocessing | Estimator | Weight | |---:|:------------------------------------------------------------------|:----------------------------------------------------------------|---------:| | 0 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.44 | @@ -167,9 +167,9 @@ Print the final ensemble performance | 2 | None | LGBMLearner | 0.08 | | 3 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.06 | autoPyTorch results: - Dataset name: 2b8bcd39-4bc0-11ec-8727-df8c3c818998 + Dataset name: 37d86b62-4bdf-11ec-877d-1fe05d8e2eef Optimisation Metric: r2 - Best validation score: 0.8645385039886702 + Best validation score: 0.8644967965917701 Number of target algorithm runs: 24 Number of successful target algorithm runs: 20 Number of crashed target algorithm runs: 2 @@ -183,7 +183,7 @@ Print the final ensemble performance .. rst-class:: sphx-glr-timing - **Total running time of the script:** ( 5 minutes 37.415 seconds) + **Total running time of the script:** ( 5 minutes 36.563 seconds) .. _sphx_glr_download_examples_20_basics_example_tabular_regression.py: diff --git a/development/_sources/examples/20_basics/sg_execution_times.rst.txt b/development/_sources/examples/20_basics/sg_execution_times.rst.txt index 205b1e5f8..26d8d9f10 100644 --- a/development/_sources/examples/20_basics/sg_execution_times.rst.txt +++ b/development/_sources/examples/20_basics/sg_execution_times.rst.txt @@ -5,12 +5,12 @@ Computation times ================= -**11:05.781** total execution time for **examples_20_basics** files: +**11:19.139** total execution time for **examples_20_basics** files: +--------------------------------------------------------------------------------------------------------------+-----------+--------+ -| :ref:`sphx_glr_examples_20_basics_example_tabular_regression.py` (``example_tabular_regression.py``) | 05:37.415 | 0.0 MB | +| :ref:`sphx_glr_examples_20_basics_example_tabular_regression.py` (``example_tabular_regression.py``) | 05:36.563 | 0.0 MB | +--------------------------------------------------------------------------------------------------------------+-----------+--------+ -| :ref:`sphx_glr_examples_20_basics_example_tabular_classification.py` (``example_tabular_classification.py``) | 05:22.150 | 0.0 MB | +| :ref:`sphx_glr_examples_20_basics_example_tabular_classification.py` (``example_tabular_classification.py``) | 05:35.087 | 0.0 MB | +--------------------------------------------------------------------------------------------------------------+-----------+--------+ -| :ref:`sphx_glr_examples_20_basics_example_image_classification.py` (``example_image_classification.py``) | 00:06.216 | 0.0 MB | +| :ref:`sphx_glr_examples_20_basics_example_image_classification.py` (``example_image_classification.py``) | 00:07.489 | 0.0 MB | +--------------------------------------------------------------------------------------------------------------+-----------+--------+ diff --git a/development/_sources/examples/40_advanced/example_custom_configuration_space.rst.txt b/development/_sources/examples/40_advanced/example_custom_configuration_space.rst.txt index aa417947f..64ea8b736 100644 --- a/development/_sources/examples/40_advanced/example_custom_configuration_space.rst.txt +++ b/development/_sources/examples/40_advanced/example_custom_configuration_space.rst.txt @@ -24,6 +24,7 @@ Tabular Classification with Custom Configuration Space The following example shows how adjust the configuration space of the search. Currently, there are two changes that can be made to the space:- + 1. Adjust individual hyperparameters in the pipeline 2. Include or exclude components: a) include: Dictionary containing components to include. Key is the node @@ -35,64 +36,7 @@ the search. Currently, there are two changes that can be made to the space:- to exclude. All except these components will be present in the search space. -.. GENERATED FROM PYTHON SOURCE LINES 19-137 - - - - -.. rst-class:: sphx-glr-script-out - - Out: - - .. code-block:: none - - {'accuracy': 0.8554913294797688} - | | Preprocessing | Estimator | Weight | - |---:|:------------------------------------------------------------------|:----------------------------------------------------------|---------:| - | 0 | None | CBLearner | 0.46 | - | 1 | SimpleImputer,OneHotEncoder,Normalizer,PolynomialFeatures | no embedding,MLPBackbone,FullyConnectedHead,nn.Sequential | 0.38 | - | 2 | None | SVMLearner | 0.06 | - | 3 | None | KNNLearner | 0.06 | - | 4 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,MLPBackbone,FullyConnectedHead,nn.Sequential | 0.04 | - autoPyTorch results: - Dataset name: 85726e36-4bc4-11ec-8727-df8c3c818998 - Optimisation Metric: accuracy - Best validation score: 0.8654970760233918 - Number of target algorithm runs: 18 - Number of successful target algorithm runs: 14 - Number of crashed target algorithm runs: 3 - Number of target algorithms that exceeded the time limit: 1 - Number of target algorithms that exceeded the memory limit: 0 - - {'accuracy': 0.861271676300578} - | | Preprocessing | Estimator | Weight | - |---:|:--------------------------------------------------------------|:----------------------------------------------------------------|---------:| - | 0 | SimpleImputer,NoEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.28 | - | 1 | None | RFLearner | 0.24 | - | 2 | None | KNNLearner | 0.16 | - | 3 | SimpleImputer,NoEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.12 | - | 4 | None | CBLearner | 0.08 | - | 5 | SimpleImputer,NoEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.06 | - | 6 | SimpleImputer,NoEncoder,StandardScaler,KernelPCA | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | - | 7 | None | ETLearner | 0.02 | - | 8 | None | SVMLearner | 0.02 | - autoPyTorch results: - Dataset name: ea262f4f-4bc4-11ec-8727-df8c3c818998 - Optimisation Metric: accuracy - Best validation score: 0.8654970760233918 - Number of target algorithm runs: 21 - Number of successful target algorithm runs: 15 - Number of crashed target algorithm runs: 5 - Number of target algorithms that exceeded the time limit: 1 - Number of target algorithms that exceeded the memory limit: 0 - - - - - - - -| +.. GENERATED FROM PYTHON SOURCE LINES 20-61 .. code-block:: default @@ -137,88 +81,253 @@ the search. Currently, there are two changes that can be made to the space:- return updates - if __name__ == '__main__': - - ############################################################################ - # Data Loading - # ============ - X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) - X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - X, - y, - random_state=1, - ) - - ############################################################################ - # Build and fit a classifier with include components - # ================================================== - api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - include_components={'network_backbone': ['MLPBackbone', 'ResNetBackbone'], - 'encoder': ['OneHotEncoder']} - ) - - ############################################################################ - # Search for an ensemble of machine learning algorithms - # ===================================================== - api.search( - X_train=X_train.copy(), - y_train=y_train.copy(), - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=150, - func_eval_time_limit_secs=30 - ) - - ############################################################################ - # Print the final ensemble performance - # ==================================== - y_pred = api.predict(X_test) - score = api.score(y_pred, y_test) - print(score) - print(api.show_models()) - - # Print statistics from search - print(api.sprint_statistics()) - - ############################################################################ - # Build and fit a classifier with exclude components - # ================================================== - api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - exclude_components={'network_backbone': ['MLPBackbone'], - 'encoder': ['OneHotEncoder']} - ) - - ############################################################################ - # Search for an ensemble of machine learning algorithms - # ===================================================== - api.search( - X_train=X_train, - y_train=y_train, - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=150, - func_eval_time_limit_secs=30 - ) - - ############################################################################ - # Print the final ensemble performance - # ==================================== - y_pred = api.predict(X_test) - score = api.score(y_pred, y_test) - print(score) - print(api.show_models()) - - # Print statistics from search - print(api.sprint_statistics()) + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 62-64 + +Data Loading +============ + +.. GENERATED FROM PYTHON SOURCE LINES 64-71 + +.. code-block:: default + + X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) + X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, + y, + random_state=1, + ) + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 72-74 + +Build and fit a classifier with include components +================================================== + +.. GENERATED FROM PYTHON SOURCE LINES 74-80 + +.. code-block:: default + + api = TabularClassificationTask( + search_space_updates=get_search_space_updates(), + include_components={'network_backbone': ['MLPBackbone', 'ResNetBackbone'], + 'encoder': ['OneHotEncoder']} + ) + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 81-83 + +Search for an ensemble of machine learning algorithms +===================================================== + +.. GENERATED FROM PYTHON SOURCE LINES 83-93 + +.. code-block:: default + + api.search( + X_train=X_train.copy(), + y_train=y_train.copy(), + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=150, + func_eval_time_limit_secs=30 + ) + + + + + +.. rst-class:: sphx-glr-script-out + + Out: + + .. code-block:: none + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 94-96 + +Print the final ensemble performance +==================================== + +.. GENERATED FROM PYTHON SOURCE LINES 96-104 + +.. code-block:: default + + y_pred = api.predict(X_test) + score = api.score(y_pred, y_test) + print(score) + print(api.show_models()) + + # Print statistics from search + print(api.sprint_statistics()) + + + + + +.. rst-class:: sphx-glr-script-out + + Out: + + .. code-block:: none + + {'accuracy': 0.8670520231213873} + | | Preprocessing | Estimator | Weight | + |---:|:------------------------------------------------------------------|:----------------------------------------------------------|---------:| + | 0 | None | RFLearner | 0.44 | + | 1 | None | ETLearner | 0.26 | + | 2 | None | LGBMLearner | 0.1 | + | 3 | None | SVMLearner | 0.08 | + | 4 | None | KNNLearner | 0.08 | + | 5 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,MLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | + | 6 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,MLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | + autoPyTorch results: + Dataset name: 5482ec4e-4be2-11ec-877d-1fe05d8e2eef + Optimisation Metric: accuracy + Best validation score: 0.8596491228070176 + Number of target algorithm runs: 16 + Number of successful target algorithm runs: 10 + Number of crashed target algorithm runs: 4 + Number of target algorithms that exceeded the time limit: 2 + Number of target algorithms that exceeded the memory limit: 0 + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 105-107 + +Build and fit a classifier with exclude components +================================================== + +.. GENERATED FROM PYTHON SOURCE LINES 107-113 + +.. code-block:: default + + api = TabularClassificationTask( + search_space_updates=get_search_space_updates(), + exclude_components={'network_backbone': ['MLPBackbone'], + 'encoder': ['OneHotEncoder']} + ) + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 114-116 + +Search for an ensemble of machine learning algorithms +===================================================== + +.. GENERATED FROM PYTHON SOURCE LINES 116-126 + +.. code-block:: default + + api.search( + X_train=X_train, + y_train=y_train, + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=150, + func_eval_time_limit_secs=30 + ) + + + + + +.. rst-class:: sphx-glr-script-out + + Out: + + .. code-block:: none + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 127-129 + +Print the final ensemble performance +==================================== + +.. GENERATED FROM PYTHON SOURCE LINES 129-136 + +.. code-block:: default + + y_pred = api.predict(X_test) + score = api.score(y_pred, y_test) + print(score) + print(api.show_models()) + + # Print statistics from search + print(api.sprint_statistics()) + + + + +.. rst-class:: sphx-glr-script-out + + Out: + + .. code-block:: none + + {'accuracy': 0.8728323699421965} + | | Preprocessing | Estimator | Weight | + |---:|:--------------------------------------------------------------|:----------------------------------------------------------------|---------:| + | 0 | None | RFLearner | 0.34 | + | 1 | None | ETLearner | 0.26 | + | 2 | None | LGBMLearner | 0.2 | + | 3 | None | KNNLearner | 0.08 | + | 4 | SimpleImputer,NoEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.06 | + | 5 | None | SVMLearner | 0.04 | + | 6 | SimpleImputer,NoEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | + autoPyTorch results: + Dataset name: c0472c05-4be2-11ec-877d-1fe05d8e2eef + Optimisation Metric: accuracy + Best validation score: 0.8596491228070176 + Number of target algorithm runs: 20 + Number of successful target algorithm runs: 13 + Number of crashed target algorithm runs: 6 + Number of target algorithms that exceeded the time limit: 1 + Number of target algorithms that exceeded the memory limit: 0 + + + + .. rst-class:: sphx-glr-timing - **Total running time of the script:** ( 5 minutes 45.859 seconds) + **Total running time of the script:** ( 5 minutes 57.148 seconds) .. _sphx_glr_download_examples_40_advanced_example_custom_configuration_space.py: diff --git a/development/_sources/examples/40_advanced/example_parallel_n_jobs.rst.txt b/development/_sources/examples/40_advanced/example_parallel_n_jobs.rst.txt index 453fac4c9..c5dd6f475 100644 --- a/development/_sources/examples/40_advanced/example_parallel_n_jobs.rst.txt +++ b/development/_sources/examples/40_advanced/example_parallel_n_jobs.rst.txt @@ -18,14 +18,14 @@ .. _sphx_glr_examples_40_advanced_example_parallel_n_jobs.py: -====================== -Tabular Classification -====================== +============================================ +Tabular Classification with n parallel jobs +============================================ The following example shows how to fit a sample classification model parallely on 2 cores with AutoPyTorch -.. GENERATED FROM PYTHON SOURCE LINES 9-69 +.. GENERATED FROM PYTHON SOURCE LINES 10-70 @@ -36,196 +36,17 @@ with AutoPyTorch .. code-block:: none - [TrajEntry(train_perf=2147483648, incumbent_id=1, incumbent=Configuration: - data_loader:batch_size, Value: 64 - encoder:__choice__, Value: 'OneHotEncoder' - feature_preprocessor:__choice__, Value: 'NoFeaturePreprocessor' - imputer:categorical_strategy, Value: 'most_frequent' - imputer:numerical_strategy, Value: 'mean' - lr_scheduler:ReduceLROnPlateau:factor, Value: 0.1 - lr_scheduler:ReduceLROnPlateau:mode, Value: 'min' - lr_scheduler:ReduceLROnPlateau:patience, Value: 10 - lr_scheduler:__choice__, Value: 'ReduceLROnPlateau' - network_backbone:ShapedMLPBackbone:activation, Value: 'relu' - network_backbone:ShapedMLPBackbone:max_units, Value: 200 - network_backbone:ShapedMLPBackbone:mlp_shape, Value: 'funnel' - network_backbone:ShapedMLPBackbone:num_groups, Value: 5 - network_backbone:ShapedMLPBackbone:output_dim, Value: 200 - network_backbone:ShapedMLPBackbone:use_dropout, Value: False - network_backbone:__choice__, Value: 'ShapedMLPBackbone' - network_embedding:__choice__, Value: 'NoEmbedding' - network_head:__choice__, Value: 'fully_connected' - network_head:fully_connected:activation, Value: 'relu' - network_head:fully_connected:num_layers, Value: 2 - network_head:fully_connected:units_layer_1, Value: 128 - network_init:XavierInit:bias_strategy, Value: 'Normal' - network_init:__choice__, Value: 'XavierInit' - optimizer:AdamOptimizer:beta1, Value: 0.9 - optimizer:AdamOptimizer:beta2, Value: 0.9 - optimizer:AdamOptimizer:lr, Value: 0.01 - optimizer:AdamOptimizer:weight_decay, Value: 0.0 - optimizer:__choice__, Value: 'AdamOptimizer' - scaler:__choice__, Value: 'StandardScaler' - trainer:StandardTrainer:weighted_loss, Value: True - trainer:__choice__, Value: 'StandardTrainer' - , ta_runs=0, ta_time_used=0.0, wallclock_time=0.001379251480102539, budget=0), TrajEntry(train_perf=0.4444444444444444, incumbent_id=1, incumbent=Configuration: - data_loader:batch_size, Value: 142 - encoder:__choice__, Value: 'NoEncoder' - feature_preprocessor:PowerTransformer:standardize, Value: True - feature_preprocessor:__choice__, Value: 'PowerTransformer' - imputer:categorical_strategy, Value: 'constant_!missing!' - imputer:numerical_strategy, Value: 'median' - lr_scheduler:__choice__, Value: 'NoScheduler' - network_backbone:ShapedResNetBackbone:activation, Value: 'relu' - network_backbone:ShapedResNetBackbone:blocks_per_group, Value: 1 - network_backbone:ShapedResNetBackbone:max_shake_drop_probability, Value: 0.7993610769045779 - network_backbone:ShapedResNetBackbone:max_units, Value: 175 - network_backbone:ShapedResNetBackbone:num_groups, Value: 3 - network_backbone:ShapedResNetBackbone:output_dim, Value: 550 - network_backbone:ShapedResNetBackbone:resnet_shape, Value: 'funnel' - network_backbone:ShapedResNetBackbone:use_dropout, Value: False - network_backbone:ShapedResNetBackbone:use_shake_drop, Value: True - network_backbone:ShapedResNetBackbone:use_shake_shake, Value: True - network_backbone:__choice__, Value: 'ShapedResNetBackbone' - network_embedding:__choice__, Value: 'NoEmbedding' - network_head:__choice__, Value: 'fully_connected' - network_head:fully_connected:activation, Value: 'sigmoid' - network_head:fully_connected:num_layers, Value: 2 - network_head:fully_connected:units_layer_1, Value: 308 - network_init:KaimingInit:bias_strategy, Value: 'Normal' - network_init:__choice__, Value: 'KaimingInit' - optimizer:AdamOptimizer:beta1, Value: 0.8660298289969375 - optimizer:AdamOptimizer:beta2, Value: 0.9517157453274235 - optimizer:AdamOptimizer:lr, Value: 1.0377748473731365e-05 - optimizer:AdamOptimizer:weight_decay, Value: 0.07437634123996516 - optimizer:__choice__, Value: 'AdamOptimizer' - scaler:Normalizer:norm, Value: 'mean_abs' - scaler:__choice__, Value: 'Normalizer' - trainer:MixUpTrainer:alpha, Value: 0.13179357367568267 - trainer:MixUpTrainer:weighted_loss, Value: True - trainer:__choice__, Value: 'MixUpTrainer' - , ta_runs=1, ta_time_used=4.371152400970459, wallclock_time=5.563146114349365, budget=5.555555555555555), TrajEntry(train_perf=0.1578947368421053, incumbent_id=2, incumbent=Configuration: - data_loader:batch_size, Value: 64 - encoder:__choice__, Value: 'OneHotEncoder' - feature_preprocessor:__choice__, Value: 'NoFeaturePreprocessor' - imputer:categorical_strategy, Value: 'most_frequent' - imputer:numerical_strategy, Value: 'mean' - lr_scheduler:ReduceLROnPlateau:factor, Value: 0.1 - lr_scheduler:ReduceLROnPlateau:mode, Value: 'min' - lr_scheduler:ReduceLROnPlateau:patience, Value: 10 - lr_scheduler:__choice__, Value: 'ReduceLROnPlateau' - network_backbone:ShapedMLPBackbone:activation, Value: 'relu' - network_backbone:ShapedMLPBackbone:max_units, Value: 200 - network_backbone:ShapedMLPBackbone:mlp_shape, Value: 'funnel' - network_backbone:ShapedMLPBackbone:num_groups, Value: 5 - network_backbone:ShapedMLPBackbone:output_dim, Value: 200 - network_backbone:ShapedMLPBackbone:use_dropout, Value: False - network_backbone:__choice__, Value: 'ShapedMLPBackbone' - network_embedding:__choice__, Value: 'NoEmbedding' - network_head:__choice__, Value: 'fully_connected' - network_head:fully_connected:activation, Value: 'relu' - network_head:fully_connected:num_layers, Value: 2 - network_head:fully_connected:units_layer_1, Value: 128 - network_init:XavierInit:bias_strategy, Value: 'Normal' - network_init:__choice__, Value: 'XavierInit' - optimizer:AdamOptimizer:beta1, Value: 0.9 - optimizer:AdamOptimizer:beta2, Value: 0.9 - optimizer:AdamOptimizer:lr, Value: 0.01 - optimizer:AdamOptimizer:weight_decay, Value: 0.0 - optimizer:__choice__, Value: 'AdamOptimizer' - scaler:__choice__, Value: 'StandardScaler' - trainer:StandardTrainer:weighted_loss, Value: True - trainer:__choice__, Value: 'StandardTrainer' - , ta_runs=2, ta_time_used=9.677488327026367, wallclock_time=6.544711351394653, budget=5.555555555555555), TrajEntry(train_perf=0.14035087719298245, incumbent_id=3, incumbent=Configuration: - data_loader:batch_size, Value: 32 - encoder:__choice__, Value: 'OneHotEncoder' - feature_preprocessor:PowerTransformer:standardize, Value: False - feature_preprocessor:__choice__, Value: 'PowerTransformer' - imputer:categorical_strategy, Value: 'most_frequent' - imputer:numerical_strategy, Value: 'median' - lr_scheduler:CosineAnnealingLR:T_max, Value: 103 - lr_scheduler:__choice__, Value: 'CosineAnnealingLR' - network_backbone:MLPBackbone:activation, Value: 'sigmoid' - network_backbone:MLPBackbone:num_groups, Value: 2 - network_backbone:MLPBackbone:num_units_1, Value: 464 - network_backbone:MLPBackbone:num_units_2, Value: 434 - network_backbone:MLPBackbone:use_dropout, Value: False - network_backbone:__choice__, Value: 'MLPBackbone' - network_embedding:__choice__, Value: 'NoEmbedding' - network_head:__choice__, Value: 'fully_connected' - network_head:fully_connected:activation, Value: 'sigmoid' - network_head:fully_connected:num_layers, Value: 3 - network_head:fully_connected:units_layer_1, Value: 434 - network_head:fully_connected:units_layer_2, Value: 118 - network_init:KaimingInit:bias_strategy, Value: 'Zero' - network_init:__choice__, Value: 'KaimingInit' - optimizer:AdamWOptimizer:beta1, Value: 0.9175282300716818 - optimizer:AdamWOptimizer:beta2, Value: 0.9148484215934113 - optimizer:AdamWOptimizer:lr, Value: 0.0010242582346087676 - optimizer:AdamWOptimizer:weight_decay, Value: 0.07425046847093834 - optimizer:__choice__, Value: 'AdamWOptimizer' - scaler:__choice__, Value: 'StandardScaler' - trainer:MixUpTrainer:alpha, Value: 0.9660122415558304 - trainer:MixUpTrainer:weighted_loss, Value: True - trainer:__choice__, Value: 'MixUpTrainer' - , ta_runs=6, ta_time_used=27.81866765022278, wallclock_time=30.25464129447937, budget=5.555555555555555), TrajEntry(train_perf=0.1286549707602339, incumbent_id=4, incumbent=Configuration: - data_loader:batch_size, Value: 268 - encoder:__choice__, Value: 'OneHotEncoder' - feature_preprocessor:TruncatedSVD:target_dim, Value: 5 - feature_preprocessor:__choice__, Value: 'TruncatedSVD' - imputer:categorical_strategy, Value: 'constant_!missing!' - imputer:numerical_strategy, Value: 'mean' - lr_scheduler:CosineAnnealingLR:T_max, Value: 290 - lr_scheduler:__choice__, Value: 'CosineAnnealingLR' - network_backbone:MLPBackbone:activation, Value: 'tanh' - network_backbone:MLPBackbone:num_groups, Value: 9 - network_backbone:MLPBackbone:num_units_1, Value: 854 - network_backbone:MLPBackbone:num_units_2, Value: 409 - network_backbone:MLPBackbone:num_units_3, Value: 904 - network_backbone:MLPBackbone:num_units_4, Value: 89 - network_backbone:MLPBackbone:num_units_5, Value: 875 - network_backbone:MLPBackbone:num_units_6, Value: 352 - network_backbone:MLPBackbone:num_units_7, Value: 687 - network_backbone:MLPBackbone:num_units_8, Value: 863 - network_backbone:MLPBackbone:num_units_9, Value: 422 - network_backbone:MLPBackbone:use_dropout, Value: False - network_backbone:__choice__, Value: 'MLPBackbone' - network_embedding:__choice__, Value: 'NoEmbedding' - network_head:__choice__, Value: 'fully_connected' - network_head:fully_connected:activation, Value: 'relu' - network_head:fully_connected:num_layers, Value: 2 - network_head:fully_connected:units_layer_1, Value: 229 - network_init:KaimingInit:bias_strategy, Value: 'Zero' - network_init:__choice__, Value: 'KaimingInit' - optimizer:AdamOptimizer:beta1, Value: 0.8670243442016202 - optimizer:AdamOptimizer:beta2, Value: 0.9877701057476191 - optimizer:AdamOptimizer:lr, Value: 0.0005235907018237285 - optimizer:AdamOptimizer:weight_decay, Value: 0.061106742612507106 - optimizer:__choice__, Value: 'AdamOptimizer' - scaler:Normalizer:norm, Value: 'mean_abs' - scaler:__choice__, Value: 'Normalizer' - trainer:StandardTrainer:weighted_loss, Value: False - trainer:__choice__, Value: 'StandardTrainer' - , ta_runs=25, ta_time_used=190.4385905265808, wallclock_time=151.50818800926208, budget=50.0)] - {'accuracy': 0.861271676300578} - | | Preprocessing | Estimator | Weight | - |---:|:------------------------------------------------------------------|:----------------------------------------------------------------|---------:| - | 0 | None | SVMLearner | 0.12 | - | 1 | SimpleImputer,OneHotEncoder,StandardScaler,PolynomialFeatures | embedding,MLPBackbone,FullyConnectedHead,nn.Sequential | 0.1 | - | 2 | SimpleImputer,OneHotEncoder,StandardScaler,PowerTransformer | embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.1 | - | 3 | SimpleImputer,OneHotEncoder,StandardScaler,PowerTransformer | no embedding,MLPBackbone,FullyConnectedHead,nn.Sequential | 0.08 | - | 4 | SimpleImputer,OneHotEncoder,StandardScaler,PowerTransformer | no embedding,MLPBackbone,FullyConnectedHead,nn.Sequential | 0.08 | - | 5 | SimpleImputer,OneHotEncoder,NoScaler,TruncSVD | embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.08 | - | 6 | SimpleImputer,OneHotEncoder,NoScaler,Nystroem | embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.08 | - | 7 | SimpleImputer,OneHotEncoder,NoScaler,Nystroem | embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.06 | - | 8 | None | CBLearner | 0.06 | - | 9 | None | RFLearner | 0.06 | - | 10 | None | KNNLearner | 0.06 | - | 11 | None | ETLearner | 0.04 | - | 12 | SimpleImputer,OneHotEncoder,StandardScaler,PowerTransformer | no embedding,MLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | - | 13 | SimpleImputer,OneHotEncoder,NoScaler,PolynomialFeatures | embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | - | 14 | SimpleImputer,NoEncoder,MinMaxScaler,KernelPCA | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | - | 15 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | + {'accuracy': 0.8728323699421965} + autoPyTorch results: + Dataset name: 8cc9600d-4be1-11ec-877d-1fe05d8e2eef + Optimisation Metric: accuracy + Best validation score: 0.8713450292397661 + Number of target algorithm runs: 40 + Number of successful target algorithm runs: 31 + Number of crashed target algorithm runs: 6 + Number of target algorithms that exceeded the time limit: 3 + Number of target algorithms that exceeded the memory limit: 0 + @@ -290,17 +111,17 @@ with AutoPyTorch ############################################################################ # Print the final ensemble performance # ==================================== - print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) # Print the final ensemble built by AutoPyTorch - print(api.show_models()) + print(api.sprint_statistics()) + .. rst-class:: sphx-glr-timing - **Total running time of the script:** ( 5 minutes 43.309 seconds) + **Total running time of the script:** ( 5 minutes 34.761 seconds) .. _sphx_glr_download_examples_40_advanced_example_parallel_n_jobs.py: diff --git a/development/_sources/examples/40_advanced/example_resampling_strategy.rst.txt b/development/_sources/examples/40_advanced/example_resampling_strategy.rst.txt index 910e80b90..7a4da47ed 100644 --- a/development/_sources/examples/40_advanced/example_resampling_strategy.rst.txt +++ b/development/_sources/examples/40_advanced/example_resampling_strategy.rst.txt @@ -27,7 +27,7 @@ with different resampling strategies in AutoPyTorch By default, AutoPyTorch uses Holdout Validation with a 67% train size split. -.. GENERATED FROM PYTHON SOURCE LINES 11-30 +.. GENERATED FROM PYTHON SOURCE LINES 11-29 .. code-block:: default @@ -56,13 +56,17 @@ a 67% train size split. +.. GENERATED FROM PYTHON SOURCE LINES 30-32 -.. GENERATED FROM PYTHON SOURCE LINES 31-33 +Default Resampling Strategy +============================ + +.. GENERATED FROM PYTHON SOURCE LINES 34-36 Data Loading -============ +------------ -.. GENERATED FROM PYTHON SOURCE LINES 33-40 +.. GENERATED FROM PYTHON SOURCE LINES 36-43 .. code-block:: default @@ -80,12 +84,12 @@ Data Loading -.. GENERATED FROM PYTHON SOURCE LINES 41-43 +.. GENERATED FROM PYTHON SOURCE LINES 44-46 Build and fit a classifier with default resampling strategy -=========================================================== +----------------------------------------------------------- -.. GENERATED FROM PYTHON SOURCE LINES 43-52 +.. GENERATED FROM PYTHON SOURCE LINES 46-55 .. code-block:: default @@ -105,12 +109,12 @@ Build and fit a classifier with default resampling strategy -.. GENERATED FROM PYTHON SOURCE LINES 53-55 +.. GENERATED FROM PYTHON SOURCE LINES 56-58 Search for an ensemble of machine learning algorithms -===================================================== +----------------------------------------------------- -.. GENERATED FROM PYTHON SOURCE LINES 55-65 +.. GENERATED FROM PYTHON SOURCE LINES 58-68 .. code-block:: default @@ -135,16 +139,16 @@ Search for an ensemble of machine learning algorithms .. code-block:: none - + -.. GENERATED FROM PYTHON SOURCE LINES 66-68 +.. GENERATED FROM PYTHON SOURCE LINES 69-71 Print the final ensemble performance -==================================== +------------------------------------ -.. GENERATED FROM PYTHON SOURCE LINES 68-77 +.. GENERATED FROM PYTHON SOURCE LINES 71-80 .. code-block:: default @@ -170,32 +174,38 @@ Print the final ensemble performance {'accuracy': 0.8670520231213873} | | Preprocessing | Estimator | Weight | |---:|:--------------------------------------------------------------|:----------------------------------------------------------------|---------:| - | 0 | None | CBLearner | 0.36 | - | 1 | SimpleImputer,OneHotEncoder,StandardScaler,PolynomialFeatures | embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.22 | - | 2 | None | ETLearner | 0.2 | - | 3 | SimpleImputer,OneHotEncoder,Normalizer,KitchenSink | embedding,MLPBackbone,FullyConnectedHead,nn.Sequential | 0.14 | - | 4 | SimpleImputer,OneHotEncoder,Normalizer,KitchenSink | embedding,MLPBackbone,FullyConnectedHead,nn.Sequential | 0.06 | - | 5 | None | RFLearner | 0.02 | + | 0 | None | RFLearner | 0.44 | + | 1 | None | ETLearner | 0.22 | + | 2 | SimpleImputer,OneHotEncoder,StandardScaler,PolynomialFeatures | embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.16 | + | 3 | SimpleImputer,OneHotEncoder,Normalizer,KitchenSink | embedding,MLPBackbone,FullyConnectedHead,nn.Sequential | 0.06 | + | 4 | None | KNNLearner | 0.06 | + | 5 | None | LGBMLearner | 0.04 | + | 6 | None | SVMLearner | 0.02 | autoPyTorch results: - Dataset name: 42a87b39-4bc3-11ec-8727-df8c3c818998 + Dataset name: 2991f4d3-4be3-11ec-877d-1fe05d8e2eef Optimisation Metric: accuracy - Best validation score: 0.8654970760233918 - Number of target algorithm runs: 19 - Number of successful target algorithm runs: 17 + Best validation score: 0.8596491228070176 + Number of target algorithm runs: 16 + Number of successful target algorithm runs: 13 Number of crashed target algorithm runs: 2 - Number of target algorithms that exceeded the time limit: 0 + Number of target algorithms that exceeded the time limit: 1 Number of target algorithms that exceeded the memory limit: 0 -.. GENERATED FROM PYTHON SOURCE LINES 80-82 +.. GENERATED FROM PYTHON SOURCE LINES 83-85 + +Cross validation Resampling Strategy +===================================== + +.. GENERATED FROM PYTHON SOURCE LINES 87-89 Build and fit a classifier with Cross validation resampling strategy -==================================================================== +-------------------------------------------------------------------- -.. GENERATED FROM PYTHON SOURCE LINES 82-87 +.. GENERATED FROM PYTHON SOURCE LINES 89-94 .. code-block:: default @@ -211,15 +221,16 @@ Build and fit a classifier with Cross validation resampling strategy -.. GENERATED FROM PYTHON SOURCE LINES 88-90 +.. GENERATED FROM PYTHON SOURCE LINES 95-97 Search for an ensemble of machine learning algorithms -===================================================== +----------------------------------------------------------------------- -.. GENERATED FROM PYTHON SOURCE LINES 90-100 +.. GENERATED FROM PYTHON SOURCE LINES 97-108 .. code-block:: default + api.search( X_train=X_train, y_train=y_train, @@ -241,16 +252,16 @@ Search for an ensemble of machine learning algorithms .. code-block:: none - + -.. GENERATED FROM PYTHON SOURCE LINES 101-103 +.. GENERATED FROM PYTHON SOURCE LINES 109-111 Print the final ensemble performance -==================================== +------------ -.. GENERATED FROM PYTHON SOURCE LINES 103-112 +.. GENERATED FROM PYTHON SOURCE LINES 111-120 .. code-block:: default @@ -273,17 +284,21 @@ Print the final ensemble performance .. code-block:: none - {'accuracy': 0.884393063583815} - | | Preprocessing | Estimator | Weight | - |---:|:----------------|:------------------------|---------:| - | 0 | None | TabularTraditionalModel | 0.98 | - | 1 | None | TabularTraditionalModel | 0.02 | + {'accuracy': 0.8728323699421965} + | | Preprocessing | Estimator | Weight | + |---:|:------------------------------------------------------------------|:----------------------------------------------------------------|---------:| + | 0 | None | TabularTraditionalModel | 0.42 | + | 1 | None | TabularTraditionalModel | 0.3 | + | 2 | None | TabularTraditionalModel | 0.14 | + | 3 | None | TabularTraditionalModel | 0.1 | + | 4 | None | TabularTraditionalModel | 0.02 | + | 5 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | autoPyTorch results: - Dataset name: aad5970e-4bc3-11ec-8727-df8c3c818998 + Dataset name: 927e5856-4be3-11ec-877d-1fe05d8e2eef Optimisation Metric: accuracy - Best validation score: 0.8742452475491833 - Number of target algorithm runs: 14 - Number of successful target algorithm runs: 11 + Best validation score: 0.8626733083495604 + Number of target algorithm runs: 12 + Number of successful target algorithm runs: 9 Number of crashed target algorithm runs: 2 Number of target algorithms that exceeded the time limit: 1 Number of target algorithms that exceeded the memory limit: 0 @@ -292,12 +307,17 @@ Print the final ensemble performance -.. GENERATED FROM PYTHON SOURCE LINES 115-117 +.. GENERATED FROM PYTHON SOURCE LINES 123-125 + +Stratified Resampling Strategy +=============================== + +.. GENERATED FROM PYTHON SOURCE LINES 127-129 Build and fit a classifier with Stratified resampling strategy -============================================================== +-------------------------------------------------------------- -.. GENERATED FROM PYTHON SOURCE LINES 117-125 +.. GENERATED FROM PYTHON SOURCE LINES 129-137 .. code-block:: default @@ -316,12 +336,12 @@ Build and fit a classifier with Stratified resampling strategy -.. GENERATED FROM PYTHON SOURCE LINES 126-128 +.. GENERATED FROM PYTHON SOURCE LINES 138-140 Search for an ensemble of machine learning algorithms -===================================================== +----------------------------------------------------- -.. GENERATED FROM PYTHON SOURCE LINES 128-138 +.. GENERATED FROM PYTHON SOURCE LINES 140-150 .. code-block:: default @@ -346,16 +366,16 @@ Search for an ensemble of machine learning algorithms .. code-block:: none - + -.. GENERATED FROM PYTHON SOURCE LINES 139-141 +.. GENERATED FROM PYTHON SOURCE LINES 151-153 Print the final ensemble performance ==================================== -.. GENERATED FROM PYTHON SOURCE LINES 141-149 +.. GENERATED FROM PYTHON SOURCE LINES 153-161 .. code-block:: default @@ -380,21 +400,23 @@ Print the final ensemble performance {'accuracy': 0.861271676300578} | | Preprocessing | Estimator | Weight | |---:|:------------------------------------------------------------------|:----------------------------------------------------------------|---------:| - | 0 | None | LGBMLearner | 0.34 | - | 1 | SimpleImputer,OneHotEncoder,Normalizer,KitchenSink | embedding,MLPBackbone,FullyConnectedHead,nn.Sequential | 0.22 | - | 2 | None | CBLearner | 0.14 | - | 3 | SimpleImputer,OneHotEncoder,StandardScaler,PolynomialFeatures | embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.12 | - | 4 | None | ETLearner | 0.08 | - | 5 | None | SVMLearner | 0.06 | - | 6 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | - | 7 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | + | 0 | None | RFLearner | 0.28 | + | 1 | SimpleImputer,OneHotEncoder,StandardScaler,PolynomialFeatures | embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.2 | + | 2 | None | KNNLearner | 0.16 | + | 3 | None | SVMLearner | 0.1 | + | 4 | SimpleImputer,OneHotEncoder,Normalizer,KitchenSink | embedding,MLPBackbone,FullyConnectedHead,nn.Sequential | 0.08 | + | 5 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.06 | + | 6 | None | LGBMLearner | 0.04 | + | 7 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.04 | + | 8 | None | ETLearner | 0.02 | + | 9 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | autoPyTorch results: - Dataset name: 1e21343e-4bc4-11ec-8727-df8c3c818998 + Dataset name: 050d416d-4be4-11ec-877d-1fe05d8e2eef Optimisation Metric: accuracy - Best validation score: 0.847953216374269 + Best validation score: 0.8362573099415205 Number of target algorithm runs: 18 - Number of successful target algorithm runs: 15 - Number of crashed target algorithm runs: 2 + Number of successful target algorithm runs: 14 + Number of crashed target algorithm runs: 3 Number of target algorithms that exceeded the time limit: 1 Number of target algorithms that exceeded the memory limit: 0 @@ -405,7 +427,7 @@ Print the final ensemble performance .. rst-class:: sphx-glr-timing - **Total running time of the script:** ( 9 minutes 1.230 seconds) + **Total running time of the script:** ( 9 minutes 0.433 seconds) .. _sphx_glr_download_examples_40_advanced_example_resampling_strategy.py: diff --git a/development/_sources/examples/40_advanced/example_run_with_portfolio.rst.txt b/development/_sources/examples/40_advanced/example_run_with_portfolio.rst.txt index b5c067e3b..a79d35cde 100644 --- a/development/_sources/examples/40_advanced/example_run_with_portfolio.rst.txt +++ b/development/_sources/examples/40_advanced/example_run_with_portfolio.rst.txt @@ -25,7 +25,101 @@ Tabular Classification with Greedy Portfolio The following example shows how to fit a sample classification model with AutoPyTorch using the greedy portfolio -.. GENERATED FROM PYTHON SOURCE LINES 9-74 +.. GENERATED FROM PYTHON SOURCE LINES 9-27 + +.. code-block:: default + + import os + import tempfile as tmp + import warnings + + os.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir() + os.environ['OMP_NUM_THREADS'] = '1' + os.environ['OPENBLAS_NUM_THREADS'] = '1' + os.environ['MKL_NUM_THREADS'] = '1' + + warnings.simplefilter(action='ignore', category=UserWarning) + warnings.simplefilter(action='ignore', category=FutureWarning) + + import sklearn.datasets + import sklearn.model_selection + + from autoPyTorch.api.tabular_classification import TabularClassificationTask + + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 28-30 + +Data Loading +============ + +.. GENERATED FROM PYTHON SOURCE LINES 30-37 + +.. code-block:: default + + X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) + X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, + y, + random_state=42, + ) + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 38-40 + +Build and fit a classifier +========================== + +.. GENERATED FROM PYTHON SOURCE LINES 40-44 + +.. code-block:: default + + api = TabularClassificationTask( + seed=42, + ) + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 45-47 + +Search for an ensemble of machine learning algorithms +===================================================== + +.. GENERATED FROM PYTHON SOURCE LINES 47-61 + +.. code-block:: default + + api.search( + X_train=X_train, + y_train=y_train, + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=300, + func_eval_time_limit_secs=50, + # Setting this option to "greedy" + # will make smac run the configurations + # present in 'autoPyTorch/configs/greedy_portfolio.json' + portfolio_selection="greedy" + ) + @@ -36,112 +130,68 @@ with AutoPyTorch using the greedy portfolio .. code-block:: none - {'accuracy': 0.861271676300578} - | | Preprocessing | Estimator | Weight | - |---:|:------------------------------------------------------------------|:-------------------------------------------------------------------|---------:| - | 0 | None | CBLearner | 0.34 | - | 1 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.12 | - | 2 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.1 | - | 3 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.1 | - | 4 | None | RFLearner | 0.08 | - | 5 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.08 | - | 6 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.04 | - | 7 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.04 | - | 8 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.02 | - | 9 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.02 | - | 10 | SimpleImputer,OneHotEncoder,StandardScaler,PolynomialFeatures | embedding,ResNetBackbone,FullyConnectedHead,nn.Sequential | 0.02 | - | 11 | SimpleImputer,OneHotEncoder,NoScaler,NoFeaturePreprocessing | embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.02 | - | 12 | None | ETLearner | 0.02 | - autoPyTorch results: - Dataset name: 81fb4553-4bc2-11ec-8727-df8c3c818998 - Optimisation Metric: accuracy - Best validation score: 0.8771929824561403 - Number of target algorithm runs: 31 - Number of successful target algorithm runs: 28 - Number of crashed target algorithm runs: 3 - Number of target algorithms that exceeded the time limit: 0 - Number of target algorithms that exceeded the memory limit: 0 + +.. GENERATED FROM PYTHON SOURCE LINES 62-64 +Print the final ensemble performance +==================================== - -| +.. GENERATED FROM PYTHON SOURCE LINES 64-72 .. code-block:: default - import os - import tempfile as tmp - import warnings + y_pred = api.predict(X_test) + score = api.score(y_pred, y_test) + print(score) + # Print the final ensemble built by AutoPyTorch + print(api.show_models()) - os.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir() - os.environ['OMP_NUM_THREADS'] = '1' - os.environ['OPENBLAS_NUM_THREADS'] = '1' - os.environ['MKL_NUM_THREADS'] = '1' + # Print statistics from search + print(api.sprint_statistics()) - warnings.simplefilter(action='ignore', category=UserWarning) - warnings.simplefilter(action='ignore', category=FutureWarning) - import sklearn.datasets - import sklearn.model_selection - from autoPyTorch.api.tabular_classification import TabularClassificationTask + +.. rst-class:: sphx-glr-script-out + + Out: + + .. code-block:: none + + {'accuracy': 0.8670520231213873} + | | Preprocessing | Estimator | Weight | + |---:|:------------------------------------------------------------------|:-------------------------------------------------------------------|---------:| + | 0 | None | CBLearner | 0.54 | + | 1 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.12 | + | 2 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential | 0.08 | + | 3 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.08 | + | 4 | None | RFLearner | 0.06 | + | 5 | None | ETLearner | 0.04 | + | 6 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.04 | + | 7 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.02 | + | 8 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedResNetBackbone,FullyConnectedHead,nn.Sequential | 0.02 | + autoPyTorch results: + Dataset name: c9873975-4be0-11ec-877d-1fe05d8e2eef + Optimisation Metric: accuracy + Best validation score: 0.8771929824561403 + Number of target algorithm runs: 23 + Number of successful target algorithm runs: 20 + Number of crashed target algorithm runs: 1 + Number of target algorithms that exceeded the time limit: 2 + Number of target algorithms that exceeded the memory limit: 0 + + - if __name__ == '__main__': - - ############################################################################ - # Data Loading - # ============ - X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) - X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - X, - y, - random_state=42, - ) - - ############################################################################ - # Build and fit a classifier - # ========================== - api = TabularClassificationTask( - seed=42, - ) - - ############################################################################ - # Search for an ensemble of machine learning algorithms - # ===================================================== - api.search( - X_train=X_train, - y_train=y_train, - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=300, - func_eval_time_limit_secs=50, - # Setting this option to "greedy" - # will make smac run the configurations - # present in 'autoPyTorch/configs/greedy_portfolio.json' - portfolio_selection="greedy" - ) - - ############################################################################ - # Print the final ensemble performance - # ==================================== - y_pred = api.predict(X_test) - score = api.score(y_pred, y_test) - print(score) - # Print the final ensemble built by AutoPyTorch - print(api.show_models()) - - # Print statistics from search - print(api.sprint_statistics()) .. rst-class:: sphx-glr-timing - **Total running time of the script:** ( 5 minutes 22.967 seconds) + **Total running time of the script:** ( 5 minutes 27.265 seconds) .. _sphx_glr_download_examples_40_advanced_example_run_with_portfolio.py: diff --git a/development/_sources/examples/40_advanced/example_visualization.rst.txt b/development/_sources/examples/40_advanced/example_visualization.rst.txt index defbe3f43..ca72a45d4 100644 --- a/development/_sources/examples/40_advanced/example_visualization.rst.txt +++ b/development/_sources/examples/40_advanced/example_visualization.rst.txt @@ -135,7 +135,7 @@ Build and fit a classifier .. code-block:: none - + @@ -272,7 +272,7 @@ Plotting the model performance .. rst-class:: sphx-glr-timing - **Total running time of the script:** ( 4 minutes 11.809 seconds) + **Total running time of the script:** ( 3 minutes 47.282 seconds) .. _sphx_glr_download_examples_40_advanced_example_visualization.py: diff --git a/development/_sources/examples/40_advanced/sg_execution_times.rst.txt b/development/_sources/examples/40_advanced/sg_execution_times.rst.txt index 8e94f9bcb..7eb25058b 100644 --- a/development/_sources/examples/40_advanced/sg_execution_times.rst.txt +++ b/development/_sources/examples/40_advanced/sg_execution_times.rst.txt @@ -5,16 +5,16 @@ Computation times ================= -**30:05.174** total execution time for **examples_40_advanced** files: +**29:46.891** total execution time for **examples_40_advanced** files: +------------------------------------------------------------------------------------------------------------------------+-----------+--------+ -| :ref:`sphx_glr_examples_40_advanced_example_resampling_strategy.py` (``example_resampling_strategy.py``) | 09:01.230 | 0.0 MB | +| :ref:`sphx_glr_examples_40_advanced_example_resampling_strategy.py` (``example_resampling_strategy.py``) | 09:00.433 | 0.0 MB | +------------------------------------------------------------------------------------------------------------------------+-----------+--------+ -| :ref:`sphx_glr_examples_40_advanced_example_custom_configuration_space.py` (``example_custom_configuration_space.py``) | 05:45.859 | 0.0 MB | +| :ref:`sphx_glr_examples_40_advanced_example_custom_configuration_space.py` (``example_custom_configuration_space.py``) | 05:57.148 | 0.0 MB | +------------------------------------------------------------------------------------------------------------------------+-----------+--------+ -| :ref:`sphx_glr_examples_40_advanced_example_parallel_n_jobs.py` (``example_parallel_n_jobs.py``) | 05:43.309 | 0.0 MB | +| :ref:`sphx_glr_examples_40_advanced_example_parallel_n_jobs.py` (``example_parallel_n_jobs.py``) | 05:34.761 | 0.0 MB | +------------------------------------------------------------------------------------------------------------------------+-----------+--------+ -| :ref:`sphx_glr_examples_40_advanced_example_run_with_portfolio.py` (``example_run_with_portfolio.py``) | 05:22.967 | 0.0 MB | +| :ref:`sphx_glr_examples_40_advanced_example_run_with_portfolio.py` (``example_run_with_portfolio.py``) | 05:27.265 | 0.0 MB | +------------------------------------------------------------------------------------------------------------------------+-----------+--------+ -| :ref:`sphx_glr_examples_40_advanced_example_visualization.py` (``example_visualization.py``) | 04:11.809 | 0.0 MB | +| :ref:`sphx_glr_examples_40_advanced_example_visualization.py` (``example_visualization.py``) | 03:47.282 | 0.0 MB | +------------------------------------------------------------------------------------------------------------------------+-----------+--------+ diff --git a/development/_sources/examples/index.rst.txt b/development/_sources/examples/index.rst.txt index ace8d26a0..b0abcbcc5 100644 --- a/development/_sources/examples/index.rst.txt +++ b/development/_sources/examples/index.rst.txt @@ -115,14 +115,14 @@ Advanced examples for using *Auto-PyTorch* on tabular datasets. .. raw:: html -
+
.. only:: html - .. figure:: /examples/40_advanced/images/thumb/sphx_glr_example_parallel_n_jobs_thumb.png - :alt: Tabular Classification + .. figure:: /examples/40_advanced/images/thumb/sphx_glr_example_run_with_portfolio_thumb.png + :alt: Tabular Classification with Greedy Portfolio - :ref:`sphx_glr_examples_40_advanced_example_parallel_n_jobs.py` + :ref:`sphx_glr_examples_40_advanced_example_run_with_portfolio.py` .. raw:: html @@ -132,18 +132,18 @@ Advanced examples for using *Auto-PyTorch* on tabular datasets. .. toctree:: :hidden: - /examples/40_advanced/example_parallel_n_jobs + /examples/40_advanced/example_run_with_portfolio .. raw:: html -
+
.. only:: html - .. figure:: /examples/40_advanced/images/thumb/sphx_glr_example_run_with_portfolio_thumb.png - :alt: Tabular Classification with Greedy Portfolio + .. figure:: /examples/40_advanced/images/thumb/sphx_glr_example_parallel_n_jobs_thumb.png + :alt: Tabular Classification with n parallel jobs - :ref:`sphx_glr_examples_40_advanced_example_run_with_portfolio.py` + :ref:`sphx_glr_examples_40_advanced_example_parallel_n_jobs.py` .. raw:: html @@ -153,18 +153,18 @@ Advanced examples for using *Auto-PyTorch* on tabular datasets. .. toctree:: :hidden: - /examples/40_advanced/example_run_with_portfolio + /examples/40_advanced/example_parallel_n_jobs .. raw:: html -
+
.. only:: html - .. figure:: /examples/40_advanced/images/thumb/sphx_glr_example_resampling_strategy_thumb.png - :alt: Tabular Classification with different resampling strategy + .. figure:: /examples/40_advanced/images/thumb/sphx_glr_example_custom_configuration_space_thumb.png + :alt: Tabular Classification with Custom Configuration Space - :ref:`sphx_glr_examples_40_advanced_example_resampling_strategy.py` + :ref:`sphx_glr_examples_40_advanced_example_custom_configuration_space.py` .. raw:: html @@ -174,18 +174,18 @@ Advanced examples for using *Auto-PyTorch* on tabular datasets. .. toctree:: :hidden: - /examples/40_advanced/example_resampling_strategy + /examples/40_advanced/example_custom_configuration_space .. raw:: html -
+
.. only:: html - .. figure:: /examples/40_advanced/images/thumb/sphx_glr_example_custom_configuration_space_thumb.png - :alt: Tabular Classification with Custom Configuration Space + .. figure:: /examples/40_advanced/images/thumb/sphx_glr_example_resampling_strategy_thumb.png + :alt: Tabular Classification with different resampling strategy - :ref:`sphx_glr_examples_40_advanced_example_custom_configuration_space.py` + :ref:`sphx_glr_examples_40_advanced_example_resampling_strategy.py` .. raw:: html @@ -195,7 +195,7 @@ Advanced examples for using *Auto-PyTorch* on tabular datasets. .. toctree:: :hidden: - /examples/40_advanced/example_custom_configuration_space + /examples/40_advanced/example_resampling_strategy .. raw:: html diff --git a/development/examples/20_basics/example_image_classification.html b/development/examples/20_basics/example_image_classification.html index bb4d22290..58ea9a3ec 100644 --- a/development/examples/20_basics/example_image_classification.html +++ b/development/examples/20_basics/example_image_classification.html @@ -30332,16 +30332,17 @@ ________________________________________ Configuration: image_augmenter:GaussianBlur:use_augmenter, Value: False - image_augmenter:GaussianNoise:use_augmenter, Value: False - image_augmenter:RandomAffine:rotate, Value: 65 - image_augmenter:RandomAffine:scale_offset, Value: 0.10364752234694663 - image_augmenter:RandomAffine:shear, Value: 5 - image_augmenter:RandomAffine:translate_percent_offset, Value: 0.12303023778688212 + image_augmenter:GaussianNoise:sigma_offset, Value: 2.939416777169848 + image_augmenter:GaussianNoise:use_augmenter, Value: True + image_augmenter:RandomAffine:rotate, Value: 182 + image_augmenter:RandomAffine:scale_offset, Value: 0.2653043460236015 + image_augmenter:RandomAffine:shear, Value: 7 + image_augmenter:RandomAffine:translate_percent_offset, Value: 0.2981323677443226 image_augmenter:RandomAffine:use_augmenter, Value: True - image_augmenter:RandomCutout:p, Value: 0.9516838776905963 + image_augmenter:RandomCutout:p, Value: 0.46400105311756495 image_augmenter:RandomCutout:use_augmenter, Value: True image_augmenter:Resize:use_augmenter, Value: False - image_augmenter:ZeroPadAndCrop:percent, Value: 0.2732380932749452 + image_augmenter:ZeroPadAndCrop:percent, Value: 0.05116366366228997 normalizer:__choice__, Value: 'ImageNormalizer' Fitting the pipeline... @@ -30414,7 +30415,7 @@ print(pipeline)
-

Total running time of the script: ( 0 minutes 6.216 seconds)

+

Total running time of the script: ( 0 minutes 7.489 seconds)

Out:

-
<autoPyTorch.api.tabular_classification.TabularClassificationTask object at 0x7f2f6f8c4940>
+
<autoPyTorch.api.tabular_classification.TabularClassificationTask object at 0x7f4c008a98e0>
 

 

 
@@ -204,30 +204,32 @@ 
Print the final ensemble performanceOut:

-
{'accuracy': 0.8554913294797688}
-|    | Preprocessing                                             | Estimator                                                       |   Weight |
-|---:|:----------------------------------------------------------|:----------------------------------------------------------------|---------:|
-|  0 | None                                                      | RFLearner                                                       |     0.24 |
-|  1 | SimpleImputer,OneHotEncoder,Normalizer,KernelPCA          | embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential    |     0.22 |
-|  2 | None                                                      | SVMLearner                                                      |     0.18 |
-|  3 | None                                                      | CBLearner                                                       |     0.1  |
-|  4 | None                                                      | ETLearner                                                       |     0.08 |
-|  5 | None                                                      | KNNLearner                                                      |     0.08 |
-|  6 | SimpleImputer,OneHotEncoder,MinMaxScaler,PowerTransformer | embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential    |     0.04 |
-|  7 | SimpleImputer,NoEncoder,Normalizer,Nystroem               | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential |     0.04 |
-|  8 | None                                                      | LGBMLearner                                                     |     0.02 |
+
{'accuracy': 0.838150289017341}
+|    | Preprocessing                                                     | Estimator                                                       |   Weight |
+|---:|:------------------------------------------------------------------|:----------------------------------------------------------------|---------:|
+|  0 | SimpleImputer,OneHotEncoder,Normalizer,KernelPCA                  | embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential    |     0.2  |
+|  1 | None                                                              | KNNLearner                                                      |     0.16 |
+|  2 | None                                                              | CBLearner                                                       |     0.14 |
+|  3 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential |     0.12 |
+|  4 | SimpleImputer,OneHotEncoder,Normalizer,PowerTransformer           | embedding,ResNetBackbone,FullyConnectedHead,nn.Sequential       |     0.08 |
+|  5 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential |     0.08 |
+|  6 | SimpleImputer,OneHotEncoder,MinMaxScaler,PowerTransformer         | embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential    |     0.06 |
+|  7 | SimpleImputer,NoEncoder,Normalizer,Nystroem                       | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential |     0.06 |
+|  8 | None                                                              | SVMLearner                                                      |     0.04 |
+|  9 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential |     0.04 |
+| 10 | SimpleImputer,OneHotEncoder,MinMaxScaler,TruncSVD                 | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential |     0.02 |
 autoPyTorch results:
         Dataset name: Australian
         Optimisation Metric: accuracy
         Best validation score: 0.8713450292397661
-        Number of target algorithm runs: 23
+        Number of target algorithm runs: 22
         Number of successful target algorithm runs: 19
         Number of crashed target algorithm runs: 2
-        Number of target algorithms that exceeded the time limit: 2
+        Number of target algorithms that exceeded the time limit: 1
         Number of target algorithms that exceeded the memory limit: 0
 

 

-
Total running time of the script: ( 5 minutes  22.150 seconds)

+
Total running time of the script: ( 5 minutes  35.087 seconds)

 
 
Out:

-
<autoPyTorch.api.tabular_regression.TabularRegressionTask object at 0x7f30125fdca0>
+
<autoPyTorch.api.tabular_regression.TabularRegressionTask object at 0x7f4ca3439dc0>
 

 

 
@@ -198,7 +198,7 @@ 
Print the final ensemble performanceOut:

-
{'r2': 0.9445248186059718}
+
{'r2': 0.944631023189658}
 |    | Preprocessing                                                     | Estimator                                                       |   Weight |
 |---:|:------------------------------------------------------------------|:----------------------------------------------------------------|---------:|
 |  0 | SimpleImputer,OneHotEncoder,StandardScaler,NoFeaturePreprocessing | no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential |     0.44 |
@@ -206,9 +206,9 @@ 
Print the final ensemble performanceTotal running time of the script: ( 5 minutes  37.415 seconds)

+
Total running time of the script: ( 5 minutes  36.563 seconds)