Make backend deletion robust after fit

autosklearn/automl.py

@@ -47,7 +47,7 @@
 from autosklearn.evaluation.abstract_evaluator import _fit_and_suppress_warnings
 from autosklearn.evaluation.train_evaluator import TrainEvaluator, _fit_with_budget
 from autosklearn.metrics import calculate_metric
-from autosklearn.util.backend import Backend
+from autosklearn.util.backend import Backend, create
 from autosklearn.util.stopwatch import StopWatch
 from autosklearn.util.logging_ import (
     setup_logger,
@@ -113,9 +113,10 @@ def send_warnings_to_log(
 class AutoML(BaseEstimator):
 
     def __init__(self,
-                 backend: Backend,
                  time_left_for_this_task,
                  per_run_time_limit,
+                 temporary_directory: Optional[str] = None,
+                 delete_tmp_folder_after_terminate: bool = True,
                  initial_configurations_via_metalearning=25,
                  ensemble_size=1,
                  ensemble_nbest=1,
@@ -141,7 +142,9 @@ def __init__(self,
                  ):
         super(AutoML, self).__init__()
         self.configuration_space = None
-        self._backend = backend
+        self._backend: Optional[Backend] = None
+        self._temporary_directory = temporary_directory
+        self._delete_tmp_folder_after_terminate = delete_tmp_folder_after_terminate
         # self._tmp_dir = tmp_dir
         self._time_for_task = time_left_for_this_task
         self._per_run_time_limit = per_run_time_limit
@@ -228,6 +231,12 @@ def __init__(self,
         # saved to disk
         self.num_run = 0
 
+    def _create_backend(self) -> Backend:
+        return create(
+            temporary_directory=self._temporary_directory,
+            delete_tmp_folder_after_terminate=self._delete_tmp_folder_after_terminate,
+        )
+
     def _create_dask_client(self):
         self._is_dask_client_internally_created = True
         self._dask_client = dask.distributed.Client(
@@ -436,6 +445,9 @@ def fit(
         if dataset_name is None:
             dataset_name = str(uuid.uuid1(clock_seq=os.getpid()))
 
+        # Create the backend
+        self._backend = self._create_backend()
+
         # By default try to use the TCP logging port or get a new port
         self._logger_port = logging.handlers.DEFAULT_TCP_LOGGING_PORT
         self._logger = self._get_logger(dataset_name)
@@ -805,6 +817,10 @@ def _fit_cleanup(self):
         # Clean up the logger
         self._logger.info("Starting to clean up the logger")
         self._clean_logger()
+
+        # Clean up the backend
+        if self._delete_tmp_folder_after_terminate:
+            self._backend.context.delete_directories(force=False)
         return
 
     def _check_resampling_strategy(
@@ -1594,11 +1610,6 @@ def __del__(self):
 
         self._close_dask_client()
 
-        # When a multiprocessing work is done, the
-        # objects are deleted. We don't want to delete run areas
-        # until the estimator is deleted
-        self._backend.context.delete_directories(force=False)
-
 
 class AutoMLClassifier(AutoML):
     def __init__(self, *args, **kwargs):

autosklearn/estimators.py

@@ -18,7 +18,6 @@
 from autosklearn.pipeline.base import BasePipeline
 from autosklearn.automl import AutoMLClassifier, AutoMLRegressor, AutoML
 from autosklearn.metrics import Scorer
-from autosklearn.util.backend import create
 
 
 class AutoSklearnEstimator(BaseEstimator):
@@ -284,13 +283,9 @@ def __getstate__(self):
 
     def build_automl(self):
 
-        backend = create(
+        automl = self._get_automl_class()(
             temporary_directory=self.tmp_folder,
             delete_tmp_folder_after_terminate=self.delete_tmp_folder_after_terminate,
-            )
-
-        automl = self._get_automl_class()(
-            backend=backend,
             time_left_for_this_task=self.time_left_for_this_task,
             per_run_time_limit=self.per_run_time_limit,
             initial_configurations_via_metalearning=self.initial_configurations_via_metalearning,

examples/60_search/example_parallel_manual_spawning_cli.py

@@ -175,6 +175,7 @@ def cli_start_worker(scheduler_file_name):
     X_train, X_test, y_train, y_test = \
         sklearn.model_selection.train_test_split(X, y, random_state=1)
     automl = AutoSklearnClassifier(
+        delete_tmp_folder_after_terminate=False,
         time_left_for_this_task=30,
         per_run_time_limit=10,
         memory_limit=1024,

examples/60_search/example_parallel_manual_spawning_python.py

@@ -123,6 +123,7 @@ async def do_work():
         # 3. Start the client
         with dask.distributed.Client(address=cluster.scheduler_address) as client:
             automl = AutoSklearnClassifier(
+                delete_tmp_folder_after_terminate=False,
                 time_left_for_this_task=30,
                 per_run_time_limit=10,
                 memory_limit=1024,