Small stability and logging fixes (#522)

This concludes the important changes I have on the SIGMOD branch. These
changes are too small to open more PRs for them, so I grouped them in a
single PR.

modyn/evaluator/internal/grpc/evaluator_grpc_server.py

@@ -26,7 +26,7 @@ def __init__(self, modyn_config: dict, tempdir: pathlib.Path) -> None:
         self.tempdir = tempdir
         self.server = grpc.server(
             futures.ThreadPoolExecutor(
-                max_workers=10,
+                max_workers=64,
             ),
             options=[
                 ("grpc.max_receive_message_length", MAX_MESSAGE_SIZE),

modyn/evaluator/internal/pytorch_evaluator.py

@@ -70,7 +70,7 @@ def _prepare_dataloader(self, evaluation_info: EvaluationInfo) -> torch.utils.da
         )
         self._debug("Creating DataLoader.")
         dataloader = torch.utils.data.DataLoader(
-            dataset, batch_size=evaluation_info.batch_size, num_workers=evaluation_info.num_dataloaders
+            dataset, batch_size=evaluation_info.batch_size, num_workers=evaluation_info.num_dataloaders, timeout=60
         )
 
         return dataloader

modyn/supervisor/internal/supervisor.py

@@ -206,7 +206,8 @@ def start_pipeline(
             start_timestamp = self.grpc.get_time_at_storage()
             pipeline_id = self.register_pipeline(pipeline_config_model)
             logger.info(f"Pipeline {pipeline_id} registered, start executing.")
-        except Exception:  # pylint: disable=broad-except
+        except Exception as ex:  # pylint: disable=broad-except
+            logger.error(f"Failed to register pipeline: {ex}")
             return pipeline_res_msg(exception="Failed to register pipeline")
 
         try:

modyn/trainer_server/internal/dataset/data_utils.py

@@ -67,7 +67,7 @@ def prepare_dataloaders(
     )
     logger.debug("Creating DataLoader.")
     train_dataloader = torch.utils.data.DataLoader(
-        train_set, batch_size=batch_size, num_workers=num_dataloaders, drop_last=drop_last
+        train_set, batch_size=batch_size, num_workers=num_dataloaders, drop_last=drop_last, timeout=60
     )
 
     # TODO(#50): what to do with the val set in the general case?

modyn/trainer_server/internal/dataset/key_sources/local_key_source.py

@@ -32,3 +32,12 @@ def get_num_data_partitions(self) -> int:
 
     def end_of_trigger_cleaning(self) -> None:
         self._trigger_sample_storage.clean_trigger_data(self._pipeline_id, self._trigger_id)
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        del state["_trigger_sample_storage"]  # not pickable
+        return state
+
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+        self._trigger_sample_storage = TriggerSampleStorage(self.offline_dataset_path)