Skip to content
This repository has been archived by the owner on Sep 28, 2022. It is now read-only.

Commit

Permalink
[Fault Tolerance] Don't check the len of a dataset, but its instance. (
Browse files Browse the repository at this point in the history
  • Loading branch information
tchaton authored and Raalsky committed Nov 23, 2021
1 parent 893bcee commit 13b8c3b
Showing 1 changed file with 4 additions and 3 deletions.
7 changes: 4 additions & 3 deletions pytorch_lightning/trainer/data_loading.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
CaptureMapDataset,
FastForwardSampler,
)
from pytorch_lightning.utilities.data import has_iterable_dataset, has_len_all_ranks
from pytorch_lightning.utilities.data import get_len, has_iterable_dataset, has_len_all_ranks
from pytorch_lightning.utilities.enums import DistributedType
from pytorch_lightning.utilities.exceptions import MisconfigurationException
from pytorch_lightning.utilities.imports import _fault_tolerant_training
Expand Down Expand Up @@ -282,10 +282,11 @@ def _get_dataloader_init_kwargs(
dl_kwargs["sampler"] = None

if _fault_tolerant_training():
if isinstance(dl_kwargs["dataset"], IterableDataset):
dataset = dl_kwargs["dataset"]
if isinstance(dataset, IterableDataset):
# wrap the `IterableDataset` into a `CaptureIterableDataset` to record sampler states.
dl_kwargs["dataset"] = CaptureIterableDataset(dataset=dl_kwargs["dataset"])
elif len(dl_kwargs["dataset"]):
elif get_len(dataset) != float("inf"):
dl_kwargs["dataset"] = CaptureMapDataset(dataset=dl_kwargs["dataset"])
else:
raise MisconfigurationException(
Expand Down

0 comments on commit 13b8c3b

Please sign in to comment.