del

Author: nikg4

src/oumi/core/datasets/base_dataset.py

@@ -211,7 +211,9 @@ def _load_parquet_dataset(self, path: str) -> pd.DataFrame:
 
     def _load_dataset_from_disk(self, path: str) -> pd.DataFrame:
         dataset: datasets.Dataset = datasets.Dataset.load_from_disk(path)
-        return cast(pd.DataFrame, dataset.to_pandas())
+        result = dataset.to_pandas()
+        del dataset
+        return cast(pd.DataFrame, result)
 
 
 class BaseLMSftDataset(BaseMapDataset, ABC):