Fix bug in dataloader

hemanthkumarak · Oct 18, 2023 · 5120753 · 5120753
1 parent d9e78ad
commit 5120753
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/pretrain/tinyllama_code.py b/pretrain/tinyllama_code.py
@@ -120,7 +120,7 @@ def main(fabric, train_data_dir, val_data_dir, resume):
         fabric=fabric,
         train_data_dir=train_data_dir,
         val_data_dir=val_data_dir,
-        seed=(3407 + fabric.global_rank),
+        seed=3407,
     )
     if val_dataloader is None:
         train_dataloader = fabric.setup_dataloaders(train_dataloader)
@@ -319,7 +319,7 @@ def create_dataloader(
             n_chunks=8,
             block_size=block_size,
             shuffle=shuffle,
-            seed=seed,
+            seed=seed+fabric.global_rank,
             num_processes=fabric.world_size,
             process_rank=fabric.global_rank,
         )