Update rp effective seq len sampling

m2kulkarni · Sep 19, 2024 · fd7be7f · fd7be7f
1 parent 9df4a12
commit fd7be7f
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/src/dataloaders/redpajama_sample_contig.py b/src/dataloaders/redpajama_sample_contig.py
@@ -115,8 +115,8 @@ def load_data(name: str, dataset_config: dict, pretrained_model_config: dict,
         _data_attr = '-d='.join(_data_attr).replace('/', '_').replace('.json', '')
         _data_attr = _data_attr.replace('[','_').replace(']','')
 
-        fname = f'd={_data_attr}-nts={num_train_samples}-mts={max_train_samples}-dcs={chunk_size}-max={max_length}-min={min_length}-s={seed}'
-        # fname = f'd={_data_attr}-mts={max_train_samples}-dcs={chunk_size}-max={max_length}-min={min_length}-s={seed}'
+        # fname = f'd={_data_attr}-nts={num_train_samples}-mts={max_train_samples}-dcs={chunk_size}-max={max_length}-min={min_length}-s={seed}'
+        fname = f'd={_data_attr}-mts={max_train_samples}-dcs={chunk_size}-max={max_length}-min={min_length}-s={seed}'
         fname = join(dataset_config['dataloaders_dir'], fname)