make file name as a hashing string to aviod long file names (microsof…

…t#274) * make file name as a hashing string to aviod long file names * fix format issue
deep-cognition · Apr 12, 2023 · 5618083 · 5618083
1 parent e320e75
commit 5618083
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/applications/DeepSpeed-Chat/training/utils/data/data_utils.py b/applications/DeepSpeed-Chat/training/utils/data/data_utils.py
@@ -252,6 +252,7 @@ def create_prompt_dataset(local_rank,
     tokenizer_name = tokenizer.init_kwargs['name_or_path'].replace('/', '_')
     fname = f"{fname}_split{data_split}_phase{train_phase}_seed{seed}_tokenizer{tokenizer_name}_seqlen{max_seq_len}"
     fname = '_'.join(fname.split('/'))
+    fname = str(hash(fname))  # hash the file name to avoid too long file name
     train_fname = f"{output_path}/traindata_{fname}.pt"
     eval_fname = f"{output_path}/evaldata_{fname}.pt"