rem print in get datastore data

Patrick Wang · Patrick Wang · commit 56a810cfeabb · 2024-03-25T15:17:32.000Z
diff --git a/datastore/get_datastore_chat.py b/datastore/get_datastore_chat.py
@@ -43,7 +43,6 @@
     dataset = load_dataset('Aeala/ShareGPT_Vicuna_unfiltered', split='train')
     total_length = len(dataset)
     for conversations in tqdm(dataset, total=total_length):
-        print('conversations', conversations, type(conversations))
         for sample in conversations['conversations']:
             token_list = tokenizer.encode(sample['value'])
             writer.add_entry(token_list)
diff --git a/requirements.txt b/requirements.txt
@@ -9,3 +9,4 @@ openai
 anthropic
 sentencepiece
 protobuf
+shortuuid
