diff --git a/data_utils.py b/data_utils.py index bff806c..6496e17 100644 --- a/data_utils.py +++ b/data_utils.py @@ -214,6 +214,12 @@ def make_dataset_all(self): @cache def load_dataset_files(self): data_args = self.data_args + if not data_args.convert_file: + return { + "train_files": self.train_files, + "eval_files": self.eval_files, + "test_files": self.test_files, + } filename = os.path.join(data_args.output_dir, 'intermediate_file_index.json') assert os.path.exists(filename), 'make you dataset firstly' with open(filename, mode='r', encoding='utf-8') as f: