quick fix on concatenating text to support more datasets (huggingface…

…#8474)
fabiocapsouza · Nov 15, 2020 · 677c6d9 · 677c6d9
1 parent 4b03705
commit 677c6d9
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/examples/language-modeling/run_clm.py b/examples/language-modeling/run_clm.py
@@ -254,7 +254,7 @@ def tokenize_function(examples):
         tokenize_function,
         batched=True,
         num_proc=data_args.preprocessing_num_workers,
-        remove_columns=[text_column_name],
+        remove_columns=column_names,
         load_from_cache_file=not data_args.overwrite_cache,
     )
 

diff --git a/examples/language-modeling/run_mlm.py b/examples/language-modeling/run_mlm.py
@@ -292,7 +292,7 @@ def tokenize_function(examples):
             tokenize_function,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
-            remove_columns=[text_column_name],
+            remove_columns=column_names,
             load_from_cache_file=not data_args.overwrite_cache,
         )
 

diff --git a/examples/language-modeling/run_plm.py b/examples/language-modeling/run_plm.py
@@ -279,7 +279,7 @@ def tokenize_function(examples):
             tokenize_function,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
-            remove_columns=[text_column_name],
+            remove_columns=column_names,
             load_from_cache_file=not data_args.overwrite_cache,
         )