fix: wrong dataset paths, was using non-tokenized data in pre-tokeniz…

…ed dataset tests Signed-off-by: Harikrishnan Balagopal <harikrishmenon@gmail.com>
foundation-model-stack · Sep 2, 2024 · b6fc949 · b6fc949
1 parent 654bbf1
commit b6fc949
Showing 1 changed file with 2 additions and 3 deletions.
diff --git a/tests/test_sft_trainer.py b/tests/test_sft_trainer.py
@@ -35,7 +35,6 @@
     EMPTY_DATA,
     MALFORMATTED_DATA,
     MODEL_NAME,
-    TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT_JSON,
     TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT_JSONL,
     TWITTER_COMPLAINTS_DATA_JSON,
     TWITTER_COMPLAINTS_DATA_JSONL,
@@ -850,8 +849,8 @@ def test_run_with_good_experimental_metadata():
 @pytest.mark.parametrize(
     "dataset_path",
     [
-        TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT_JSONL,
-        TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT_JSON,
+        TWITTER_COMPLAINTS_TOKENIZED_JSONL,
+        TWITTER_COMPLAINTS_TOKENIZED_JSON,
     ],
 )
 ### Tests for pretokenized data