Revamp test selection for the example tests (#23737)

* Revamp test selection for the example tests

* Rename old XLA test and fake modif in run_glue

* Fixes

* Fake Trainer modif

* Remove fake modifs

.circleci/config.yml

@@ -43,6 +43,12 @@ jobs:
                 else
                     touch test_preparation/test_list.txt
                 fi
+            - run: |
+                  if [ -f examples_test_list.txt ]; then
+                      mv examples_test_list.txt test_preparation/examples_test_list.txt
+                  else
+                      touch test_preparation/examples_test_list.txt
+                  fi
             - run: |
                 if [ -f doctest_list.txt ]; then
                     cp doctest_list.txt test_preparation/doctest_list.txt
@@ -62,19 +68,6 @@ jobs:
                 else
                     touch test_preparation/filtered_test_list.txt
                 fi
-            - run: python utils/tests_fetcher.py --filters tests examples | tee examples_tests_fetched_summary.txt
-            - run: |
-                  if [ -f test_list.txt ]; then
-                      mv test_list.txt test_preparation/examples_test_list.txt
-                  else
-                      touch test_preparation/examples_test_list.txt
-                  fi
-            - run: |
-                  if [ -f filtered_test_list_cross_tests.txt ]; then
-                      mv filtered_test_list_cross_tests.txt test_preparation/filtered_test_list_cross_tests.txt
-                  else
-                      touch test_preparation/filtered_test_list_cross_tests.txt
-                  fi
             - store_artifacts:
                   path: test_preparation/test_list.txt
             - store_artifacts:
@@ -111,7 +104,7 @@ jobs:
             - run: |
                   mkdir test_preparation
                   echo -n "tests" > test_preparation/test_list.txt
-                  echo -n "tests" > test_preparation/examples_test_list.txt
+                  echo -n "all" > test_preparation/examples_test_list.txt
                   echo -n "tests/repo_utils" > test_preparation/test_repo_utils.txt
             - run: |
                   echo -n "tests" > test_list.txt

.circleci/create_circleci_config.py

@@ -342,7 +342,6 @@ def job_name(self):
         "pip install .[sklearn,torch,sentencepiece,testing,torch-speech]",
         "pip install -r examples/pytorch/_tests_requirements.txt",
     ],
-    tests_to_run="./examples/pytorch/",
 )
 
 
@@ -355,7 +354,6 @@ def job_name(self):
         "pip install .[sklearn,tensorflow,sentencepiece,testing]",
         "pip install -r examples/tensorflow/_tests_requirements.txt",
     ],
-    tests_to_run="./examples/tensorflow/",
 )
 
 
@@ -367,7 +365,6 @@ def job_name(self):
         "pip install .[flax,testing,sentencepiece]",
         "pip install -r examples/flax/_tests_requirements.txt",
     ],
-    tests_to_run="./examples/flax/",
 )
 
 
@@ -551,7 +548,17 @@ def create_circleci_config(folder=None):
 
     example_file = os.path.join(folder, "examples_test_list.txt")
     if os.path.exists(example_file) and os.path.getsize(example_file) > 0:
-        jobs.extend(EXAMPLES_TESTS)
+        with open(example_file, "r", encoding="utf-8") as f:
+            example_tests = f.read().split(" ")
+        for job in EXAMPLES_TESTS:
+            framework = job.name.replace("examples_", "").replace("torch", "pytorch")
+            if example_tests == "all":
+                job.tests_to_run = [f"examples/{framework}"]
+            else:
+                job.tests_to_run = [f for f in example_tests if f.startswith(f"examples/{framework}")]
+
+            if len(job.tests_to_run) > 0:
+                jobs.append(job)
 
     doctest_file = os.path.join(folder, "doctest_list.txt")
     if os.path.exists(doctest_file):

tests/repo_utils/test_tests_fetcher.py

@@ -42,6 +42,7 @@
     get_module_dependencies,
     get_tree_starting_at,
     infer_tests_to_run,
+    init_test_examples_dependencies,
     parse_commit_message,
     print_tree_deps_of,
 )
@@ -149,7 +150,19 @@ def create_tmp_repo(tmp_dir, models=None):
                 f"from transformers import {cls}Config, {cls}Model\nfrom ...test_modeling_common import ModelTesterMixin\n\ncode"
             )
 
-    repo.index.add(["src", "tests"])
+    example_dir = tmp_dir / "examples"
+    example_dir.mkdir(exist_ok=True)
+    for framework in ["flax", "pytorch", "tensorflow"]:
+        framework_dir = example_dir / framework
+        framework_dir.mkdir(exist_ok=True)
+        with open(framework_dir / f"test_{framework}_examples.py", "w") as f:
+            f.write("""test_args = "run_glue.py"\n""")
+        glue_dir = framework_dir / "text-classification"
+        glue_dir.mkdir(exist_ok=True)
+        with open(glue_dir / "run_glue.py", "w") as f:
+            f.write("from transformers import BertModel\n\ncode")
+
+    repo.index.add(["examples", "src", "tests"])
     repo.index.commit("Initial commit")
     repo.create_head("main")
     repo.head.reference = repo.refs.main
@@ -164,12 +177,14 @@ def patch_transformer_repo_path(new_folder):
     """
     old_repo_path = tests_fetcher.PATH_TO_REPO
     tests_fetcher.PATH_TO_REPO = Path(new_folder).resolve()
+    tests_fetcher.PATH_TO_EXAMPLES = tests_fetcher.PATH_TO_REPO / "examples"
     tests_fetcher.PATH_TO_TRANFORMERS = tests_fetcher.PATH_TO_REPO / "src/transformers"
     tests_fetcher.PATH_TO_TESTS = tests_fetcher.PATH_TO_REPO / "tests"
     try:
         yield
     finally:
         tests_fetcher.PATH_TO_REPO = old_repo_path
+        tests_fetcher.PATH_TO_EXAMPLES = tests_fetcher.PATH_TO_REPO / "examples"
         tests_fetcher.PATH_TO_TRANFORMERS = tests_fetcher.PATH_TO_REPO / "src/transformers"
         tests_fetcher.PATH_TO_TESTS = tests_fetcher.PATH_TO_REPO / "tests"
 
@@ -409,6 +424,17 @@ def test_get_module_dependencies(self):
             with patch_transformer_repo_path(tmp_folder):
                 assert get_module_dependencies(BERT_MODELING_FILE) == expected_bert_dependencies
 
+            # Test with an example
+            create_tmp_repo(tmp_folder)
+
+            expected_example_dependencies = ["src/transformers/models/bert/modeling_bert.py"]
+
+            with patch_transformer_repo_path(tmp_folder):
+                assert (
+                    get_module_dependencies("examples/pytorch/text-classification/run_glue.py")
+                    == expected_example_dependencies
+                )
+
     def test_create_reverse_dependency_tree(self):
         with tempfile.TemporaryDirectory() as tmp_folder:
             tmp_folder = Path(tmp_folder)
@@ -494,6 +520,33 @@ def test_print_tree_deps_of(self):
 
             assert cs.out.strip() in [expected_std_out, expected_std_out_2]
 
+    def test_init_test_examples_dependencies(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            create_tmp_repo(tmp_folder)
+
+            expected_example_deps = {
+                "examples/flax/test_flax_examples.py": ["examples/flax/text-classification/run_glue.py"],
+                "examples/pytorch/test_pytorch_examples.py": ["examples/pytorch/text-classification/run_glue.py"],
+                "examples/tensorflow/test_tensorflow_examples.py": [
+                    "examples/tensorflow/text-classification/run_glue.py"
+                ],
+            }
+
+            expected_examples = {
+                "examples/flax/test_flax_examples.py",
+                "examples/flax/text-classification/run_glue.py",
+                "examples/pytorch/test_pytorch_examples.py",
+                "examples/pytorch/text-classification/run_glue.py",
+                "examples/tensorflow/test_tensorflow_examples.py",
+                "examples/tensorflow/text-classification/run_glue.py",
+            }
+
+            with patch_transformer_repo_path(tmp_folder):
+                example_deps, all_examples = init_test_examples_dependencies()
+                assert example_deps == expected_example_deps
+                assert {str(f.relative_to(tmp_folder)) for f in all_examples} == expected_examples
+
     def test_create_reverse_dependency_map(self):
         with tempfile.TemporaryDirectory() as tmp_folder:
             tmp_folder = Path(tmp_folder)
@@ -506,6 +559,12 @@ def test_create_reverse_dependency_map(self):
                 "src/transformers/__init__.py",
                 "src/transformers/models/bert/__init__.py",
                 "tests/models/bert/test_modeling_bert.py",
+                "examples/flax/test_flax_examples.py",
+                "examples/flax/text-classification/run_glue.py",
+                "examples/pytorch/test_pytorch_examples.py",
+                "examples/pytorch/text-classification/run_glue.py",
+                "examples/tensorflow/test_tensorflow_examples.py",
+                "examples/tensorflow/text-classification/run_glue.py",
             }
             assert set(reverse_map["src/transformers/models/bert/modeling_bert.py"]) == expected_bert_deps
 
@@ -521,6 +580,12 @@ def test_create_reverse_dependency_map(self):
                 "src/transformers/modeling_utils.py",
                 "tests/test_modeling_common.py",
                 "tests/models/bert/test_modeling_bert.py",
+                "examples/flax/test_flax_examples.py",
+                "examples/flax/text-classification/run_glue.py",
+                "examples/pytorch/test_pytorch_examples.py",
+                "examples/pytorch/text-classification/run_glue.py",
+                "examples/tensorflow/test_tensorflow_examples.py",
+                "examples/tensorflow/text-classification/run_glue.py",
             }
             assert set(reverse_map["src/transformers/__init__.py"]) == expected_init_deps
 
@@ -529,6 +594,12 @@ def test_create_reverse_dependency_map(self):
                 "src/transformers/models/bert/configuration_bert.py",
                 "src/transformers/models/bert/modeling_bert.py",
                 "tests/models/bert/test_modeling_bert.py",
+                "examples/flax/test_flax_examples.py",
+                "examples/flax/text-classification/run_glue.py",
+                "examples/pytorch/test_pytorch_examples.py",
+                "examples/pytorch/text-classification/run_glue.py",
+                "examples/tensorflow/test_tensorflow_examples.py",
+                "examples/tensorflow/text-classification/run_glue.py",
             }
             assert set(reverse_map["src/transformers/models/bert/__init__.py"]) == expected_init_deps
 
@@ -543,6 +614,12 @@ def test_create_reverse_dependency_map(self):
                 "src/transformers/models/bert/configuration_bert.py",
                 "src/transformers/models/bert/modeling_bert.py",
                 "tests/models/bert/test_modeling_bert.py",
+                "examples/flax/test_flax_examples.py",
+                "examples/flax/text-classification/run_glue.py",
+                "examples/pytorch/test_pytorch_examples.py",
+                "examples/pytorch/text-classification/run_glue.py",
+                "examples/tensorflow/test_tensorflow_examples.py",
+                "examples/tensorflow/text-classification/run_glue.py",
             }
             assert set(reverse_map["src/transformers/models/bert/__init__.py"]) == expected_init_deps
 
@@ -554,13 +631,26 @@ def test_create_module_to_test_map(self):
             with patch_transformer_repo_path(tmp_folder):
                 test_map = create_module_to_test_map(filter_models=True)
 
+            expected_bert_tests = {
+                "examples/flax/test_flax_examples.py",
+                "examples/pytorch/test_pytorch_examples.py",
+                "examples/tensorflow/test_tensorflow_examples.py",
+                "tests/models/bert/test_modeling_bert.py",
+            }
+
             for model in models:
-                assert test_map[f"src/transformers/models/{model}/modeling_{model}.py"] == [
-                    f"tests/models/{model}/test_modeling_{model}.py"
-                ]
+                if model != "bert":
+                    assert test_map[f"src/transformers/models/{model}/modeling_{model}.py"] == [
+                        f"tests/models/{model}/test_modeling_{model}.py"
+                    ]
+                else:
+                    assert set(test_map[f"src/transformers/models/{model}/modeling_{model}.py"]) == expected_bert_tests
 
             # Init got filtered
             expected_init_tests = {
+                "examples/flax/test_flax_examples.py",
+                "examples/pytorch/test_pytorch_examples.py",
+                "examples/tensorflow/test_tensorflow_examples.py",
                 "tests/test_modeling_common.py",
                 "tests/models/bert/test_modeling_bert.py",
                 "tests/models/gpt2/test_modeling_gpt2.py",
@@ -575,12 +665,21 @@ def test_infer_tests_to_run(self):
 
             commit_changes("src/transformers/models/bert/modeling_bert.py", BERT_MODEL_FILE_NEW_CODE, repo)
 
+            example_tests = {
+                "examples/flax/test_flax_examples.py",
+                "examples/pytorch/test_pytorch_examples.py",
+                "examples/tensorflow/test_tensorflow_examples.py",
+            }
+
             with patch_transformer_repo_path(tmp_folder):
                 infer_tests_to_run(tmp_folder / "test-output.txt", diff_with_last_commit=True)
                 with open(tmp_folder / "test-output.txt", "r") as f:
                     tests_to_run = f.read()
+                with open(tmp_folder / "examples_test_list.txt", "r") as f:
+                    example_tests_to_run = f.read()
 
             assert tests_to_run == "tests/models/bert/test_modeling_bert.py"
+            assert set(example_tests_to_run.split(" ")) == example_tests
 
             # Fake a new model addition
             repo = create_tmp_repo(tmp_folder, models=models)
@@ -617,6 +716,8 @@ def test_infer_tests_to_run(self):
                 infer_tests_to_run(tmp_folder / "test-output.txt")
                 with open(tmp_folder / "test-output.txt", "r") as f:
                     tests_to_run = f.read()
+                with open(tmp_folder / "examples_test_list.txt", "r") as f:
+                    example_tests_to_run = f.read()
 
             expected_tests = {
                 "tests/models/bert/test_modeling_bert.py",
@@ -625,15 +726,19 @@ def test_infer_tests_to_run(self):
                 "tests/test_modeling_common.py",
             }
             assert set(tests_to_run.split(" ")) == expected_tests
+            assert set(example_tests_to_run.split(" ")) == example_tests
 
             with patch_transformer_repo_path(tmp_folder):
                 infer_tests_to_run(tmp_folder / "test-output.txt", filter_models=False)
                 with open(tmp_folder / "test-output.txt", "r") as f:
                     tests_to_run = f.read()
+                with open(tmp_folder / "examples_test_list.txt", "r") as f:
+                    example_tests_to_run = f.read()
 
             expected_tests = [f"tests/models/{name}/test_modeling_{name}.py" for name in models + ["t5"]]
             expected_tests = set(expected_tests + ["tests/test_modeling_common.py"])
             assert set(tests_to_run.split(" ")) == expected_tests
+            assert set(example_tests_to_run.split(" ")) == example_tests
 
     def test_infer_tests_to_run_with_test_modifs(self):
         with tempfile.TemporaryDirectory() as tmp_folder: