tests: optimize a/b test fixture

In the previous implementation, git clone executed
for each parameter of the parametize test.
This has a large overhead, adjusted it so that
fixtures only called once per class.

Signed-off-by: Tomoya Iwata <iwata.tomoya@classmethod.jp>

Author: cm-iwata

tests/framework/ab_test.py

@@ -22,9 +22,9 @@
 does not block PRs). If not, it fails, preventing PRs from introducing new vulnerable dependencies.
 """
 import statistics
-from pathlib import Path
+from pathlib import Path, PosixPath
 from tempfile import TemporaryDirectory
-from typing import Callable, List, Optional, TypeVar
+from typing import Callable, Generator, List, Optional, Tuple, TypeVar
 
 import scipy
 
@@ -98,6 +98,32 @@ def git_ab_test(
         return result_a, result_b, comparison
 
 
+def git_clone_ab_dirs(
+    a_revision: str = DEFAULT_A_REVISION,
+    b_revision: Optional[str] = None,
+) -> Generator[Tuple[PosixPath, PosixPath], None, None]:
+    """
+    Prepare cloned Git repository to run A/B tests.
+
+    :param a_revision: The revision to checkout for the "A" part of the test. Defaults to the pull request target branch
+                       if run in CI, and "main" otherwise.
+    :param b_revision: The git revision to check out for "B" part of the test. Defaults to whatever is currently checked
+                       out (in which case no temporary directory will be created).
+    """
+
+    with TemporaryDirectory() as tmp_dir:
+        dir_a = git_clone(Path(tmp_dir) / a_revision, a_revision)
+
+        if b_revision:
+            dir_b = git_clone(Path(tmp_dir) / b_revision, b_revision)
+        else:
+            # By default, pytest execution happens inside the `tests` subdirectory. Pass the repository root, as
+            # documented.
+            dir_b = Path.cwd().parent
+
+        yield (dir_a, dir_b)
+
+
 DEFAULT_A_DIRECTORY = FC_WORKSPACE_DIR / "build" / "main"
 DEFAULT_B_DIRECTORY = FC_WORKSPACE_DIR / "build" / "cargo_target" / DEFAULT_TARGET_DIR
 

tests/integration_tests/performance/test_benchmarks.py

@@ -12,10 +12,11 @@
 import pytest
 
 from framework import utils
-from framework.ab_test import git_ab_test
+from framework.ab_test import binary_ab_test, git_clone_ab_dirs
 from host_tools.cargo_build import cargo
 
 LOGGER = logging.getLogger(__name__)
+git_clone_ab_dirs_one_time = pytest.fixture(git_clone_ab_dirs, scope="class")
 
 
 def get_benchmark_names() -> List[str]:
@@ -39,16 +40,33 @@ def get_benchmark_names() -> List[str]:
     return list(set(benchmark_names))
 
 
-@pytest.mark.no_block_pr
-@pytest.mark.parametrize("benchname", get_benchmark_names())
-def test_no_regression_relative_to_target_branch(benchname):
+class TestBenchMarks:
     """
-    Run the microbenchmarks in this repository, comparing results from pull
-    request target branch against what's achieved on HEAD
+    This class is used to prevent fixtures from being executed for each parameter in
+    a parametrize test.
     """
-    run_criterion = get_run_criterion(benchname)
-    compare_results = get_compare_results(benchname)
-    git_ab_test(run_criterion, compare_results)
+
+    @pytest.mark.no_block_pr
+    @pytest.mark.parametrize("benchname", get_benchmark_names())
+    def test_no_regression_relative_to_target_branch(
+        self, benchname, git_clone_ab_dirs_one_time
+    ):
+        """
+        Run the microbenchmarks in this repository, comparing results from pull
+        request target branch against what's achieved on HEAD
+        """
+
+        dir_a = git_clone_ab_dirs_one_time[0]
+        dir_b = git_clone_ab_dirs_one_time[1]
+        run_criterion = get_run_criterion(benchname)
+        compare_results = get_compare_results(benchname)
+
+        binary_ab_test(
+            test_runner=run_criterion,
+            comparator=compare_results,
+            a_directory=dir_a,
+            b_directory=dir_b,
+        )
 
 
 def get_run_criterion(benchmark_name) -> Callable[[Path, bool], Path]: