[placement group] fix pg benchmark regression ray-project#22441

We added a warmup time in timeit which affects the pg benchmark time accounting. add an option to cancel warmup.

python/ray/_private/ray_microbenchmark_helpers.py

@@ -11,11 +11,14 @@
 filter_pattern = os.environ.get("TESTS_TO_RUN", "")
 
 
-def timeit(name, fn, multiplier=1) -> List[Optional[Tuple[str, float, float]]]:
+def timeit(
+    name, fn, multiplier=1, warmup_time_sec=10
+) -> List[Optional[Tuple[str, float, float]]]:
     if filter_pattern not in name:
         return [None]
-    # sleep for 10 seconds to avoid noisy neighbors.
-    time.sleep(10)
+    # sleep for a while to avoid noisy neigbhors.
+    # related issue: https://github.com/ray-project/ray/issues/22045
+    time.sleep(warmup_time_sec)
     # warmup
     start = time.perf_counter()
     count = 0

release/nightly_tests/placement_group_tests/placement_group_performance_test.py

@@ -37,6 +37,7 @@ def placement_group_create(num_pgs):
         "placement group create per second",
         lambda: placement_group_create(num_pgs),
         num_pgs,
+        warmup_time_sec=0,
     )
 
     # Get fine-grained scheduling stats.