Merge pull request #449 from redis-performance/arch.fix

Ensuring architecture is present on metadata across all timeseries

Author: fcostaoliveira

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.11.32"
+version = "0.11.35"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "README.md"

redisbench_admin/compare/args.py

@@ -92,6 +92,18 @@ def create_compare_arguments(parser):
         default=1,
         help="Use the last N samples for each time-serie. by default will use last value only",
     )
+    parser.add_argument(
+        "--first_n_baseline",
+        type=int,
+        default=-1,
+        help="Use the last N samples for each time-serie. by default will use last 7 available values",
+    )
+    parser.add_argument(
+        "--first_n_comparison",
+        type=int,
+        default=-1,
+        help="Use the last N samples for each time-serie. by default will use last value only",
+    )
     parser.add_argument(
         "--from-date",
         type=lambda s: datetime.datetime.strptime(s, "%Y-%m-%d"),

redisbench_admin/compare/compare.py

@@ -135,10 +135,31 @@ def compare_command_logic(args, project_name, project_version):
         last_n_baseline = args.last_n_baseline
     if last_n_comparison < 0:
         last_n_comparison = args.last_n_comparison
-    logging.info("Using last {} samples for baseline analysis".format(last_n_baseline))
-    logging.info(
-        "Using last {} samples for comparison analysis".format(last_n_comparison)
-    )
+    first_n_baseline = args.first_n_baseline
+    first_n_comparison = args.first_n_comparison
+    # Log the interval of values considered
+    if first_n_baseline >= 0:
+        logging.info(
+            "Using samples in the range [{}:{}] for baseline analysis".format(
+                first_n_baseline, last_n_baseline
+            )
+        )
+    else:
+        logging.info(
+            "Using last {} samples for baseline analysis".format(last_n_baseline)
+        )
+
+    if first_n_comparison >= 0:
+        logging.info(
+            "Using samples in the range [{}:{}] for comparison analysis".format(
+                first_n_comparison, last_n_comparison
+            )
+        )
+    else:
+        logging.info(
+            "Using last {} samples for comparison analysis".format(last_n_comparison)
+        )
+
     verbose = args.verbose
     regressions_percent_lower_limit = args.regressions_percent_lower_limit
     metric_name = args.metric_name
@@ -280,6 +301,8 @@ def compare_command_logic(args, project_name, project_version):
         running_platform,
         baseline_architecture,
         comparison_architecture,
+        first_n_baseline,
+        first_n_comparison,
     )
     comment_body = ""
     if total_comparison_points > 0:
@@ -506,6 +529,8 @@ def compute_regression_table(
     running_platform=None,
     baseline_architecture=ARCH_X86,
     comparison_architecture=ARCH_X86,
+    first_n_baseline=-1,
+    first_n_comparison=-1,
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -594,6 +619,8 @@ def compute_regression_table(
         running_platform,
         baseline_architecture,
         comparison_architecture,
+        first_n_baseline,
+        first_n_comparison,
     )
     logging.info(
         "Printing differential analysis between {} and {}".format(
@@ -723,6 +750,8 @@ def from_rts_to_regression_table(
     running_platform=None,
     baseline_architecture=ARCH_X86,
     comparison_architecture=ARCH_X86,
+    first_n_baseline=-1,
+    first_n_comparison=-1,
 ):
     print_all = print_regressions_only is False and print_improvements_only is False
     table = []
@@ -835,6 +864,7 @@ def from_rts_to_regression_table(
                 largest_variance,
                 last_n_baseline,
                 verbose,
+                first_n_baseline,
             )
             for ts_name_comparison in comparison_timeseries:
                 datapoints_inner = rts.ts().revrange(
@@ -854,6 +884,7 @@ def from_rts_to_regression_table(
                 largest_variance,
                 last_n_comparison,
                 verbose,
+                first_n_comparison,
             )
 
             waterline = regressions_percent_lower_limit
@@ -1051,13 +1082,25 @@ def get_v_pct_change_and_largest_var(
     largest_variance,
     last_n=-1,
     verbose=False,
+    first_n=-1,
 ):
     comparison_nsamples = len(comparison_datapoints)
     if comparison_nsamples > 0:
         _, comparison_v = comparison_datapoints[0]
-        for tuple in comparison_datapoints:
-            if last_n < 0 or (last_n > 0 and len(comparison_values) < last_n):
-                comparison_values.append(tuple[1])
+
+        # Apply first_n and last_n boundaries
+        start_idx = 0 if first_n < 0 else max(0, min(first_n, comparison_nsamples))
+        end_idx = (
+            comparison_nsamples
+            if last_n < 0
+            else max(0, min(last_n, comparison_nsamples))
+        )
+
+        selected_data = comparison_datapoints[start_idx:end_idx]
+
+        for tuple in selected_data:
+            comparison_values.append(tuple[1])
+
         comparison_df = pd.DataFrame(comparison_values)
         comparison_median = float(comparison_df.median())
         comparison_v = comparison_median

redisbench_admin/run/metrics.py

@@ -5,7 +5,7 @@
 #
 import logging
 import datetime as dt
-
+import redis
 from jsonpath_ng import parse
 
 
@@ -99,28 +99,33 @@ def collect_redis_metrics(
     for conn_n, conn in enumerate(redis_conns):
         conn_res = {}
         for section in sections:
-            info = conn.info(section)
-            conn_res[section] = info
-            if section not in overall:
-                overall[section] = {}
-            for k, v in info.items():
-                collect = True
-                if section_filter is not None:
-                    if section in section_filter:
-                        if k not in section_filter[section]:
-                            collect = False
-                if collect and type(v) is float or type(v) is int:
-                    if k not in overall[section]:
-                        overall[section][k] = 0
-                    overall[section][k] += v
-                if collect and type(v) is dict:
-                    for inner_k, inner_v in v.items():
-                        if type(inner_v) is float or type(inner_v) is int:
-                            final_str_k = "{}_{}".format(k, inner_k)
-                            if multi_shard:
-                                final_str_k += "_shard_{}".format(conn_n + 1)
-                            if final_str_k not in overall[section]:
-                                overall[section][final_str_k] = inner_v
+            try:
+                info = conn.info(section)
+                conn_res[section] = info
+                if section not in overall:
+                    overall[section] = {}
+                for k, v in info.items():
+                    collect = True
+                    if section_filter is not None:
+                        if section in section_filter:
+                            if k not in section_filter[section]:
+                                collect = False
+                    if collect and type(v) is float or type(v) is int:
+                        if k not in overall[section]:
+                            overall[section][k] = 0
+                        overall[section][k] += v
+                    if collect and type(v) is dict:
+                        for inner_k, inner_v in v.items():
+                            if type(inner_v) is float or type(inner_v) is int:
+                                final_str_k = "{}_{}".format(k, inner_k)
+                                if multi_shard:
+                                    final_str_k += "_shard_{}".format(conn_n + 1)
+                                if final_str_k not in overall[section]:
+                                    overall[section][final_str_k] = inner_v
+            except redis.exceptions.ConnectionError as e:
+                logging.warning(
+                    f"Unable to collect section {section} from redis. error: {e.__str__}"
+                )
 
         res.append(conn_res)
 

redisbench_admin/run/run.py

@@ -86,7 +86,7 @@ def define_benchmark_plan(benchmark_definitions, default_specs):
             setup_contains_dbconfig = False
             if "dbconfig" in setup_settings:
                 setup_contains_dbconfig = True
-            logging.info(
+            logging.debug(
                 f"setup ({setup_name}): {setup_settings}. contains dbconfig {setup_contains_dbconfig}"
             )
 

redisbench_admin/run_remote/run_remote.py

@@ -134,7 +134,7 @@ def run_remote_command_logic(args, project_name, project_version):
     keep_env_and_topo = args.keep_env_and_topo
     skip_remote_db_setup = args.skip_db_setup
     flushall_on_every_test_start = args.flushall_on_every_test_start
-    redis_7 = args.redis_7
+    redis_7 = True
     cluster_start_port = 20000
     redis_password = args.db_pass
     ignore_keyspace_errors = args.ignore_keyspace_errors
@@ -317,6 +317,7 @@ def run_remote_command_logic(args, project_name, project_version):
     spot_instance_error = False
     ts_key_spot_price = f"ts:{tf_triggering_env}:tests:spot_price"
     ts_key_full_price = f"ts:{tf_triggering_env}:tests:full_price"
+    ts_key_architecture = f"ts:{tf_triggering_env}:tests:arch:{architecture}"
 
     for benchmark_type, bench_by_dataset_map in benchmark_runs_plan.items():
         if return_code != 0 and args.fail_fast:
@@ -365,6 +366,8 @@ def run_remote_command_logic(args, project_name, project_version):
                         )
                         continue
                     metadata_tags = get_metadata_tags(benchmark_config)
+                    if "arch" not in metadata_tags:
+                        metadata_tags["arch"] = architecture
                     logging.info(
                         "Including the extra metadata tags into this test generated time-series: {}".format(
                             metadata_tags
@@ -462,6 +465,15 @@ def run_remote_command_logic(args, project_name, project_version):
                                         testcase_start_time_str,
                                     ) = get_start_time_vars()
                                     if args.push_results_redistimeseries:
+                                        logging.info(
+                                            f"Updating overall arch tests counter {ts_key_architecture}"
+                                        )
+                                        rts.ts().add(
+                                            ts_key_architecture,
+                                            start_time_setup_ms,
+                                            1,
+                                            duplicate_policy="sum",
+                                        )
                                         logging.info(
                                             f"Updating overall spot price tests counter {ts_key_spot_price}"
                                         )
@@ -843,7 +855,10 @@ def run_remote_command_logic(args, project_name, project_version):
                                                     tf_github_org,
                                                     tf_github_repo,
                                                     tf_triggering_env,
-                                                    {"metric-type": "redis-metrics"},
+                                                    {
+                                                        "metric-type": "redis-metrics",
+                                                        "arch": architecture,
+                                                    },
                                                     expire_ms,
                                                 )
                                                 if collect_commandstats:
@@ -866,7 +881,10 @@ def run_remote_command_logic(args, project_name, project_version):
                                                         tf_github_org,
                                                         tf_github_repo,
                                                         tf_triggering_env,
-                                                        {"metric-type": "commandstats"},
+                                                        {
+                                                            "metric-type": "commandstats",
+                                                            "arch": architecture,
+                                                        },
                                                         expire_ms,
                                                     )
                                                     (
@@ -888,7 +906,10 @@ def run_remote_command_logic(args, project_name, project_version):
                                                         tf_github_org,
                                                         tf_github_repo,
                                                         tf_triggering_env,
-                                                        {"metric-type": "latencystats"},
+                                                        {
+                                                            "metric-type": "latencystats",
+                                                            "arch": architecture,
+                                                        },
                                                         expire_ms,
                                                     )
                                             except (