Added the warm_up_command parameter so that any command can be executed before the workload is executed.

Author: ssl-oyamata

conf/postgres_opttune.conf

@@ -27,7 +27,8 @@ required_recovery_time_second = 0
 benchmark = my_workload # Benchmark tool name('my_workload' or 'sampled_workload' or 'pgbench' or 'oltpbench' or 'star_schema_benchmark')
 parameter_json_dir = ./conf/
 number_trail = 100 # Number of benchmarks to run for turning
-data_load_interval = 10 # Specify the data load interval by the number of benchmarks
+data_load_interval = 10 # Specify the execution interval of data_load_command by the number of benchmarks
+warm_up_interval = 1 # Specify the execution interval of warm_up_command by the number of benchmarks
 sample_mode = TPE # Sampling mode(TPE, RandomSampler, SkoptSampler, CmaEsSampler)
 debug = False # debug mode
 save_study_history = True # Whether to save study history
@@ -38,6 +39,7 @@ history_database_url = sqlite:///study-history.db   # Example PostgreSQL. postgr
 work_directory = current_directory # Specifies the directory where the workload will run. Example: /opt/test
                                    # The default value of current_directory runs the workload without moving the directory.
 data_load_command = /usr/pgsql-12/bin/pgbench -i -s 10 tpcc
+warm_up_command = psql -c "SELECT 'warm up!'"
 run_workload_command = /usr/pgsql-12/bin/pgbench tpcc -T 1200
 
 [sampled-workload]

pgopttune/config/my_workload_config.py

@@ -14,6 +14,10 @@ def work_directory(self):
     def data_load_command(self):
         return self.get_parameter_value('data_load_command')
 
+    @property
+    def warm_up_command(self):
+        return self.get_parameter_value('warm_up_command')
+
     @property
     def run_workload_command(self):
         return self.get_parameter_value('run_workload_command')

pgopttune/config/tune_config.py

@@ -30,6 +30,10 @@ def number_trail(self):
     def data_load_interval(self):
         return int(self.get_parameter_value('data_load_interval'))
 
+    @property
+    def warm_up_interval(self):
+        return int(self.get_parameter_value('warm_up_interval'))
+
     @property
     def sample_mode(self):
         return self.get_parameter_value('sample_mode')

pgopttune/objective/objective.py

@@ -14,6 +14,7 @@ def __init__(self,
         self.params = PostgresTuneParameter(postgres_server_config,  # turning parameter
                                             params_json_dir=tune_config.parameter_json_dir)
         self.data_load_interval = tune_config.data_load_interval
+        self.warm_up_interval = tune_config.warm_up_interval
         self.workload = None
 
     def __call__(self, trial):
@@ -39,6 +40,8 @@ def run_workload(self, trial):
             self.workload.prepare_workload_database()
         self.workload.vacuum_database()  # vacuum analyze
         self.params.reset_database()  # cache free and database restart
+        if (int(trial.number) == 0) or (int(trial.number) % self.warm_up_interval == 0):
+            self.workload.warm_up()  # run warm_up_command
         objective_value = self.workload.run()  # benchmark run
         return objective_value
 

pgopttune/parameter/pg_parameter.py

@@ -109,7 +109,7 @@ def _reload_conf(self):
 
     def _free_cache(self):
         free_cache_cmd = 'sudo bash -c "sync"; sudo bash -c "echo 1 > /proc/sys/vm/drop_caches"'
-
+        logger.debug("Free pagecache.")
         # localhost PostgreSQL
         if self.postgres_server_config.host == '127.0.0.1' or self.postgres_server_config.host == 'localhost':
             run_command(free_cache_cmd)

pgopttune/workload/my_workload.py

@@ -23,18 +23,25 @@ def __init__(self, postgres_server_config: PostgresServerConfig, my_workload_con
 
     def data_load(self):
         cwd = os.getcwd()
-        if self.my_workload_config.work_directory != "current_directory":
-            os.chdir(self.my_workload_config.work_directory)
+        self._change_work_directory()
         data_load_cmd = self.my_workload_config.data_load_command
-        logger.debug('run my workload data load command : {}'.format(data_load_cmd))
+        logger.debug('Run data_load_command : {}'.format(data_load_cmd))
         run_command(data_load_cmd)
         if self.my_workload_config.work_directory != "current_directory":
             os.chdir(cwd)
 
-    def run(self, measurement_time_second: int = None):
+    def warm_up(self):
         cwd = os.getcwd()
+        self._change_work_directory()
+        warm_up_command = self.my_workload_config.warm_up_command
+        logger.debug('Run warm_up_command : {}'.format(warm_up_command))
+        run_command(warm_up_command)
         if self.my_workload_config.work_directory != "current_directory":
-            os.chdir(self.my_workload_config.work_directory)
+            os.chdir(cwd)
+
+    def run(self, measurement_time_second: int = None):
+        cwd = os.getcwd()
+        self._change_work_directory()
         run_workload_command = self.my_workload_config.run_workload_command
         start_number_of_xact_commit = self.get_number_of_xact_commit()
         workload_start_time = time.time()  # start measurement time
@@ -59,6 +66,10 @@ def run(self, measurement_time_second: int = None):
         tps = self.calculate_transaction_per_second(workload_number_of_xact_commit, self.workload_elapsed_time)
         return tps
 
+    def _change_work_directory(self):
+        if self.my_workload_config.work_directory != "current_directory":
+            os.chdir(self.my_workload_config.work_directory)
+
     @staticmethod
     def calculate_transaction_per_second(number_of_xact_commit, elapsed_seconds):
         return round(number_of_xact_commit / elapsed_seconds, 6)

pgopttune/workload/sampled_workload.py

@@ -109,11 +109,14 @@ def load_sampled_workload(cls, load_file_path, postgres_server_config: PostgresS
             workload.postgres_server_config = postgres_server_config
         return workload
 
-    @staticmethod
-    def data_load():
+    def data_load(self):
         # TODO:
         logger.warning("At the moment, in the sampled workload, The data reload function is not implemented.")
 
+    def warm_up(self):
+        # TODO:
+        logger.warning("At the moment, in the sampled workload, The warm up function is not implemented.")
+
     def _run_transaction(self, transaction_index=0):
         # logger.debug("Transaction's statement : {}".format(self.my_transactions[transaction_index].statement))
         transaction_elapsed_time = self.my_transactions[transaction_index].run(self.postgres_server_config)

pgopttune/workload/workload.py

@@ -26,13 +26,17 @@ def prepare_workload_database(self):
     def data_load(self):
         raise NotImplementedError("subclasses of Workload must provide a data_load() method.")
 
+    def warm_up(self):
+        raise NotImplementedError("subclasses of Workload must provide a warm_up() method.")
+
     def run(self, measurement_time_second: int = None):
         raise NotImplementedError("subclasses of Workload must provide a run() method.")
 
     def vacuum_database(self):
         """
         run vacuum analyze
         """
+        logger.debug("Run VACUUM ANALYZE.")
         vacuum_analyze_sql = "VACUUM ANALYZE"
         with get_pg_connection(dsn=self.postgres_server_config.dsn) as conn:
             conn.set_session(autocommit=True)
@@ -116,4 +120,4 @@ def _get_backup_database_dsn(self):
                                                     self.postgres_server_config.password,
                                                     self.postgres_server_config.host,
                                                     self.postgres_server_config.port,
-                                                    self._get_backup_database_name())
+                                                    self._get_backup_database_name())