Fixed workload sampling function.

Author: ssl-oyamata

conf/postgres_opttune.conf

@@ -18,26 +18,13 @@ pg_os_user = postgres # PostgrSQL ower user(OS user)
 ssh_port = 22 # ssh port
 ssh_password = postgres # pg_os_user's ssh password
 
-[workload-sampling]
-workload_sampling_time_second = 300
-# Time (in seconds) to sample the workload running on the database in the [PostgreSQL] section
-my_workload_save_dir = ./workload_data/
-# Workload save database settings
-pghost = localhost # PostgreSQL server host
-pgport = 5432 # PostgreSQL server port
-pguser = postgres # PostgreSQL user name(Database user)
-pgpassword = postgres12 # PostgreSQL user password(Database user)
-pgdatabase = sampling # PostgreSQL Database
-# workload save directory
-
-
 [turning]
 study_name = pgbench_study # study name
 required_recovery_time_second = 0
 # The maximum recovery time allowed by the user in case of a PostgreSQL crash,
 # which is used to estimate the wax_wal_size parameter.
 # Note: The default value of 0 does not perform the estimation of the wax_wal_size parameter.
-benchmark = pgbench # Benchmark tool name('my_workload' or pgbench' or 'oltpbench' or 'star_schema_benchmark')
+benchmark = pgbench # Benchmark tool name('sampled_workload' or pgbench' or 'oltpbench' or 'star_schema_benchmark')
 parameter_json_dir = ./conf/
 number_trail = 100 # Number of benchmarks to run for turning
 data_load_interval = 10 # Specify the data load interval by the number of benchmarks
@@ -47,8 +34,8 @@ save_study_history = True # Whether to save study history
 load_study_history = True # Whether to load study history if a study name already exists.
 history_database_url = sqlite:///study-history.db   # Example PostgreSQL. postgresql://postgres@localhost/study_history
 
-[my-workload]
-my_workload_save_file = workload_data/2020-07-05_180647.531417-2020-07-05_180657.531661.pkl
+[sampled_workload]
+sampled_workload_save_file = workload_data/2020-09-13_202209.011708-2020-09-13_202239.011973.pkl
 # File saved using workload_sampler.py
 
 [pgbench]
@@ -71,4 +58,15 @@ sql_file_path = ./pgopttune/workload/star_schema_sql/
 sql_key = Q1.1, Q2.1, Q3.1
 # List of queries to be executed by the client (comma-separated)
 # Please specify the name of the file in pgopttune/workload/star_schema_sql/ directory(sql_file_path parameter diretory)
-# (e.g., Q1.1,Q2,1).
+# (e.g., Q1.1,Q2,1).
+
+[workload-sampling]
+workload_sampling_time_second = 30
+# Time (in seconds) to sample the workload running on the database in the [PostgreSQL] section
+my_workload_save_dir = ./workload_data/ # workload save directory
+# Database settings to temporarily store workload information
+pghost = localhost # PostgreSQL server host
+pgport = 5432 # PostgreSQL server port
+pguser = postgres # PostgreSQL user name(Database user)
+pgpassword = postgres12 # PostgreSQL user password(Database user)
+pgdatabase = sampling # PostgreSQL Database

pgopttune/config/my_workload_config.py

@@ -0,0 +1,20 @@
+import os
+from pgopttune.config.config import Config
+
+
+class SampledWorkloadConfig(Config):
+    def __init__(self, conf_path, section='sampled_workload'):
+        super().__init__(conf_path)
+        self.conf_path = conf_path
+        self.config_dict = dict(self.config.items(section))
+        self._check_is_exist_sampled_workload_save_file()
+
+    def _check_is_exist_sampled_workload_save_file(self):
+        if not os.path.exists(self.get_parameter_value('sampled_workload_save_file')):
+            raise ValueError("{} does not exist."
+                             "Check the sampled_workload_save_file parameter in {}."
+                             .format(self.get_parameter_value('sampled_workload_save_file'), self.conf_path))
+
+    @property
+    def my_workload_save_file(self):
+        return self.get_parameter_value('sampled_workload_save_file')

pgopttune/config/sampled_workload_config.py

@@ -0,0 +1,19 @@
+from logging import getLogger
+from pgopttune.workload.sampled_workload import SampledWorkload
+from pgopttune.objective.objective import Objective
+from pgopttune.config.postgres_server_config import PostgresServerConfig
+from pgopttune.config.tune_config import TuneConfig
+from pgopttune.config.sampled_workload_config import SampledWorkloadConfig
+
+logger = getLogger(__name__)
+
+
+class ObjectiveSampledWorkload(Objective):
+
+    def __init__(self,
+                 postgres_server_config: PostgresServerConfig,
+                 tune_config: TuneConfig,
+                 my_workload_config: SampledWorkloadConfig):
+        super().__init__(postgres_server_config, tune_config)
+        self.workload = SampledWorkload.load_sampled_workload(my_workload_config.my_workload_save_file,
+                                                              postgres_server_config=postgres_server_config)

pgopttune/objective/objective_my_workload.py

@@ -6,7 +6,7 @@
 logger = getLogger(__name__)
 
 
-class MyTransaction:
+class SampledTransaction:
     def __init__(self, session_id: str, query_start_time: list, statement: list):
         self.session_id = session_id
         self.query_start_time = query_start_time

pgopttune/objective/objective_sampled_workload.py

@@ -8,12 +8,12 @@
 from pgopttune.utils.pg_connect import get_pg_connection
 from pgopttune.config.postgres_server_config import PostgresServerConfig
 from pgopttune.config.workload_sampling_config import WorkloadSamplingConfig
-from pgopttune.workload.my_transaction import MyTransaction
+from pgopttune.workload.sampled_transaction import SampledTransaction
 
 logger = getLogger(__name__)
 
 
-class MyWorkload(Workload):
+class SampledWorkload(Workload):
     def __init__(self,
                  postgres_server_config: PostgresServerConfig,
                  workload_sampling_config: WorkloadSamplingConfig,
@@ -68,13 +68,13 @@ def _create_transactions(self, workload_rows):
                 session_id = row[1]
                 statement.append(row[2])
             else:
-                my_transaction = MyTransaction(session_id, query_stat_time, statement)
+                my_transaction = SampledTransaction(session_id, query_stat_time, statement)
                 self.my_transactions.append(my_transaction)
                 query_stat_time = [row[0]]
                 session_id = row[1]
                 statement = [row[2]]
 
-    def save_my_workload(self):
+    def save_sampled_workload(self):
         save_file_name = datetime.datetime.fromtimestamp(self.start_unix_time).strftime("%Y-%m-%d_%H%M%S.%f") + \
                          "-" \
                          + datetime.datetime.fromtimestamp(self.end_unix_time).strftime("%Y-%m-%d_%H%M%S.%f") + ".pkl"
@@ -102,7 +102,7 @@ def run(self):
         return elapsed_time
 
     @classmethod
-    def load_my_workload(cls, load_file_path, postgres_server_config: PostgresServerConfig = None):
+    def load_sampled_workload(cls, load_file_path, postgres_server_config: PostgresServerConfig = None):
         with open(load_file_path, 'rb') as f:
             workload = pickle.load(f)
         if postgres_server_config is not None:
@@ -112,7 +112,7 @@ def load_my_workload(cls, load_file_path, postgres_server_config: PostgresServer
     @staticmethod
     def data_load():
         # TODO:
-        logger.warning("At this time, the data loading function to the sampled database is not implemented.")
+        logger.warning("At the moment, in the sampled workload, The data reload function is not implemented.")
 
     def _run_transaction(self, transaction_index=0):
         # logger.debug("Transaction's statement : {}".format(self.my_transactions[transaction_index].statement))
@@ -130,19 +130,19 @@ def _run_transaction(self, transaction_index=0):
     conf_path = './conf/postgres_opttune.conf'
     postgres_server_config_test = PostgresServerConfig(conf_path)  # PostgreSQL Server config
     workload_sampling_config_test = WorkloadSamplingConfig(conf_path)
-    my_workload = MyWorkload(start_unix_time=1593093506.9530554, end_unix_time=1593093567.088895,
-                             workload_sampling_config=workload_sampling_config_test,
-                             postgres_server_config=postgres_server_config_test)
-    save_file = my_workload.save_my_workload()
+    sampled_workload = SampledWorkload(start_unix_time=1593093506.9530554, end_unix_time=1593093567.088895,
+                                       workload_sampling_config=workload_sampling_config_test,
+                                       postgres_server_config=postgres_server_config_test)
+    save_file = sampled_workload.save_sampled_workload()
     logger.debug("run transactions ")
-    my_workload_elapsed_time = my_workload.run()
-    logger.debug(my_workload_elapsed_time)
-    load_workload = MyWorkload.load_my_workload(save_file, postgres_server_config=postgres_server_config_test)
+    workload_elapsed_time = sampled_workload.run()
+    logger.debug(workload_elapsed_time)
+    load_workload = SampledWorkload.load_sampled_workload(save_file, postgres_server_config=postgres_server_config_test)
     logger.debug("run transactions using saved file")
     load_workload_elapsed_time = load_workload.run()
     logger.debug(load_workload_elapsed_time)
     logger.debug("finised...")
-    logger.debug(my_workload_elapsed_time)
+    logger.debug(workload_elapsed_time)
     logger.debug(load_workload_elapsed_time)
 
     # my_workload.extract_workload()

pgopttune/workload/my_transaction.py renamed to pgopttune/workload/sampled_transaction.py

@@ -4,7 +4,7 @@
 from pgopttune.log.pg_csv_log import PostgresCsvLog
 from pgopttune.config.postgres_server_config import PostgresServerConfig
 from pgopttune.config.workload_sampling_config import WorkloadSamplingConfig
-from pgopttune.workload.my_workload import MyWorkload
+from pgopttune.workload.sampled_workload import SampledWorkload
 
 logger = getLogger(__name__)
 
@@ -31,10 +31,10 @@ def save(self):
         logger.debug("Start importing the CSV file(saved executed SQL) into the table.")
         self.csv_log.load_csv_to_database(copy_dir=self.workload_sampling_config.my_workload_save_dir,
                                           dsn=self.workload_sampling_config.dsn)
-        my_workload = MyWorkload(start_unix_time=csv_log_start_time, end_unix_time=csv_log_end_time,
-                                 postgres_server_config=self.postgres_server_config,
-                                 workload_sampling_config=self.workload_sampling_config)
-        save_file = my_workload.save_my_workload()
+        workload = SampledWorkload(start_unix_time=csv_log_start_time, end_unix_time=csv_log_end_time,
+                                      postgres_server_config=self.postgres_server_config,
+                                      workload_sampling_config=self.workload_sampling_config)
+        save_file = workload.save_sampled_workload()
         logger.info("The workload has been recorded in '{}'".format(save_file))
         return save_file
 

pgopttune/workload/my_workload.py renamed to pgopttune/workload/sampled_workload.py

@@ -28,10 +28,10 @@ def main(
     logger.info(
         "You can automatically tune the saved workload by setting the following in'./conf/postgres_opttune.conf'.\n"
         "[turning]\n"
-        "benchmark = my_workload \n"
+        "benchmark = sampled_workload \n"
         ":\n"
-        "[my-workload]\n"
-        "my_workload_save_file = {}".format(workload_save_file_path))
+        "[sampled_workload]\n"
+        "sampled_workload_save_file = {}".format(workload_save_file_path))
 
 
 if __name__ == "__main__":

pgopttune/workload/workload_sampler.py

@@ -10,7 +10,7 @@
 from pgopttune.objective.objective_pgbench import ObjectivePgbench
 from pgopttune.objective.objective_oltpbench import ObjectiveOltpbench
 from pgopttune.objective.objective_star_schema_benchmark import ObjectiveStarSchemaBenchmark
-from pgopttune.objective.objective_my_workload import ObjectiveMyWorkload
+from pgopttune.objective.objective_sampled_workload import ObjectiveSampledWorkload
 from pgopttune.parameter.reset import reset_postgres_param
 from pgopttune.parameter.pg_tune_parameter import PostgresTuneParameter
 from pgopttune.recovery.pg_recovery import Recovery
@@ -19,7 +19,7 @@
 from pgopttune.config.pgbench_config import PgbenchConfig
 from pgopttune.config.oltpbench_config import OltpbenchConfig
 from pgopttune.config.star_schema_benchmark_config import StarSchemaBenchmarkConfig
-from pgopttune.config.my_workload_config import MyWorkloadConfig
+from pgopttune.config.sampled_workload_config import SampledWorkloadConfig
 
 
 def main(
@@ -49,9 +49,9 @@ def main(
         star_schema_benchmark_config = StarSchemaBenchmarkConfig(conf_path)  # star schema benchmark config
         objective = ObjectiveStarSchemaBenchmark(postgres_server_config, tune_config, star_schema_benchmark_config)
     # my workload (save using sampling_workload.py)
-    elif tune_config.benchmark == 'my_workload':
-        my_workload_config = MyWorkloadConfig(conf_path)  # my worklod config config
-        objective = ObjectiveMyWorkload(postgres_server_config, tune_config, my_workload_config)
+    elif tune_config.benchmark == 'sampled_workload':
+        sampled_workload_config = SampledWorkloadConfig(conf_path)  # my workload sampled config
+        objective = ObjectiveSampledWorkload(postgres_server_config, tune_config, sampled_workload_config)
     else:
         raise NotImplementedError('This benchmark tool is not supported at this time.')
 
@@ -81,7 +81,7 @@ def main(
     # tuning using optuna
     try:
         sampler = get_sampler(tune_config.sample_mode)  # sampler setting
-        if tune_config.benchmark == 'my_workload':
+        if tune_config.benchmark == 'sampled_workload':
             logger.info("The purpose of optimization is to minimize the total SQL execution time")
             study = create_study(study_name=tune_config.study_name,  # create study
                                  sampler=sampler,