Fixed data load per trial.

Author: ssl-oyamata

pgopttune/objective/objective.py

@@ -35,7 +35,14 @@ def change_tune_param(self, trial):
 
     def run_workload(self, trial):
         if (int(trial.number) == 0) or (int(trial.number) % self.data_load_interval == 0):
-            self.workload.data_load()  # data load
+            if self.workload.check_exist_backup_database():
+                # Recreate the database using the backed up database as a template
+                self.workload.drop_database()
+                self.workload.create_database_use_backup_database()
+            else:
+                self.workload.data_load()  # data load
+                self.workload.create_backup_database()  # backup database
+        self.workload.vacuum_database()  # vacuum analyze
         self.params.reset_database()  # cache free and database restart
         objective_value = self.workload.run()  # benchmark run
         return objective_value

pgopttune/workload/my_workload.py

@@ -21,16 +21,9 @@ def __init__(self, postgres_server_config: PostgresServerConfig, my_workload_con
         os.environ['PGPASSWORD'] = postgres_server_config.password
 
     def data_load(self):
-        if self._check_exist_backup_database():
-            # Recreate the database using the backed up database as a template
-            self._drop_database()
-            self._create_database_use_backup_database()
-        else:
-            data_load_cmd = self.my_workload_config.data_load_command
-            logger.debug('run my workload data load command : {}'.format(data_load_cmd))
-            run_command(data_load_cmd)
-            self._create_backup_database()  # backup database
-        self.vacuum_database()  # vacuum analyze
+        data_load_cmd = self.my_workload_config.data_load_command
+        logger.debug('run my workload data load command : {}'.format(data_load_cmd))
+        run_command(data_load_cmd)
 
     def run(self):
         run_workload_command = self.my_workload_config.run_workload_command

pgopttune/workload/oltpbench.py

@@ -21,23 +21,16 @@ def __init__(self, postgres_server_config: PostgresServerConfig, oltpbench_confi
         self.config_hash = get_file_hash(oltpbench_config.oltpbench_config_path)
 
     def data_load(self):
-        if self._check_exist_backup_database():
-            # Recreate the database using the backed up database as a template
-            self._drop_database()
-            self._create_database_use_backup_database()
-        else:  # First data load
-            cwd = os.getcwd()
-            config_path = os.path.join(cwd, self.oltpbench_config.oltpbench_config_path)
-            data_load_cmd = "{}/oltpbenchmark -b {} -c {} --create=true --load=true".format(
-                self.oltpbench_config.oltpbench_path,
-                self.oltpbench_config.benchmark_kind,
-                config_path)
-            os.chdir(self.oltpbench_config.oltpbench_path)
-            logger.debug('run oltpbench data load command : {}'.format(data_load_cmd))
-            run_command(data_load_cmd)
-            os.chdir(cwd)
-            self._create_backup_database()  # backup database
-        self.vacuum_database()  # vacuum analyze
+        cwd = os.getcwd()
+        config_path = os.path.join(cwd, self.oltpbench_config.oltpbench_config_path)
+        data_load_cmd = "{}/oltpbenchmark -b {} -c {} --create=true --load=true".format(
+            self.oltpbench_config.oltpbench_path,
+            self.oltpbench_config.benchmark_kind,
+            config_path)
+        os.chdir(self.oltpbench_config.oltpbench_path)
+        logger.debug('run oltpbench data load command : {}'.format(data_load_cmd))
+        run_command(data_load_cmd)
+        os.chdir(cwd)
 
     def run(self, measurement_time_second: int = None):
         grep_string = "requests\/sec"

pgopttune/workload/pgbench.py

@@ -19,21 +19,14 @@ def __init__(self, postgres_server_config: PostgresServerConfig, pgbench_config:
         os.environ['PGPASSWORD'] = postgres_server_config.password
 
     def data_load(self):
-        if self._check_exist_backup_database():
-            # Recreate the database using the backed up database as a template
-            self._drop_database()
-            self._create_database_use_backup_database()
-        else:
-            data_load_cmd = "{}/pgbench -h {} -p {} -U {} {} -i -s {}".format(self.postgres_server_config.pgbin,
-                                                                              self.postgres_server_config.host,
-                                                                              self.postgres_server_config.port,
-                                                                              self.postgres_server_config.user,
-                                                                              self.postgres_server_config.database,
-                                                                              self.pgbench_config.scale_factor)
-            logger.debug('run pgbench data load command : {}'.format(data_load_cmd))
-            run_command(data_load_cmd)
-            self._create_backup_database()  # backup database
-        self.vacuum_database()  # vacuum analyze
+        data_load_cmd = "{}/pgbench -h {} -p {} -U {} {} -i -s {}".format(self.postgres_server_config.pgbin,
+                                                                          self.postgres_server_config.host,
+                                                                          self.postgres_server_config.port,
+                                                                          self.postgres_server_config.user,
+                                                                          self.postgres_server_config.database,
+                                                                          self.pgbench_config.scale_factor)
+        logger.debug('run pgbench data load command : {}'.format(data_load_cmd))
+        run_command(data_load_cmd)
 
     def run(self, measurement_time_second: int = None):
         grep_string = "excluding"

pgopttune/workload/star_schema_benchmark.py

@@ -18,17 +18,10 @@ def __init__(self, postgres_server_config: PostgresServerConfig, ssb_config: Sta
         self.backup_database_prefix = 'ssb_backup_'
 
     def data_load(self):
-        if self._check_exist_backup_database():
-            # Recreate the database using the backed up database as a template
-            self._drop_database()
-            self._create_database_use_backup_database()
-        else:
-            self._create_table()  # create table
-            self._truncate_table()  # truncate table
-            self._data_file_generate()  # create data file
-            self._load_data()  # data load
-            self._create_backup_database()  # backup database
-        self.vacuum_database()  # vacuum database
+        self._create_table()  # create table
+        self._truncate_table()  # truncate table
+        self._data_file_generate()  # create data file
+        self._load_data()  # data load
 
     def _create_table(self):
         for table_sql_filepath in self.ssb_config.table_sql_filepath_list:

pgopttune/workload/workload.py

@@ -41,7 +41,7 @@ def get_number_of_xact_commit(self):
                 xact_commit = cur.fetchone()["xact_commit"]
         return xact_commit
 
-    def _check_exist_backup_database(self):
+    def check_exist_backup_database(self):
         check_exist_backup_database_sql = "SELECT count(datname) FROM pg_database WHERE datname = %s"
         with get_pg_connection(dsn=self.postgres_server_config.dsn) as conn:
             conn.set_session(autocommit=True)
@@ -57,7 +57,7 @@ def _check_exist_backup_database(self):
             return False
 
     @retry(stop_max_attempt_number=5, wait_fixed=10000)
-    def _create_backup_database(self):
+    def create_backup_database(self):
         logger.debug(
             "Start backing up the database. Database : {} ".format(self.postgres_server_config.database))
 
@@ -74,7 +74,7 @@ def _get_backup_database_name(self):
         return self.backup_database_prefix + self.config_hash
 
     @retry(stop_max_attempt_number=5, wait_fixed=10000)
-    def _drop_database(self):
+    def drop_database(self):
         drop_database_sql = "DROP DATABASE {} ".format(self.postgres_server_config.database)
         backup_database_dsn = self._get_backup_database_dsn()
         with get_pg_connection(dsn=backup_database_dsn) as conn:
@@ -84,7 +84,7 @@ def _drop_database(self):
         logger.debug("The database has been deleted. Database : {}".format(self.postgres_server_config.database))
 
     @retry(stop_max_attempt_number=5, wait_fixed=10000)
-    def _create_database_use_backup_database(self):
+    def create_database_use_backup_database(self):
         create_database_use_backup_sql = "CREATE DATABASE {} TEMPLATE {}".format(self.postgres_server_config.database,
                                                                                  self._get_backup_database_name())
         backup_database_dsn = self._get_backup_database_dsn()