Refactor database handlers to use a shared code, and support running …

…multiple database engines at the same time

[ci skip]

libcodechecker/libhandlers/server.py

@@ -25,9 +25,13 @@
 from libcodechecker.logger import LoggerFactory
 from libcodechecker.logger import add_verbose_arguments
 from libcodechecker.server import client_db_access_server
-from libcodechecker.server import config_database
+from libcodechecker.server import database_handler
 from libcodechecker.server import instance_manager
-from libcodechecker.server import run_database
+from libcodechecker.server.config_db_model \
+    import IDENTIFIER as CONFIG_META
+from libcodechecker.server.run_db_model \
+    import IDENTIFIER as RUN_META
+
 
 LOG = LoggerFactory.get_new_logger('SERVER')
 
@@ -430,33 +434,35 @@ def main(args):
                              not os.path.exists(args.sqlite) and \
                              not os.path.exists(default_product_path)
 
-    sql_server = config_database.SQLServer.from_cmdline_args(
-        args, context.config_migration_root, check_env)
+    sql_server = database_handler.SQLServer.from_cmdline_args(
+        vars(args), CONFIG_META, context.config_migration_root,
+        interactive=True, env=check_env)
 
     LOG.debug("Connecting to product configuration database.")
     sql_server.connect(context.product_db_version_info, init=True)
 
-    # Start database viewer.
-    db_connection_string = sql_server.get_connection_string()
-
     if create_default_product:
         # Create a default product and add it to the configuration database.
 
         LOG.debug("Create default product...")
         LOG.debug("Configuring schema and migration...")
-        prod_server = run_database.SQLiteDatabase(
-            default_product_path, context.run_migration_root, check_env)
+
+        prod_server = database_handler.SQLiteDatabase(
+            default_product_path, RUN_META,
+            context.run_migration_root, check_env)
         prod_server.connect(context.run_db_version_info, init=True)
+
         LOG.debug("Connecting database engine for default product")
         product_conn_string = prod_server.get_connection_string()
         LOG.debug("Default database created and connected.")
 
         client_db_access_server.add_initial_run_database(
-            db_connection_string, product_conn_string)
+            sql_server, product_conn_string)
 
         LOG.info("Product 'Default' at '{0}' created and set up."
                  .format(default_product_path))
 
+    # Start database viewer.
     checker_md_docs = os.path.join(context.doc_root, 'checker_md_docs')
     checker_md_docs_map = os.path.join(checker_md_docs,
                                        'checker_doc_map.json')
@@ -472,7 +478,7 @@ def main(args):
     try:
         client_db_access_server.start_server(package_data,
                                              args.view_port,
-                                             db_connection_string,
+                                             sql_server,
                                              suppress_handler,
                                              args.listen_address,
                                              context,

libcodechecker/server/client_db_access_server.py

@@ -35,13 +35,13 @@
 from libcodechecker import session_manager
 from libcodechecker.logger import LoggerFactory
 
-from . import config_database
+from . import database_handler
 from . import instance_manager
-from . import run_database
 from client_auth_handler import ThriftAuthHandler
 from client_db_access_handler import ThriftRequestHandler
-from product_db_access_handler import ThriftProductHandler
 from config_db_model import Product as ORMProduct
+from product_db_access_handler import ThriftProductHandler
+from run_db_model import IDENTIFIER as RUN_META
 
 LOG = LoggerFactory.get_new_logger('DB ACCESS')
 
@@ -366,11 +366,15 @@ class Product(object):
     """
 
     def __init__(self, orm_object, context, check_env):
+        """
+        Set up a new managed product object for the configuration given.
+        """
         self.__orm_object = orm_object
         self.__context = context
         self.__check_env = check_env
-
-        self.connect()
+        self.__engine = None
+        self.__session = None
+        self.__connected = False
 
     @property
     def id(self):
@@ -379,7 +383,7 @@ def id(self):
     @property
     def endpoint(self):
         """
-        Returns the URL endpoint of the product.
+        Returns the accessible URL endpoint of the product.
         """
         return self.__orm_object.endpoint
 
@@ -429,10 +433,12 @@ def connect(self):
         # We need to connect to the database and perform setting up the
         # schema.
         LOG.debug("Configuring schema and migration...")
-        sql_server = run_database.SQLServer.from_connection_string(
+        sql_server = database_handler.SQLServer.from_connection_string(
             self.__orm_object.connection,
+            RUN_META,
             self.__context.config_migration_root,
-            self.__check_env)
+            interactive=False,
+            env=self.__check_env)
 
         try:
             sql_server.connect(self.__context.run_db_version_info, init=True)
@@ -452,8 +458,13 @@ def connect(self):
             self.__connected = False
 
     def teardown(self):
-        """???"""
-        raise Exception("NYI!")
+        """
+        Disposes the database connection to the product's backend.
+        """
+        if not self.__connected:
+            return
+
+        self.__engine.dispose()
 
 
 class CCSimpleHttpServer(HTTPServer):
@@ -466,7 +477,7 @@ class CCSimpleHttpServer(HTTPServer):
     def __init__(self,
                  server_address,
                  RequestHandlerClass,
-                 product_db_connection_string,
+                 product_db_sql_server,
                  pckg_data,
                  suppress_handler,
                  context,
@@ -488,17 +499,15 @@ def __init__(self,
 
         # Create a database engine for the configuration database.
         LOG.debug("Creating database engine for CONFIG DATABASE...")
-        self.__engine = config_database.SQLServer.create_engine(
-            product_db_connection_string)
-        self.product_session = scoped_session(sessionmaker())
-        self.product_session.configure(bind=self.__engine)
+        self.__engine = product_db_sql_server.create_engine()
+        self.product_session = sessionmaker(bind=self.__engine)
 
         # Load the initial list of products and create the connections.
         sess = self.product_session()
         products = sess.query(ORMProduct).all()
         for product in products:
             self.add_product(product)
-        self.product_session.remove()
+        sess.close()
 
         self.__request_handlers = ThreadPool(processes=10)
         try:
@@ -535,19 +544,13 @@ def add_product(self, orm_product):
         by the server.
         """
         if orm_product.endpoint in self.__products:
-            raise Exception("This product is already connected!")
+            raise Exception("This product is already configured!")
 
-        try:
-            conn = Product(orm_product, self.context, self.check_env)
+        LOG.info("Setting up product '{0}'".format(orm_product.endpoint))
+        conn = Product(orm_product, self.context, self.check_env)
+        self.__products[conn.endpoint] = conn
 
-            LOG.info("Product '{0}' database connection set up..."
-                     .format(orm_product.endpoint))
-            self.__products[conn.endpoint] = conn
-        except Exception as ex:
-            LOG.error("The product '{0}' cannot be connected."
-                      .format(orm_product.endpoint))
-            LOG.error(ex.message)
-            # TODO: Some better error handling here.
+        conn.connect()
 
     def get_product(self, endpoint):
         """
@@ -604,7 +607,7 @@ def unregister_handler(pid):
     LOG.info("Webserver quit.")
 
 
-def add_initial_run_database(config_connection, product_connection):
+def add_initial_run_database(config_sql_server, product_connection):
     """
     Create a default run database as SQLite in the config directory,
     and add it to the list of products in the config database specified by
@@ -613,7 +616,7 @@ def add_initial_run_database(config_connection, product_connection):
 
     # Connect to the configuration database
     LOG.debug("Creating database engine for CONFIG DATABASE...")
-    __engine = config_database.SQLServer.create_engine(config_connection)
+    __engine = config_sql_server.create_engine()
     product_session = sessionmaker(bind=__engine)
 
     # Load the initial list of products and create the connections.
@@ -628,6 +631,6 @@ def add_initial_run_database(config_connection, product_connection):
                          "Default product created at server start.")
     sess.add(product)
     sess.commit()
-    product_session.remove()
+    sess.close()
 
     LOG.debug("Default product set up.")