From 28b3e78b04ecac94fe721e295ba620c80a7ecdb8 Mon Sep 17 00:00:00 2001 From: Whisperity Date: Wed, 2 Aug 2017 15:17:24 +0200 Subject: [PATCH] Refactor database handlers to use a shared code, and support running multiple database engines at the same time [ci skip] --- libcodechecker/libhandlers/server.py | 28 +- .../server/client_db_access_server.py | 63 +-- libcodechecker/server/config_database.py | 361 ------------------ libcodechecker/server/config_db_model.py | 20 +- .../{run_database.py => database_handler.py} | 161 +++++--- .../server/product_db_access_handler.py | 4 - libcodechecker/server/run_db_model.py | 20 +- 7 files changed, 180 insertions(+), 477 deletions(-) delete mode 100644 libcodechecker/server/config_database.py rename libcodechecker/server/{run_database.py => database_handler.py} (66%) diff --git a/libcodechecker/libhandlers/server.py b/libcodechecker/libhandlers/server.py index 12728a9a9a..3283919213 100644 --- a/libcodechecker/libhandlers/server.py +++ b/libcodechecker/libhandlers/server.py @@ -25,9 +25,13 @@ from libcodechecker.logger import LoggerFactory from libcodechecker.logger import add_verbose_arguments from libcodechecker.server import client_db_access_server -from libcodechecker.server import config_database +from libcodechecker.server import database_handler from libcodechecker.server import instance_manager -from libcodechecker.server import run_database +from libcodechecker.server.config_db_model \ + import IDENTIFIER as CONFIG_META +from libcodechecker.server.run_db_model \ + import IDENTIFIER as RUN_META + LOG = LoggerFactory.get_new_logger('SERVER') @@ -430,33 +434,35 @@ def main(args): not os.path.exists(args.sqlite) and \ not os.path.exists(default_product_path) - sql_server = config_database.SQLServer.from_cmdline_args( - args, context.config_migration_root, check_env) + sql_server = database_handler.SQLServer.from_cmdline_args( + vars(args), CONFIG_META, context.config_migration_root, + interactive=True, env=check_env) LOG.debug("Connecting to product configuration database.") sql_server.connect(context.product_db_version_info, init=True) - # Start database viewer. - db_connection_string = sql_server.get_connection_string() - if create_default_product: # Create a default product and add it to the configuration database. LOG.debug("Create default product...") LOG.debug("Configuring schema and migration...") - prod_server = run_database.SQLiteDatabase( - default_product_path, context.run_migration_root, check_env) + + prod_server = database_handler.SQLiteDatabase( + default_product_path, RUN_META, + context.run_migration_root, check_env) prod_server.connect(context.run_db_version_info, init=True) + LOG.debug("Connecting database engine for default product") product_conn_string = prod_server.get_connection_string() LOG.debug("Default database created and connected.") client_db_access_server.add_initial_run_database( - db_connection_string, product_conn_string) + sql_server, product_conn_string) LOG.info("Product 'Default' at '{0}' created and set up." .format(default_product_path)) + # Start database viewer. checker_md_docs = os.path.join(context.doc_root, 'checker_md_docs') checker_md_docs_map = os.path.join(checker_md_docs, 'checker_doc_map.json') @@ -472,7 +478,7 @@ def main(args): try: client_db_access_server.start_server(package_data, args.view_port, - db_connection_string, + sql_server, suppress_handler, args.listen_address, context, diff --git a/libcodechecker/server/client_db_access_server.py b/libcodechecker/server/client_db_access_server.py index 8b39b9076f..50f303eb83 100644 --- a/libcodechecker/server/client_db_access_server.py +++ b/libcodechecker/server/client_db_access_server.py @@ -35,13 +35,13 @@ from libcodechecker import session_manager from libcodechecker.logger import LoggerFactory -from . import config_database +from . import database_handler from . import instance_manager -from . import run_database from client_auth_handler import ThriftAuthHandler from client_db_access_handler import ThriftRequestHandler -from product_db_access_handler import ThriftProductHandler from config_db_model import Product as ORMProduct +from product_db_access_handler import ThriftProductHandler +from run_db_model import IDENTIFIER as RUN_META LOG = LoggerFactory.get_new_logger('DB ACCESS') @@ -366,11 +366,15 @@ class Product(object): """ def __init__(self, orm_object, context, check_env): + """ + Set up a new managed product object for the configuration given. + """ self.__orm_object = orm_object self.__context = context self.__check_env = check_env - - self.connect() + self.__engine = None + self.__session = None + self.__connected = False @property def id(self): @@ -379,7 +383,7 @@ def id(self): @property def endpoint(self): """ - Returns the URL endpoint of the product. + Returns the accessible URL endpoint of the product. """ return self.__orm_object.endpoint @@ -429,10 +433,12 @@ def connect(self): # We need to connect to the database and perform setting up the # schema. LOG.debug("Configuring schema and migration...") - sql_server = run_database.SQLServer.from_connection_string( + sql_server = database_handler.SQLServer.from_connection_string( self.__orm_object.connection, + RUN_META, self.__context.config_migration_root, - self.__check_env) + interactive=False, + env=self.__check_env) try: sql_server.connect(self.__context.run_db_version_info, init=True) @@ -452,8 +458,13 @@ def connect(self): self.__connected = False def teardown(self): - """???""" - raise Exception("NYI!") + """ + Disposes the database connection to the product's backend. + """ + if not self.__connected: + return + + self.__engine.dispose() class CCSimpleHttpServer(HTTPServer): @@ -466,7 +477,7 @@ class CCSimpleHttpServer(HTTPServer): def __init__(self, server_address, RequestHandlerClass, - product_db_connection_string, + product_db_sql_server, pckg_data, suppress_handler, context, @@ -488,17 +499,15 @@ def __init__(self, # Create a database engine for the configuration database. LOG.debug("Creating database engine for CONFIG DATABASE...") - self.__engine = config_database.SQLServer.create_engine( - product_db_connection_string) - self.product_session = scoped_session(sessionmaker()) - self.product_session.configure(bind=self.__engine) + self.__engine = product_db_sql_server.create_engine() + self.product_session = sessionmaker(bind=self.__engine) # Load the initial list of products and create the connections. sess = self.product_session() products = sess.query(ORMProduct).all() for product in products: self.add_product(product) - self.product_session.remove() + sess.close() self.__request_handlers = ThreadPool(processes=10) try: @@ -535,19 +544,13 @@ def add_product(self, orm_product): by the server. """ if orm_product.endpoint in self.__products: - raise Exception("This product is already connected!") + raise Exception("This product is already configured!") - try: - conn = Product(orm_product, self.context, self.check_env) + LOG.info("Setting up product '{0}'".format(orm_product.endpoint)) + conn = Product(orm_product, self.context, self.check_env) + self.__products[conn.endpoint] = conn - LOG.info("Product '{0}' database connection set up..." - .format(orm_product.endpoint)) - self.__products[conn.endpoint] = conn - except Exception as ex: - LOG.error("The product '{0}' cannot be connected." - .format(orm_product.endpoint)) - LOG.error(ex.message) - # TODO: Some better error handling here. + conn.connect() def get_product(self, endpoint): """ @@ -604,7 +607,7 @@ def unregister_handler(pid): LOG.info("Webserver quit.") -def add_initial_run_database(config_connection, product_connection): +def add_initial_run_database(config_sql_server, product_connection): """ Create a default run database as SQLite in the config directory, and add it to the list of products in the config database specified by @@ -613,7 +616,7 @@ def add_initial_run_database(config_connection, product_connection): # Connect to the configuration database LOG.debug("Creating database engine for CONFIG DATABASE...") - __engine = config_database.SQLServer.create_engine(config_connection) + __engine = config_sql_server.create_engine() product_session = sessionmaker(bind=__engine) # Load the initial list of products and create the connections. @@ -628,6 +631,6 @@ def add_initial_run_database(config_connection, product_connection): "Default product created at server start.") sess.add(product) sess.commit() - product_session.remove() + sess.close() LOG.debug("Default product set up.") diff --git a/libcodechecker/server/config_database.py b/libcodechecker/server/config_database.py deleted file mode 100644 index 0b793b4b76..0000000000 --- a/libcodechecker/server/config_database.py +++ /dev/null @@ -1,361 +0,0 @@ -# ------------------------------------------------------------------------- -# The CodeChecker Infrastructure -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. -# ------------------------------------------------------------------------- -""" -Database server handling for the server's configuration database. -""" - -# TODO: Merge with run_database. - -from abc import ABCMeta, abstractmethod -import os -import sys -import threading - -from alembic import command, config -from alembic.util import CommandError -import sqlalchemy -from sqlalchemy import event -from sqlalchemy.engine.url import URL, make_url -from sqlalchemy.sql.elements import quoted_name - -from libcodechecker import host_check -from libcodechecker import pgpass -from libcodechecker import util -from libcodechecker.logger import LoggerFactory - -from config_db_model import CC_META -from config_db_model import DBVersion - -LOG = LoggerFactory.get_new_logger('CONFIG DB HANDLER') - - -class SQLServer(object): - """ - Abstract base class for database server handling. An SQLServer instance is - responsible for the connection management towards the database. - - SQLServer implementations are created via SQLServer.from_cmdline_args(). - - How to add a new database server implementation: - 1, Derive from SQLServer and implement the abstract methods - 2, Add/modify some command line options in CodeChecker.py - 3, Modify SQLServer.from_cmdline_args() in order to create an - instance of the new server type if needed - """ - - __metaclass__ = ABCMeta - - def __init__(self, migration_root): - """ - Sets self.migration_root. migration_root should be the path to the - alembic migration scripts. - """ - - self.migration_root = migration_root - - def _create_or_update_schema(self, use_migration=True): - """ - Creates or updates the database schema. The database server has to be - started before this method is called. - - If use_migration is True, this method runs an alembic upgrade to HEAD. - - In the False case, there is no migration support and only SQLAlchemy - meta data is used for schema creation. - - On error sys.exit(1) is called. - """ - - try: - db_uri = self.get_connection_string() - engine = SQLServer.create_engine(db_uri) - - LOG.debug("Update/create database schema") - if use_migration: - LOG.debug("Creating new database session") - session = CreateSession(engine) - connection = session.connection() - - cfg = config.Config() - cfg.set_main_option("script_location", self.migration_root) - cfg.attributes["connection"] = connection - command.upgrade(cfg, "head") - - session.commit() - else: - CC_META.create_all(engine) - - engine.dispose() - LOG.debug("Update/create database schema: Done") - return True - - except sqlalchemy.exc.SQLAlchemyError as alch_err: - LOG.error(str(alch_err)) - sys.exit(1) - except CommandError as cerr: - LOG.error("Database schema and CodeChecker is incompatible." - "Please update CodeChecker.") - LOG.debug(str(cerr)) - sys.exit(1) - - @abstractmethod - def connect(self, db_version_info, init=False): - """ - Starts the database server and initializes the database server. - - On init == True, this it also initializes the database data and schema - if needed. - - On error sys.exit(1) should be called. - """ - pass - - @abstractmethod - def get_connection_string(self): - """ - Returns the connection string for SQLAlchemy. - - DO NOT LOG THE CONNECTION STRING BECAUSE IT MAY CONTAIN THE PASSWORD - FOR THE DATABASE! - """ - pass - - @staticmethod - def create_engine(connection_string): - """ - Creates a new SQLAlchemy engine. - """ - - if make_url(connection_string).drivername == 'sqlite+pysqlite': - # FIXME: workaround for locking errors - return sqlalchemy.create_engine( - connection_string, - encoding='utf8', - connect_args={'timeout': 600}) - else: - return sqlalchemy.create_engine(connection_string, - encoding='utf8') - - @staticmethod - def from_cmdline_args(args, migration_root, env=None): - """ - Normally only this method is called form outside of this module in - order to instance the proper server implementation. - - Parameters: - args: the command line arguments from CodeChecker.py - migration_root: path to the database migration scripts - env: a run environment dictionary. - """ - - if not host_check.check_sql_driver(args.postgresql): - LOG.error("The selected SQL driver is not available!") - sys.exit(1) - - if args.postgresql: - LOG.debug("Using PostgreSQL:") - return PostgreSQLServer(migration_root, - args.dbaddress, - args.dbport, - args.dbusername, - args.dbname, - run_env=env) - else: - LOG.debug("Using SQLite:") - data_file = os.path.abspath(args.sqlite) - LOG.debug("Database at " + data_file) - return SQLiteDatabase(data_file, migration_root, run_env=env) - - def check_db_version(self, db_version_info, session=None): - """ - Checks the database version and prints an error message on database - version mismatch. - - - On mismatching or on missing version a sys.exit(1) is called. - - On missing DBVersion table, it returns False - - On compatible DB version, it returns True - - Parameters: - db_version_info (db_version.DBVersionInfo): required database - version. - session: an open database session or None. If session is None, a - new session is created. - """ - - try: - dispose_engine = False - if session is None: - engine = SQLServer.create_engine(self.get_connection_string()) - dispose_engine = True - session = CreateSession(engine) - else: - engine = session.get_bind() - - if not engine.has_table(quoted_name(DBVersion.__tablename__, - True)): - LOG.debug("Missing DBVersion table!") - return False - - version = session.query(DBVersion).first() - if version is None: - # Version is not populated yet - LOG.error("No version information found in the database.") - sys.exit(1) - elif not db_version_info.is_compatible(version.major, - version.minor): - LOG.error("Version mismatch. Expected database version: " + - str(db_version_info)) - version_from_db = 'v' + str(version.major) + '.' + str( - version.minor) - LOG.error("Version from the database is: " + version_from_db) - LOG.error("Please update your database.") - sys.exit(1) - - LOG.debug("Database version is compatible.") - return True - finally: - session.commit() - if dispose_engine: - engine.dispose() - - def _add_version(self, db_version_info, session=None): - """ - Fills the DBVersion table. - """ - - engine = None - if session is None: - engine = SQLServer.create_engine(self.get_connection_string()) - session = CreateSession(engine) - - expected = db_version_info.get_expected_version() - LOG.debug("Adding DB version: " + str(expected)) - - session.add(DBVersion(expected[0], expected[1])) - session.commit() - - if engine: - engine.dispose() - - LOG.debug("Adding DB version done!") - - -class PostgreSQLServer(SQLServer): - """ - Handler for PostgreSQL. - """ - - def __init__(self, migration_root, host, port, user, database, - password=None, run_env=None): - super(PostgreSQLServer, self).__init__(migration_root) - - self.host = host - self.port = port - self.user = user - self.database = database - self.password = password - self.run_env = run_env - - def _get_connection_string(self, database): - """ - Helper method for getting the connection string for the given database. - - database -- The user can force the database name in the returning - connection string. However the password, if any, provided e.g. in a - .pgpass file will be queried based on the database name which is given - as a command line argument, even if it has a default value. The reason - is that sometimes a connection with a common database name is needed, - (e.g. 'postgres'), which requires less user permission. - """ - - driver = host_check.get_postgresql_driver_name() - password = self.password - if driver == 'pg8000' and not password: - pfilepath = os.environ.get('PGPASSFILE') - if pfilepath: - password = pgpass.get_password_from_file(pfilepath, - self.host, - str(self.port), - self.database, - self.user) - - extra_args = {'client_encoding': 'utf8'} - return str(URL('postgresql+' + driver, - username=self.user, - password=password, - host=self.host, - port=str(self.port), - database=database, - query=extra_args)) - - def connect(self, db_version_info, init=False): - """ - Connect to a PostgreSQL instance with given path, host and port. - """ - - LOG.debug("Connecting to database...") - - LOG.debug("Checking if database is running at [{0}:{1}]" - .format(self.host, str(self.port))) - - check_db = ['psql', - '-h', self.host, - '-p', str(self.port), - '-U', self.user, - '-d', self.database, - '-c', 'SELECT version();'] - err, code = util.call_command(check_db, self.run_env) - - if code: - LOG.error("Database is not running, or cannot be connected to.") - LOG.error(err) - sys.exit(1) - - add_version = False - if init: - add_version = not self.check_db_version(db_version_info) - self._create_or_update_schema(use_migration=False) - - if add_version: - self._add_version(db_version_info) - - LOG.debug("Done. Connected to database.") - - def get_connection_string(self): - return self._get_connection_string(self.database) - - -class SQLiteDatabase(SQLServer): - """ - Handler for SQLite. - """ - - def __init__(self, data_file, migration_root, run_env=None): - super(SQLiteDatabase, self).__init__(migration_root) - - self.dbpath = data_file - self.run_env = run_env - - def _set_sqlite_pragma(dbapi_connection, connection_record): - cursor = dbapi_connection.cursor() - cursor.execute("PRAGMA foreign_keys=ON") - cursor.close() - - event.listen(Engine, 'connect', _set_sqlite_pragma) - - def connect(self, db_version_info, init=False): - if init: - add_version = not self.check_db_version(db_version_info) - self._create_or_update_schema(use_migration=False) - if add_version: - self._add_version(db_version_info) - - if not os.path.exists(self.dbpath): - LOG.error("Database (%s) is missing!" % self.dbpath) - sys.exit(1) - - def get_connection_string(self): - return str(URL('sqlite+pysqlite', None, None, None, None, self.dbpath)) diff --git a/libcodechecker/server/config_db_model.py b/libcodechecker/server/config_db_model.py index 9893992d7e..3b1950d852 100644 --- a/libcodechecker/server/config_db_model.py +++ b/libcodechecker/server/config_db_model.py @@ -6,12 +6,12 @@ """ SQLAlchemy ORM model for the product configuration database. """ + from __future__ import print_function from __future__ import unicode_literals from sqlalchemy import * from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import * CC_META = MetaData(naming_convention={ "ix": 'ix_%(column_0_label)s', @@ -85,16 +85,8 @@ def __init__(self, endpoint, conn_str, name=None): # self.name = name # self.is_group = is_group - -def CreateSchema(engine): - """ Creates the schema if it does not exists. - Do not check version or do migration yet. """ - Base.metadata.create_all(engine) - - -def CreateSession(engine): - """ Creates a scoped session factory that can act like a session. - The factory uses a thread_local registry, so every thread have - its own session. """ - SessionFactory = scoped_session(sessionmaker(bind=engine)) - return SessionFactory +IDENTIFIER = { + 'identifier': "ConfigDatabase", + 'orm_meta': CC_META, + 'version_class': DBVersion +} diff --git a/libcodechecker/server/run_database.py b/libcodechecker/server/database_handler.py similarity index 66% rename from libcodechecker/server/run_database.py rename to libcodechecker/server/database_handler.py index bfc4d24299..e467412536 100644 --- a/libcodechecker/server/run_database.py +++ b/libcodechecker/server/database_handler.py @@ -4,19 +4,19 @@ # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- """ -Database server handling for analysis databases. +Database connection handling to a database backend. """ -# TODO: Merge with config_database. - from abc import ABCMeta, abstractmethod import os +import threading from alembic import command, config from alembic.util import CommandError import sqlalchemy from sqlalchemy import event from sqlalchemy.engine.url import URL, make_url +from sqlalchemy.orm import sessionmaker from sqlalchemy.sql.elements import quoted_name from libcodechecker import host_check @@ -24,10 +24,7 @@ from libcodechecker import util from libcodechecker.logger import LoggerFactory -from run_db_model import CC_META -from run_db_model import DBVersion - -LOG = LoggerFactory.get_new_logger('RUN DB HANDLER') +LOG = LoggerFactory.get_new_logger('DATABASE HANDLER') class SQLServer(object): @@ -46,12 +43,15 @@ class SQLServer(object): __metaclass__ = ABCMeta - def __init__(self, migration_root): + def __init__(self, model_meta, migration_root): """ Sets self.migration_root. migration_root should be the path to the alembic migration scripts. + + Also sets the created class' model identifier to the given meta dict. """ + self.__model_meta = model_meta self.migration_root = migration_root def _create_or_update_schema(self, use_migration=True): @@ -68,13 +68,13 @@ def _create_or_update_schema(self, use_migration=True): """ try: - db_uri = self.get_connection_string() - engine = SQLServer.create_engine(db_uri) + engine = self.create_engine() - LOG.debug("Update/create database schema") + LOG.debug("Update/create database schema for {0}" + .format(self.__model_meta['identifier'])) if use_migration: LOG.debug("Creating new database session") - session = CreateSession(engine) + session = sessionmaker(bind=engine)() connection = session.connection() cfg = config.Config() @@ -83,8 +83,10 @@ def _create_or_update_schema(self, use_migration=True): command.upgrade(cfg, "head") session.commit() + session.close() else: - CC_META.create_all(engine) + LOG.debug("Creating full schema.") + self.__model_meta['orm_meta'].create_all(engine) engine.dispose() LOG.debug("Update/create database schema: Done") @@ -121,49 +123,105 @@ def get_connection_string(self): """ pass - @staticmethod - def create_engine(connection_string): + def get_model_identifier(self): + return self.__model_meta['identifier'] + + def _register_engine_hooks(self, engine): + """ + This method registers hooks, if needed, related to the engine created + by create_engine. + """ + pass + + def create_engine(self): """ Creates a new SQLAlchemy engine. """ - if make_url(connection_string).drivername == 'sqlite+pysqlite': + if make_url(self.get_connection_string()).drivername == \ + 'sqlite+pysqlite': # FIXME: workaround for locking errors - return sqlalchemy.create_engine(connection_string, - encoding='utf8', - connect_args={'timeout': 600}) + engine = sqlalchemy.create_engine(self.get_connection_string(), + encoding='utf8', + connect_args={'timeout': 600}) else: - return sqlalchemy.create_engine(connection_string, - encoding='utf8') + engine = sqlalchemy.create_engine(self.get_connection_string(), + encoding='utf8') + + self._register_engine_hooks(engine) + return engine @staticmethod - def from_connection_string(connection_string, migration_root, env=None): + def from_connection_string(connection_string, model_meta, migration_root, + interactive=False, env=None): """ Normally only this method is called form outside of this module in order to instance the proper server implementation. Parameters: args: the command line arguments from CodeChecker.py + model_meta: the meta identifier of the database model to use migration_root: path to the database migration scripts env: a run environment dictionary. """ url = make_url(connection_string) - if not host_check.check_sql_driver('postgresql' in url.drivername): - raise ValueError("The selected SQL driver is not available!") + # Create an args for from_cmdline_args. + args = {} if 'postgresql' in url.drivername: + args['postgresql'] = True + args['dbaddress'] = url.host + args['dbport'] = url.port + args['dbusername'] = url.username + args['dbpassword'] = url.password + args['dbname'] = url.database + elif 'sqlite' in url.drivername: + args['postgresql'] = False + args['sqlite'] = url.database + + return SQLServer.from_cmdline_args(args, model_meta, migration_root, + interactive, env) + + @staticmethod + def from_cmdline_args(args, model_meta, migration_root, + interactive=False, env=None): + """ + Normally only this method is called form outside of this module in + order to instance the proper server implementation. + + Parameters: + args: the command line arguments from CodeChecker.py, but as a + dictionary (if argparse.Namespace, use vars(args)). + model_meta: the meta identifier of the database model to use + migration_root: path to the database migration scripts + interactive: whether or not the database connection can be + interactive on the server's shell. + env: a run environment dictionary. + """ + + if not host_check.check_sql_driver(args['postgresql']): + LOG.error("The selected SQL driver is not available!") + raise IOError("The SQL driver to be used is not available!") + + if args['postgresql']: LOG.debug("Using PostgreSQL:") - return PostgreSQLServer(migration_root, - url.host, - url.port, - url.username, - url.database, + return PostgreSQLServer(model_meta, + migration_root, + args['dbaddress'], + args['dbport'], + args['dbusername'], + args['dbname'], + password=args['dbpassword'] + if 'dbpassword' in args else None, + interactive=interactive, run_env=env) else: LOG.debug("Using SQLite:") - LOG.debug("Database at " + url.database) - return SQLiteDatabase(url.database, migration_root, run_env=env) + data_file = os.path.abspath(args['sqlite']) + LOG.debug("Database at " + data_file) + return SQLiteDatabase(data_file, model_meta, + migration_root, run_env=env) def check_db_version(self, db_version_info, session=None): """ @@ -181,12 +239,14 @@ def check_db_version(self, db_version_info, session=None): new session is created. """ + DBVersion = self.__model_meta['version_class'] + try: dispose_engine = False if session is None: - engine = SQLServer.create_engine(self.get_connection_string()) + engine = self.create_engine() dispose_engine = True - session = CreateSession(engine) + session = sessionmaker(bind=engine)() else: engine = session.get_bind() @@ -214,6 +274,7 @@ def check_db_version(self, db_version_info, session=None): return True finally: session.commit() + session.close() if dispose_engine: engine.dispose() @@ -224,14 +285,16 @@ def _add_version(self, db_version_info, session=None): engine = None if session is None: - engine = SQLServer.create_engine(self.get_connection_string()) - session = CreateSession(engine) + engine = self.create_engine() + session = sessionmaker(bind=engine)() expected = db_version_info.get_expected_version() LOG.debug("Adding DB version: " + str(expected)) + DBVersion = self.__model_meta['version_class'] session.add(DBVersion(expected[0], expected[1])) session.commit() + session.close() if engine: engine.dispose() @@ -244,15 +307,16 @@ class PostgreSQLServer(SQLServer): Handler for PostgreSQL. """ - def __init__(self, migration_root, host, port, user, database, - password=None, run_env=None): - super(PostgreSQLServer, self).__init__(migration_root) + def __init__(self, model_meta, migration_root, host, port, user, database, + password=None, interactive=False, run_env=None): + super(PostgreSQLServer, self).__init__(model_meta, migration_root) self.host = host self.port = port self.user = user self.database = database self.password = password + self.interactive = interactive self.run_env = run_env def _get_connection_string(self, database): @@ -302,8 +366,11 @@ def connect(self, db_version_info, init=False): '-p', str(self.port), '-U', self.user, '-d', self.database, - '-c', 'SELECT version();', - '--no-password'] + '-c', 'SELECT version();'] + + if not self.interactive: + # Do not prompt for password in non-interactive mode. + check_db.append('--no-password') # If the user has a password pre-specified, use that for the # 'psql' call! @@ -339,18 +406,23 @@ class SQLiteDatabase(SQLServer): Handler for SQLite. """ - def __init__(self, data_file, migration_root, run_env=None): - super(SQLiteDatabase, self).__init__(migration_root) + def __init__(self, data_file, model_meta, migration_root, run_env=None): + super(SQLiteDatabase, self).__init__(model_meta, migration_root) self.dbpath = data_file self.run_env = run_env + def _register_engine_hooks(self, engine): + """ + SQLite databases need FOREIGN KEYs to be enabled, which is handled + through this connection hook. + """ def _set_sqlite_pragma(dbapi_connection, connection_record): cursor = dbapi_connection.cursor() cursor.execute("PRAGMA foreign_keys=ON") cursor.close() - event.listen(Engine, 'connect', _set_sqlite_pragma) + event.listen(engine, 'connect', _set_sqlite_pragma) def connect(self, db_version_info, init=False): if init: @@ -360,7 +432,8 @@ def connect(self, db_version_info, init=False): self._add_version(db_version_info) if not os.path.exists(self.dbpath): - raise IOError("Database (%s) is missing!" % self.dbpath) + LOG.error("Database file (%s) is missing!" % self.dbpath) + raise IOError("Database file (%s) is missing!" % self.dbpath) def get_connection_string(self): return str(URL('sqlite+pysqlite', None, None, None, None, self.dbpath)) diff --git a/libcodechecker/server/product_db_access_handler.py b/libcodechecker/server/product_db_access_handler.py index 74f6196e09..441c8b125f 100644 --- a/libcodechecker/server/product_db_access_handler.py +++ b/libcodechecker/server/product_db_access_handler.py @@ -7,9 +7,6 @@ Handle Thrift requests for the product manager service. """ -import base64 -import os - import sqlalchemy import shared @@ -20,7 +17,6 @@ from libcodechecker.profiler import timeit from config_db_model import * -from run_database import SQLServer LOG = LoggerFactory.get_new_logger('PRODUCT HANDLER') diff --git a/libcodechecker/server/run_db_model.py b/libcodechecker/server/run_db_model.py index bf268f88fe..b8bda9199c 100644 --- a/libcodechecker/server/run_db_model.py +++ b/libcodechecker/server/run_db_model.py @@ -4,8 +4,9 @@ # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- """ -ORM model. +SQLAlchemy ORM model for the analysis run storage database. """ + from __future__ import print_function from __future__ import unicode_literals @@ -263,15 +264,8 @@ class ReviewStatus(Base): date = Column(DateTime, nullable=False) -def CreateSchema(engine): - """ Creates the schema if it does not exists. - Do not check version or do migration yet. """ - Base.metadata.create_all(engine) - - -def CreateSession(engine): - """ Creates a scoped session factory that can act like a session. - The factory uses a thread_local registry, so every thread have - its own session. """ - SessionFactory = scoped_session(sessionmaker(bind=engine)) - return SessionFactory +IDENTIFIER = { + 'identifier': "RunDatabase", + 'orm_meta': CC_META, + 'version_class': DBVersion +}